diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000..026e4d0
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,45 @@
+name: docker
+
+on:
+ push:
+ branches:
+ - 'main'
+
+jobs:
+ docker:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Install GitVersion
+ uses: gittools/actions/gitversion/setup@v0.10.2
+ with:
+ versionSpec: '5.x'
+
+ - name: Run GitVersion
+ uses: gittools/actions/gitversion/execute@v0.10.2
+ with:
+ useConfigFile: true
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: |
+ dickwolff/export-to-ghostfolio:latest
+ dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}
diff --git a/.github/workflows/main.yml b/.github/workflows/frameworkTesting.yml
similarity index 100%
rename from .github/workflows/main.yml
rename to .github/workflows/frameworkTesting.yml
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..5532757
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,13 @@
+FROM node:20-alpine3.19
+
+WORKDIR /app
+
+COPY . .
+
+RUN npm install
+
+RUN mkdir /var/e2g-input
+RUN mkdir /var/e2g-output
+
+ENTRYPOINT [ "npm" ]
+CMD ["run", "watch"]
\ No newline at end of file
diff --git a/GitVersion.yml b/GitVersion.yml
new file mode 100644
index 0000000..67cbfb1
--- /dev/null
+++ b/GitVersion.yml
@@ -0,0 +1,15 @@
+next-version: 0.3.0
+assembly-informational-format: "{NuGetVersion}"
+mode: ContinuousDeployment
+branches:
+ master:
+ regex: main
+ mode: ContinuousDelivery
+ tag: ""
+ increment: Patch
+ feature:
+ regex: ^feature?[/-]
+ mode: ContinuousDelivery
+ tag: ""
+ increment: Patch
+ source-branches: ["main"]
diff --git a/README.md b/README.md
index 983f546..c8d95e9 100644
--- a/README.md
+++ b/README.md
@@ -14,11 +14,47 @@ This tool allows you to convert a multiple transaction exports (CSV) to an impor
Is your broker not in the list? Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)!
-## System requirements
+## How to use
-The tool requires you to install the latest LTS version of Node, which you can download [here](https://nodejs.org/en/download/). The tool can run on any OS on which you can install Node.
+You can run the tool on your local machine by cloning this repository. You can also run the tool inside a Docker container. See the runtime specific instructions below.
-## How to use
+## Docker
+
+
+View instructions
+
+### System requirements
+
+To run the Docker container you need to have [Docker](https://docs.docker.com/get-docker/) installed on your machine. The image is publishe to [Docker Hub](https://hub.docker.com/repository/docker/dickwolff/export-to-ghostfolio/). You can then run the image like:
+
+```
+docker run -d -v /C/.../docker_in:/var/e2g-input -v /C/.../docker_out:/var/e2g-output --env GHOSTFOLIO_ACCOUNT_ID=xxxxxxx dickwolff/export-to-ghostfolio
+```
+
+The following parameters can be given to the Docker run command.
+
+| Command | Optional | Description |
+| ------- | -------- | ----------- |
+| ` -v {local_in-folder}:/var/e2g-input` | N | The input folder where you put the files to be processed |
+| `-v {local_out_folder}:/var/e2g-output` | N | The output folder where the Ghostfolio import JSON will be placed. Also the input file will be moved here when an error ocurred while processing the file. |
+| `--env GHOSTFOLIO_ACCOUNT_ID=xxxxxxx` | N | Your Ghostolio account ID 1 |
+| `--env USE_POLLING=true` | Y | When set to true, the container will continously look for new files to process and the container will not stop. |
+| `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. |
+
+1: You can retrieve your Ghostfolio account ID by going to Accounts > select your account and copying the ID from the URL.
+
+
+
+
+
+## Run locally
+
+
+View instructions
+
+### System requirements
+
+The tool requires you to install the latest LTS version of Node, which you can download [here](https://nodejs.org/en/download/). The tool can run on any OS on which you can install Node.
### Download transaction export
@@ -41,6 +77,7 @@ Login to your Finpension account. Select your portfolio from the landing page. T
Login to your Swissquote account. From the bar menu click on “Transactions”. Select the desired time period as well as types and then select the “export CSV” button to the right.
#### Schwab
+
Login to your Schwab account. Go to “Accounts” then “History”. Select the account you want to download details from. Select the “Date Range” and select “Export” (csv). Save the file.
@@ -60,7 +97,7 @@ The repository contains a sample `.env` file. Rename this from `.env.sample`.
- Optionally you can enable debug logging by setting the `DEBUG_LOGGING` variable to `TRUE`.
-You can now run `npm run start [exporttype]`. See the table with run commands below. The tool will open your export and will convert this. It retrieves the symbols that are supported with YAHOO Finance (e.g. for European stocks like `ASML`, it will retrieve `ASML.AS` by the corresponding ISIN).
+You can now run `npm run start [exporttype]`. See the table with run commands below. The tool will open your export and will convert this. It retrieves the symbols that are supported with YAHOO Finance (e.g. for European stocks like `ASML`, it will retrieve `ASML.AS` by the corresponding ISIN).
| Exporter | Run command |
| ----------- | ----------------------------------- |
@@ -70,6 +107,10 @@ You can now run `npm run start [exporttype]`. See the table with run commands be
| Swissquote | `run start swissquote` (or `sq`) |
| Schwab | `run start schwab` |
+
+
+## Import to Ghostfolio
+
The export file can now be imported in Ghostfolio by going to Portfolio > Activities and pressing the 3 dots at the top right of the table. Since Ghostfolio 1.221.0, you can now preview the import and validate the data has been converted correctly. When it is to your satisfaction, press import to add the activities to your portfolio.
diff --git a/nodemon.json b/nodemon.json
deleted file mode 100644
index 5131f09..0000000
--- a/nodemon.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "watch": ["src"],
- "ext": ".ts,.js",
- "ignore": [],
- "exec": "ts-node ./src/index.ts"
- }
\ No newline at end of file
diff --git a/package.json b/package.json
index e3c8bcb..d1ef420 100644
--- a/package.json
+++ b/package.json
@@ -2,9 +2,9 @@
"name": "export-to-ghostfolio",
"version": "1.0.0",
"description": "Convert multiple broker exports to Ghostfolio import",
- "main": "index.js",
"scripts": {
- "start": "nodemon",
+ "start": "ts-node ./src/manual.ts",
+ "watch": "ts-node ./src/watcher.ts",
"test": "jest --coverage"
},
"author": "Dick Wolff",
@@ -16,15 +16,15 @@
"@types/jest": "^29.5.11",
"@types/node": "^20.10.4",
"jest": "^29.7.0",
- "nodemon": "^3.0.2",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"typescript": "^5.3.3"
},
"dependencies": {
"@types/cli-progress": "^3.11.5",
+ "chokidar": "^3.5.3",
"cli-progress": "^3.12.0",
- "cross-fetch": "^4.0.0",
+ "closest-match": "^1.3.3",
"csv-parse": "^5.5.2",
"dayjs": "^1.11.10",
"dotenv": "^16.3.1",
diff --git a/src/converter.ts b/src/converter.ts
new file mode 100644
index 0000000..a20de21
--- /dev/null
+++ b/src/converter.ts
@@ -0,0 +1,84 @@
+import path from "path";
+import * as fs from "fs";
+import { GhostfolioExport } from "./models/ghostfolioExport";
+import { DeGiroConverter } from "./converters/degiroConverter";
+import { SchwabConverter } from "./converters/schwabConverter";
+import { DeGiroConverterV2 } from "./converters/degiroConverterV2";
+import { AbstractConverter } from "./converters/abstractconverter";
+import { Trading212Converter } from "./converters/trading212Converter";
+import { SwissquoteConverter } from "./converters/swissquoteConverter";
+import { FinpensionConverter } from "./converters/finpensionConverter";
+
+export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
+
+ // Verify if Ghostolio account ID is set (because without it there can be no valid output).
+ if (!process.env.GHOSTFOLIO_ACCOUNT_ID) {
+ return errorCallback(new Error("Environment variable GHOSTFOLIO_ACCOUNT_ID not set!"));
+ }
+
+ const converterTypeLc = converterType.toLocaleLowerCase();
+
+ // Determine convertor type.
+ const converter = createConverter(converterTypeLc);
+
+ // Map the file to a Ghostfolio import.
+ converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => {
+
+ console.log("[i] Processing complete, writing to file..")
+
+ // Write result to file.
+ const outputFileName = path.join(outputFilePath, `ghostfolio-${converterTypeLc}.json`);
+ const fileContents = JSON.stringify(result);
+ fs.writeFileSync(outputFileName, fileContents, { encoding: "utf-8" });
+
+ console.log(`[i] Wrote data to '${outputFileName}.json'!`);
+
+ completionCallback();
+
+ }, (error) => errorCallback(error));
+}
+
+function createConverter(converterType: string): AbstractConverter {
+
+ let converter: AbstractConverter;
+
+ switch (converterType) {
+ case "t212":
+ case "trading212":
+ console.log("[i] Processing file using Trading212 converter");
+ converter = new Trading212Converter();
+ break;
+ case "degiro":
+ console.log("[i] Processing file using DeGiro converter");
+ console.log("[i] NOTE: There is a new version available of the DeGiro converter");
+ console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees.");
+ console.log("[i] The new converter is currently in beta and we're looking for your feedback!");
+ console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'.");
+ converter = new DeGiroConverter();
+ break;
+ case "degiro-v2":
+ console.log("[i] Processing file using DeGiro converter (V2 Beta)");
+ console.log("[i] NOTE: You are running a converter that is currently in beta.");
+ console.log("[i] If you have any issues, please report them on GitHub. Many thanks!");
+ converter = new DeGiroConverterV2();
+ break;
+ case "fp":
+ case "finpension":
+ console.log("[i] Processing file using Finpension converter");
+ converter = new FinpensionConverter();
+ break;
+ case "sq":
+ case "swissquote":
+ console.log("[i] Processing file using Swissquote converter");
+ converter = new SwissquoteConverter();
+ break;
+ case "schwab":
+ console.log("[i] Processing file using Schwab converter");
+ converter = new SchwabConverter();
+ break;
+ default:
+ throw new Error(`Unknown converter '${converterType}' provided`);
+ }
+
+ return converter;
+}
diff --git a/src/converters/degiroConverterV2.test.ts b/src/converters/degiroConverterV2.test.ts
index a034063..2a647f9 100644
--- a/src/converters/degiroConverterV2.test.ts
+++ b/src/converters/degiroConverterV2.test.ts
@@ -1,6 +1,6 @@
import { DeGiroConverterV2 } from "./degiroConverterV2";
-describe("degiroConverter", () => {
+describe("degiroConverterV2", () => {
it("should construct", () => {
diff --git a/src/converters/schwabConverter.test.ts b/src/converters/schwabConverter.test.ts
new file mode 100644
index 0000000..79a45dd
--- /dev/null
+++ b/src/converters/schwabConverter.test.ts
@@ -0,0 +1,13 @@
+import { SchwabConverter } from "./schwabConverter";
+
+describe("SchwabConverter", () => {
+
+ it("should construct", () => {
+
+ // Act
+ const sut = new SchwabConverter();
+
+ // Asssert
+ expect(sut).toBeTruthy();
+ });
+});
diff --git a/src/index.ts b/src/index.ts
deleted file mode 100644
index f1da9c4..0000000
--- a/src/index.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-import * as fs from "fs";
-import { GhostfolioExport } from "./models/ghostfolioExport";
-import { DeGiroConverter } from "./converters/degiroConverter";
-import { DeGiroConverterV2 } from "./converters/degiroConverterV2";
-import { AbstractConverter } from "./converters/abstractconverter";
-import { Trading212Converter } from "./converters/trading212Converter";
-import { SchwabConverter } from "./converters/schwabConverter";
-import { SwissquoteConverter } from "./converters/swissquoteConverter";
-import { FinpensionConverter } from "./converters/finpensionConverter";
-
-require("dotenv").config();
-
-// Define import file path.
-const inputFile = process.env.INPUT_FILE;
-
-let converter: AbstractConverter;
-
-// Determine convertor type.
-switch (process.argv[2].toLocaleLowerCase()) {
- case "t212":
- case "trading212":
- console.log("[i] Processing file using Trading212 converter");
- converter = new Trading212Converter();
- break;
- case "degiro":
- console.log("[i] Processing file using DeGiro converter");
- console.log("[i] NOTE: There is a new version available of the DeGiro converter");
- console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees.");
- console.log("[i] The new converter is currently in beta and we're looking for your feedback!");
- console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'.");
- converter = new DeGiroConverter();
- break;
- case "degiro-v2":
- console.log("[i] Processing file using DeGiro converter (V2 Beta)");
- console.log("[i] NOTE: You are running a converter that is currently in beta.");
- console.log("[i] If you have any issues, please report them on GitHub. Many thanks!");
- converter = new DeGiroConverterV2();
- break;
- case "fp":
- case "finpension":
- console.log("[i] Processing file using Finpension converter");
- converter = new FinpensionConverter();
- break;
- case "sq":
- case "swissquote":
- console.log("[i] Processing file using Swissquote converter");
- converter = new SwissquoteConverter();
- break;
- case "schwab":
- console.log("[i] Processing file using Schwab converter");
- converter = new SchwabConverter();
- break;
- default:
- throw new Error(`Unknown converter '${process.argv[2].toLocaleLowerCase()}' provided`);
-}
-
-// Map the file to a Ghostfolio import.
-converter.readAndProcessFile(inputFile, (result: GhostfolioExport) => {
-
- console.log("[i] Processing complete, writing to file..")
-
- // Write result to file.
- const fileContents = JSON.stringify(result);
- fs.writeFileSync(`ghostfolio-${process.argv[2].toLocaleLowerCase()}.json`, fileContents, { encoding: "utf-8" });
-
- console.log(`[i] Wrote data to 'ghostfolio-${process.argv[2].toLocaleLowerCase()}.json'!`);
-}, () => {});
diff --git a/src/manual.ts b/src/manual.ts
new file mode 100644
index 0000000..1896e67
--- /dev/null
+++ b/src/manual.ts
@@ -0,0 +1,16 @@
+import { createAndRunConverter } from "./converter";
+
+// Check if converter was specified.
+if (process.argv.length != 3) {
+ console.log("[e] Invalid run command: converter not specified!");;
+}
+else {
+
+ require("dotenv").config();
+
+ // Define import file path.
+ const inputFile = process.env.INPUT_FILE;
+
+ // Determine convertor type and run conversion.
+ createAndRunConverter(process.argv[2].toLocaleLowerCase(), inputFile, ".", () => { }, () => { });
+}
diff --git a/src/watcher.ts b/src/watcher.ts
new file mode 100644
index 0000000..91ed1c9
--- /dev/null
+++ b/src/watcher.ts
@@ -0,0 +1,89 @@
+import path from "path";
+import * as fs from "fs";
+import chokidar from "chokidar";
+import * as matcher from "closest-match";
+import { createAndRunConverter } from "./converter";
+
+// Define input and output.
+const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/e2g-input";
+const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output";
+const usePolling = Boolean(process.env.USE_POLLING) || false;
+
+console.log(`[i] Watching ${inputFolder}${usePolling ? " (using polling)" : ""}..`);
+
+let isProcessing = false;
+
+chokidar
+ .watch(inputFolder, { usePolling: usePolling })
+ .on("add", filePath => {
+
+ isProcessing = true;
+
+ console.log(`[i] Found ${path.basename(filePath)}!`);
+
+ const fileContents = fs.readFileSync(filePath, "utf-8");
+
+ const closestMatch = matcher.closestMatch(fileContents.split("\n")[0], [...headers.keys()]);
+
+ let converterKey = closestMatch as string;
+
+ // If multiple matches were found (type would not be 'string'), pick the first.
+ if (typeof closestMatch !== "string") {
+ converterKey = closestMatch[0];
+ }
+
+ const converter = headers.get(converterKey);
+ console.log(`[i] Determined the file type to be of kind '${converter}'.`);
+
+ // Determine convertor type and run conversion.
+ createAndRunConverter(converter, filePath, outputFolder,
+ () => {
+
+ // After conversion was succesful, remove input file.
+ console.log(`[i] Finished converting ${path.basename(filePath)}, removing file..`);
+ fs.rmSync(filePath);
+
+ isProcessing = false;
+
+ if (!usePolling) {
+ console.log("[i] Stop container as usePolling is set to false..");
+ process.exit(0);
+ }
+
+ }, (err) => {
+
+ console.log("[e] An error ocurred while processing.");
+ console.log(`[e] Error details: ${err}`);
+
+ // Move file with errors to output folder so it can be fixed manually.
+ console.log("[e] Moving file to output..");
+ const errorFilePath = path.join(outputFolder, path.basename(filePath));
+ fs.copyFileSync(filePath, errorFilePath);
+ fs.rmSync(filePath);
+
+ isProcessing = false;
+
+ if (!usePolling) {
+ console.log("[i] Stop container as usePolling is set to false..");
+ process.exit(0);
+ }
+ });
+ })
+ .on("ready", () => {
+
+ // When polling was not set to true (thus runOnce) and there is no file currently being processed, stop the container.
+ setTimeout(() => {
+ if (!usePolling && !isProcessing) {
+ console.log("[i] Found no file to convert, stop container as usePolling is set to false..");
+ process.exit(0);
+ }
+ }, 5000);
+ });
+
+// Prep header set.
+const headers: Map = new Map();
+headers.set(`Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id`, "degiro");
+headers.set(`Date;Category;"Asset Name";ISIN;"Number of Shares";"Asset Currency";"Currency Rate";"Asset Price in CHF";"Cash Flow";Balance`, "finpension");
+headers.set(`Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount`, "schwab");
+headers.set(`Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency`, "swissquote");
+headers.set(`Action,Time,ISIN,Ticker,Name,No. of shares,Price / share,Currency (Price / share),Exchange rate,Result,Currency (Result),Total,Currency (Total),Withholding tax,Currency (Withholding tax),Notes,ID,Currency conversion fee`, "trading212");