Merge branch 'main' into feature/Add-File-Cache

dickwolff · Feb 9, 2024 · 72a3b82 · 72a3b82
2 parents 97f87a0 + 6380880
commit 72a3b82
Show file tree

Hide file tree

Showing 25 changed files with 762 additions and 258 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -4,7 +4,12 @@ run-name: Build Docker Container
 on:
   push:
     branches:
-      - "main"
+      - main
+    paths:
+      - "src/**"
+  pull_request:
+    branches:
+      - main
     paths:
       - "src/**"
 
@@ -39,10 +44,21 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push (PR)
         uses: docker/build-push-action@v5
+        if: github.event_name == 'pull_request'
+        with:
+          push: true
+          tags: |
+            dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}-beta
+            
+      - name: Build and push (main)
+        uses: docker/build-push-action@v5
+        if: github.event_name != 'pull_request'
         with:          
-          push: ${{ github.event_name != "pull_request" }}
+          push: true
           tags: |
             dickwolff/export-to-ghostfolio:latest 
             dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}
+
+            
diff --git a/.github/workflows/frameworkTesting.yml b/.github/workflows/frameworkTesting.yml
@@ -11,7 +11,8 @@ on:
   pull_request:
     branches:
       - main
-
+    paths:
+      - "src/**"
 
 jobs:
   run-converter-tests:

diff --git a/GitVersion.yml b/GitVersion.yml
@@ -1,4 +1,4 @@
-next-version: 0.3.0
+next-version: 0.4.0
 assembly-informational-format: "{NuGetVersion}"
 mode: ContinuousDeployment
 branches:

diff --git a/README.md b/README.md
@@ -1,31 +1,75 @@
 # Export to Ghostfolio
 
-[![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://www.buymeacoffee.com/dickw0lff) &nbsp;
-[![Bitcoin](https://img.shields.io/badge/Bitcoin-000?style=for-the-badge&logo=bitcoin&logoColor=white)](bc1qsfkpeq8k3zav9g5w7s57gz2l84ju50uu8xqngreuxqas5tz2jrrsndgyth) &nbsp;
-[![Github-sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/dickwolff) 
+[![Github-sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/dickwolff) &nbsp;
+[![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://www.buymeacoffee.com/dickw0lff) 
+
 
 This tool allows you to convert a multiple transaction exports (CSV) to an import file that can be read by [Ghostfolio](https://github.com/ghostfolio/ghostfolio/). Currently there is support for:
 
 - [Trading 212](https://trading212.com)
 - [DEGIRO](https://degiro.com)
 - [Finpension](https://finpension.ch)
 - [Swissquote](https://en.swissquote.com/)
-- [Schwab]()
+- [Schwab](https://www.schwab.com)
+- [eToro](https://www.etoro.com/)
 
 Is your broker not in the list? Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)!
 
+## Download transaction export
+
+See the transaction export instructions for each of the supported brokers below. 
+
+<details>
+<summary>View transaction export instructions</summary>
+
+### Trading 212
+
+Login to your Trading 212 account and create an export file (via History > Download icon). Choose the period from which you wish to export your history and click download.
+
+### DEGIRO
+
+Login to your DEGIRO account and create an export file (via Inbox > Account Overview, see image below). Choose the period from which you wish to export your history and click download.
+
+![Export instructions for DEGIRO](./assets/export-degiro.jpg)
+
+### Finpension
+
+Login to your Finpension account. Select your portfolio from the landing page. Then to the right of the screen select “Transactions”, on the following page to the right notice “transaction report (CSV-file)” and click to email or click to download locally.
+
+### Swissquote
+
+Login to your Swissquote account. From the bar menu click on “Transactions”. Select the desired time period as well as types and then select the “export CSV” button to the right.
+
+### Schwab
+
+Login to your Schwab account. Go to “Accounts” then “History”. Select the account you want to download details from. Select the “Date Range” and select “Export” (csv). Save the file.
+
+![Export instructions for Schwab](./assets/export-schwab.jpg)
+
+### eToro
+
+Login to your eToro account and navigate to "Portfolio". Then select "History" in the top menu. Next, click on the icon on the far right and select "Account statement". Choose the dates of interest and click "Create". On the next page, click on the Excel icon on the top right to download the file. After downloading, open the file in Excel and delete all the tabs except the "Account Activity" tab. Then use Excel to convert the file to CSV.
+
+</details>
+
 ## How to use
 
 You can run the tool on your local machine by cloning this repository. You can also run the tool inside a Docker container. See the runtime specific instructions below.
 
 ## Docker
 
+[![Docker Pulls](https://img.shields.io/docker/pulls/dickwolff/export-to-ghostfolio?style=for-the-badge)](https://hub.docker.com/r/dickwolff/export-to-ghostfolio)
+
 <details>
 <summary>View instructions</summary>
 
 ### System requirements
 
-To run the Docker container you need to have [Docker](https://docs.docker.com/get-docker/) installed on your machine. The image is publishe to [Docker Hub](https://hub.docker.com/r/dickwolff/export-to-ghostfolio). Contrary to the locally run version of the tool, the containerized version tries to determine which file type to process by looking to the header line inside the file. So there is no need to specify which converter to use.
+To run the Docker container you need to have [Docker](https://docs.docker.com/get-docker/) installed on your machine. The image is published to [Docker Hub](https://hub.docker.com/r/dickwolff/export-to-ghostfolio). 
+
+### How to use
+
+Contrary to the locally run version of the tool, the containerized version tries to determine which file type to process by looking to the header line inside the file. So there is no need to specify which converter to use.
 
  You can then run the image like:
 
@@ -42,6 +86,7 @@ The following parameters can be given to the Docker run command.
 | `--env GHOSTFOLIO_ACCOUNT_ID=xxxxxxx` | N | Your Ghostolio account ID <sup>1</sup> |
 | `--env USE_POLLING=true` | Y | When set to true, the container will continously look for new files to process and the container will not stop. |
 | `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. |
+| `--env FORCE_DEGIRO_V2=true` | Y | When set to true, the converter will use the DEGIRO V2 converter (currently in beta) when a DEGIRO file was found. |
 
 1: You can retrieve your Ghostfolio account ID by going to Accounts > select your account and copying the ID from the URL.
 
@@ -58,32 +103,6 @@ The following parameters can be given to the Docker run command.
 
 The tool requires you to install the latest LTS version of Node, which you can download [here](https://nodejs.org/en/download/). The tool can run on any OS on which you can install Node.
 
-### Download transaction export
-
-#### Trading 212
-
-Login to your Trading 212 account and create an export file (via History > Download icon). Choose the period from which you wish to export your history and click download.
-
-#### DEGIRO
-
-Login to your DEGIRO account and create an export file (via Inbox > Account Overview, see image below). Choose the period from which you wish to export your history and click download.
-
-![Export instructions for DEGIRO](./assets/export-degiro.jpg)
-
-#### Finpension
-
-Login to your Finpension account. Select your portfolio from the landing page. Then to the right of the screen select “Transactions”, on the following page to the right notice “transaction report (CSV-file)” and click to email or click to download locally.
-
-#### Swissquote
-
-Login to your Swissquote account. From the bar menu click on “Transactions”. Select the desired time period as well as types and then select the “export CSV” button to the right.
-
-#### Schwab
-
-Login to your Schwab account. Go to “Accounts” then “History”. Select the account you want to download details from. Select the “Date Range” and select “Export” (csv). Save the file.
-
-![Export instructions for Schwab](./assets/export-schwab.jpg)
-
 ### Use the tool
 
 Next, clone the repo to your local machine and open with your editor of choice (e.g. Visual Studio Code).

diff --git a/sample-degiro-export.csv b/sample-degiro-export.csv
@@ -37,4 +37,7 @@ Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id
 01-08-2023,10:03,01-08-2023,ISHARES NASDAQ US BIOTECHNOLOGY ETF,IE00BYXG2H39,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-1.00,EUR,5.54,4c29dd81-bb01-40fa-8dda-aae3d05dae79
 01-08-2023,10:03,01-08-2023,ISHARES NASDAQ US BIOTECHNOLOGY ETF,IE00BYXG2H39,"Verkoop 1 @ 5,416 EUR",,EUR,5.42,EUR,6.54,4c29dd81-bb01-40fa-8dda-aae3d05dae79
 25-07-2023,14:31,25-07-2023,ISHARES KOREA,IE00B0M63391,DEGIRO Transactiekosten en/of kosten van derden,,EUR,-1.00,EUR,44.92,d4f45240-d763-4b95-8590-2ba158d38207
-25-07-2023,14:31,25-07-2023,ISHARES KOREA,IE00B0M63391,"Koop 1 @ 42,53 EUR",,EUR,-42.53,EUR,45.92,d4f45240-d763-4b95-8590-2ba158d38207
+25-07-2023,14:31,25-07-2023,ISHARES KOREA,IE00B0M63391,"Koop 1 @ 42,53 EUR",,EUR,-42.53,EUR,45.92,d4f45240-d763-4b95-8590-2ba158d38207
+15-05-2019,09:05,15-05-2019,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 6 ISHARES MSCI WOR A@49,785 EUR (IE00B4L5Y983)",,EUR,-298.71,EUR,0.64,a47e2746-bfbd-4654-bd6c-5e58e470d32f
+02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,Comissões de transação DEGIRO e/ou taxas de terceiros,,EUR,-1.00,EUR,2.54,7b377a93-5695-4131-8954-5c78996fbed4
+02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 1 ISHARES MSCI WOR A@82,055 EUR (IE00B4L5Y983)",,EUR,-82.06,EUR,3.54,7b377a93-5695-4131-8954-5c78996fbed4
diff --git a/sample-etoro-export.csv b/sample-etoro-export.csv
@@ -0,0 +1,22 @@
+Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA
+01/01/2024 05:50:54,Interest Payment,,0.08,-,0.08,"4,581.91",0.00,-,,0.00
+02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00
+02/01/2024 00:10:33,Dividend,NKE/USD,0.16,-,0.16,"4,581.91",99.60,2446419144,Stocks,0.00
+03/01/2024 00:04:03,Dividend,LRCX/USD,0.51,-,0.51,"4,581.91",100.11,1434892477,Stocks,0.00
+03/01/2024 13:09:09,Withdraw Fee,-,0.00,-,0.00,"4,581.91",100.11,-,,0.00
+03/01/2024 13:09:09,Withdrawal Conversion Fee,-,-1.36,-,0.00,"4,581.91",0.00,-,,0.00
+03/01/2024 13:09:09,Withdraw Request,-,-100.11,-,-100.11,"4,581.91",0.00,-,,0.00
+07/01/2024 13:21:57,Dividend,NXPI/USD,0.28,-,0.28,"4,581.91",0.00,1446737312,Stocks,0.00
+07/01/2024 13:21:57,Dividend,NXPI/USD,0.04,-,0.04,"4,581.91",0.00,1453559680,Stocks,0.00
+09/01/2024 15:30:40,Position closed,OLED/USD,18.43,0.102626,7.37,"4,581.91",0.00,2355395242,Stocks,0.00
+09/01/2024 15:36:54,Position closed,AMAT/USD,0.38,0.002556,0.04,"4,581.91",0.00,2596568830,Stocks,0.00
+09/01/2024 15:37:16,Open Position,AMD/USD,49.88,0.337209,0.00,"4,581.91",0.00,2596572937,Stocks,0.00
+09/01/2024 15:37:25,Open Position,SLAB/USD,27.37,0.219575,0.00,"4,581.91",0.00,2596576844,Stocks,0.00
+09/01/2024 15:40:09,Open Position,SWKS/USD,93.62,0.888488,0.00,"4,581.91",0.00,2596634782,Stocks,0.00
+10/01/2024 00:11:24,Dividend,DIS/USD,0.14,-,0.14,"4,581.91",0.00,1003296596,Stocks,0.00
+10/01/2024 00:11:24,Dividend,DIS/USD,0.29,-,0.29,"4,581.91",0.00,1068527084,Stocks,0.00
+16/01/2024 00:17:08,Dividend,HST/USD,0.07,-,0.07,"4,581.91",0.80,2565075017,Stocks,0.00
+16/01/2024 20:08:09,Position closed,AMD/USD,61.27,0.388423,38.57,"4,581.91",0.80,2602058508,Stocks,0.00
+17/01/2024 00:04:27,Dividend,KER/EUR,0.03,-,0.03,"4,581.91",0.80,697085768,Stocks,0.00
+17/01/2024 00:04:28,Dividend,KER/EUR,0.01,-,0.01,"4,581.91",0.80,831716072,Stocks,0.00
+17/01/2024 00:04:28,Dividend,KER/EUR,0.01,-,0.01,"4,581.91",0.80,1074146904,Stocks,0.00
diff --git a/src/converter.ts b/src/converter.ts
@@ -1,13 +1,16 @@
 import path from "path";
 import * as fs from "fs";
 import { GhostfolioExport } from "./models/ghostfolioExport";
+import { EtoroConverter } from "./converters/etoroConverter";
 import { DeGiroConverter } from "./converters/degiroConverter";
 import { SchwabConverter } from "./converters/schwabConverter";
 import { DeGiroConverterV2 } from "./converters/degiroConverterV2";
 import { AbstractConverter } from "./converters/abstractconverter";
 import { Trading212Converter } from "./converters/trading212Converter";
 import { SwissquoteConverter } from "./converters/swissquoteConverter";
 import { FinpensionConverter } from "./converters/finpensionConverter";
+import { YahooFinanceService } from "./yahooFinanceService";
+
 
 export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
 
@@ -40,41 +43,47 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri
 
 function createConverter(converterType: string): AbstractConverter {
 
+    const yahooFinanceService = new YahooFinanceService();
+
     let converter: AbstractConverter;
 
     switch (converterType) {
         case "t212":
         case "trading212":
             console.log("[i] Processing file using Trading212 converter");
-            converter = new Trading212Converter();
+            converter = new Trading212Converter(yahooFinanceService);
             break;
         case "degiro":
             console.log("[i] Processing file using DeGiro converter");
             console.log("[i] NOTE: There is a new version available of the DeGiro converter");
             console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees.");
             console.log("[i] The new converter is currently in beta and we're looking for your feedback!");
             console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'.");
-            converter = new DeGiroConverter();
+            converter = new DeGiroConverter(yahooFinanceService);
             break;
         case "degiro-v2":
             console.log("[i] Processing file using DeGiro converter (V2 Beta)");
             console.log("[i] NOTE: You are running a converter that is currently in beta.");
             console.log("[i] If you have any issues, please report them on GitHub. Many thanks!");
-            converter = new DeGiroConverterV2();
+            converter = new DeGiroConverterV2(yahooFinanceService);
             break;
         case "fp":
         case "finpension":
             console.log("[i] Processing file using Finpension converter");
-            converter = new FinpensionConverter();
+            converter = new FinpensionConverter(yahooFinanceService);
             break;
         case "sq":
         case "swissquote":
             console.log("[i] Processing file using Swissquote converter");
-            converter = new SwissquoteConverter();
+            converter = new SwissquoteConverter(yahooFinanceService);
             break;
         case "schwab":
             console.log("[i] Processing file using Schwab converter");
-            converter = new SchwabConverter();
+            converter = new SchwabConverter(yahooFinanceService);
+            break;
+        case "etoro":
+            console.log("[i] Processing file using Etoro converter");
+            converter = new EtoroConverter(yahooFinanceService);
             break;
         default:
             throw new Error(`Unknown converter '${converterType}' provided`);

diff --git a/src/converters/abstractconverter.ts b/src/converters/abstractconverter.ts
@@ -1,11 +1,17 @@
 import * as fs from "fs";
 import * as cliProgress from "cli-progress";
+import { YahooFinanceService } from "../yahooFinanceService";
 
 export abstract class AbstractConverter {
 
+    protected yahooFinanceService: YahooFinanceService;
+
     protected progress: cliProgress.MultiBar;
 
-    constructor() {
+    constructor(yahooFinanceService: YahooFinanceService) {
+
+        this.yahooFinanceService = yahooFinanceService;
+
         this.progress = new cliProgress.MultiBar(
             {
                 stopOnComplete: true,

diff --git a/src/converters/degiroConverter.test.ts b/src/converters/degiroConverter.test.ts
@@ -1,11 +1,12 @@
+import { YahooFinanceService } from "../yahooFinanceService";
 import { DeGiroConverter } from "./degiroConverter";
 
 describe("degiroConverter", () => {
 
     it("should construct", () => {
 
       // Act
-      const sut = new DeGiroConverter();
+      const sut = new DeGiroConverter(new YahooFinanceService());
 
       // Asssert
       expect(sut).toBeTruthy();

diff --git a/src/converters/degiroConverter.ts b/src/converters/degiroConverter.ts
@@ -10,12 +10,8 @@ import { GhostfolioOrderType } from "../models/ghostfolioOrderType";
 
 export class DeGiroConverter extends AbstractConverter {
 
-  private yahooFinanceService: YahooFinanceService;
-
-  constructor() {
-    super();
-
-    this.yahooFinanceService = new YahooFinanceService();
+  constructor(yahooFinanceService: YahooFinanceService) {
+    super(yahooFinanceService);
 
     dayjs.extend(customParseFormat);
   }
@@ -37,12 +33,13 @@ export class DeGiroConverter extends AbstractConverter {
         return columnValue;
       }
     }, async (_, records: DeGiroRecord[]) => {
-      
+
       // If records is empty, parsing failed..
-      if (records === undefined) {
+      if (records === undefined || records.length === 0) {
         return errorCallback(new Error("An error ocurred while parsing!"));
       }
 
+
       console.log("[i] Read CSV file. Start processing..");
       const result: GhostfolioExport = {
         meta: {
@@ -89,7 +86,7 @@ export class DeGiroConverter extends AbstractConverter {
             this.progress);
         }
         catch (err) {
-          this.logQueryError(record.isin || record.product, idx);  
+          this.logQueryError(record.isin || record.product, idx);
           return errorCallback(err);
         }
 

diff --git a/src/converters/degiroConverterV2.test.ts b/src/converters/degiroConverterV2.test.ts
@@ -1,11 +1,12 @@
+import { YahooFinanceService } from "../yahooFinanceService";
 import { DeGiroConverterV2 } from "./degiroConverterV2";
 
 describe("degiroConverterV2", () => {
 
     it("should construct", () => {
 
       // Act
-      const sut = new DeGiroConverterV2();
+      const sut = new DeGiroConverterV2(new YahooFinanceService());
 
       // Asssert
       expect(sut).toBeTruthy();