neural.pas

unit neural;

{$mode objfpc}{$H+}

interface

uses
  Classes, SysUtils, FileUtil, process, Forms, Controls, Graphics, Dialogs, Menus,
  StdCtrls, RTTICtrls, RichMemo, Zipper, Processutils;

type

  { TMainForm }

  TMainForm = class(TForm)
    AidaXButton: TButton;
    ProteusButton: TButton;
    MainMenu1: TMainMenu;
    MemConsole: TRichMemo;
    ToolsMenuItem: TMenuItem;
    MenuItem10: TMenuItem;
    FileExitMenuItem: TMenuItem;
    MenuItem12: TMenuItem;
    MenuItem13: TMenuItem;
    MenuItem14: TMenuItem;
    HelpMenuItem: TMenuItem;
    MenuItem16: TMenuItem;
    MenuItem17: TMenuItem;
    MenuItem18: TMenuItem;
    MenuItem19: TMenuItem;
    FileMenuItem: TMenuItem;
    MenuItem20: TMenuItem;
    MenuItem21: TMenuItem;
    MenuItem22: TMenuItem;
    MenuItem23: TMenuItem;
    MenuItem24: TMenuItem;
    TrainMenuItem: TMenuItem;
    NamTrainingMenuItem: TMenuItem;
    AidaXTrainingMenuItem: TMenuItem;
    MenuItem28: TMenuItem;
    NAMMenuItem: TMenuItem;
    NAMVSTMenuItem: TMenuItem;
    MenuItem5: TMenuItem;
    MenuItem6: TMenuItem;
    MenuItem7: TMenuItem;
    MenuItem8: TMenuItem;
    MenuItem9: TMenuItem;
    NAMButton: TButton;
    procedure AidaXButtonClick(Sender: TObject);
    procedure AidaXTrainingMenuItemClick(Sender: TObject);
    procedure NAMButtonClick(Sender: TObject);

  private

  public
    procedure ProcessOutput(Sender:TProcessEx; output:string);
    procedure ProcessError(Sender:TProcessEx; {%H-}IsException:boolean);

  end;

var
  MainForm: TMainForm;

implementation

{$R *.lfm}

{ TMainForm }


 procedure TMainForm.NAMButtonClick(Sender: TObject);

 var
   Proc: TProcessEx;
   namrunner:TextFile;
   reqfile: TextFile;
 begin
   NAMButton.Enabled:=false;
   AssignFile(reqfile, 'requirements');
   Rewrite(reqfile);
   Writeln(reqfile,'absl-py==1.4.0');
   Writeln(reqfile,'aiohttp==3.8.4');
   Writeln(reqfile,'aiosignal==1.3.1');
   Writeln(reqfile,'async-timeout==4.0.2');
   Writeln(reqfile,'attrs==23.1.0');
   Writeln(reqfile,'auraloss==0.3.0');
   Writeln(reqfile,'cachetools==5.3.1');
   Writeln(reqfile,'certifi==2022.12.7');
   Writeln(reqfile,'cffi==1.15.1');
   Writeln(reqfile,'charset-normalizer==2.1.1');
   Writeln(reqfile,'colorama==0.4.6');
   Writeln(reqfile,'contourpy==1.1.0');
   Writeln(reqfile,'cycler==0.11.0');
   Writeln(reqfile,'filelock==3.9.0');
   Writeln(reqfile,'fonttools==4.40.0');
   Writeln(reqfile,'frozenlist==1.3.3');
   Writeln(reqfile,'fsspec==2023.6.0');
   Writeln(reqfile,'google-auth==2.20.0');
   Writeln(reqfile,'google-auth-oauthlib==1.0.0');
   Writeln(reqfile,'grpcio==1.56.0');
   Writeln(reqfile,'idna==3.4');
   Writeln(reqfile,'Jinja2==3.1.2');
   Writeln(reqfile,'kiwisolver==1.4.4');
   Writeln(reqfile,'lightning-utilities==0.8.0');
   Writeln(reqfile,'Markdown==3.4.3');
   Writeln(reqfile,'MarkupSafe==2.1.2');
   Writeln(reqfile,'matplotlib==3.7.1');
   Writeln(reqfile,'mpmath==1.2.1');
   Writeln(reqfile,'msvc-runtime==14.34.31931');
   Writeln(reqfile,'multidict==6.0.4');
   Writeln(reqfile,'networkx==3.0');
   Writeln(reqfile,'neural-amp-modeler==0.6.0');
   Writeln(reqfile,'numpy==1.24.1');
   Writeln(reqfile,'oauthlib==3.2.2');
   Writeln(reqfile,'packaging==23.1');
   Writeln(reqfile,'Pillow==9.3.0');
   Writeln(reqfile,'pip==23.1.2');
   Writeln(reqfile,'protobuf==4.23.3');
   Writeln(reqfile,'pyasn1==0.5.0');
   Writeln(reqfile,'pyasn1-modules==0.3.0');
   Writeln(reqfile,'pycparser==2.21');
   Writeln(reqfile,'pydantic==1.10.9');
   Writeln(reqfile,'pyparsing==3.1.0');
   Writeln(reqfile,'python-dateutil==2.8.2');
   Writeln(reqfile,'pytorch-lightning==2.0.4');
   Writeln(reqfile,'PyYAML==6.0');
   Writeln(reqfile,'requests==2.28.1');
   Writeln(reqfile,'requests-oauthlib==1.3.1');
   Writeln(reqfile,'rsa==4.9');
   Writeln(reqfile,'scipy==1.10.1');
   Writeln(reqfile,'setuptools==67.7.2');
   Writeln(reqfile,'six==1.16.0');
   Writeln(reqfile,'sounddevice==0.4.6');
   Writeln(reqfile,'sqlite-bro==0.12.2');
   Writeln(reqfile,'sympy==1.11.1');
   Writeln(reqfile,'tensorboard==2.13.0');
   Writeln(reqfile,'tensorboard-data-server==0.7.1');
   Writeln(reqfile,'torch==2.0.1+cu118');
   Writeln(reqfile,'torchaudio==2.0.2+cu118');
   Writeln(reqfile,'torchmetrics==0.11.4');
   Writeln(reqfile,'torchvision==0.15.2+cu118');
   Writeln(reqfile,'tqdm==4.65.0');
   Writeln(reqfile,'typing_extensions==4.4.0');
   Writeln(reqfile,'urllib3==1.26.13');
   Writeln(reqfile,'wavio==0.0.7');
   Writeln(reqfile,'Werkzeug==2.3.6');
   Writeln(reqfile,'wheel==0.40.0');
   Writeln(reqfile,'winpython==6.1.20230527');
   Writeln(reqfile,'yarl==1.9.2');
   CloseFile(reqfile);
   //End;


   //StartProcessAndStreamStdioToMemo('NAM-Runner.bat',MemConsole)
   AssignFile(namrunner, 'NAM-Runner070.bat');
  // Try
   Rewrite(namrunner);
   Writeln(namrunner,'');//Remember AnsiStrings are case sensitive
   Writeln(namrunner,'@echo off');
   Writeln(namrunner,'set NAMNAME=neural-amp-modeler-0.7.0');
   Writeln(namrunner,'set NAMVER=0.7.0');
   Writeln(namrunner,'if exist "%~dp0\%NAMNAME%\installed.txt" (');
   Writeln(namrunner,'echo NAM already installed!');
   Writeln(namrunner,'GOTO NAMISINSTALLED');
   Writeln(namrunner,')');
   Writeln(namrunner,'echo This program is downloading and installing the complete NAM modelling environment and all prerequisites and runtimes.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo PLEASE BE PATIENT.');
   Writeln(namrunner,'echo SOME PARTS OF THIS INSTALLATION PROCESS CAN TAKE QUITE SOME TIME!');
   Writeln(namrunner,'echo DON''T CLOSE THIS WINDOW UNTIL YOU ARE ASKED TO DO IT.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo Downloading and extracting Python archive...');
   Writeln(namrunner,'curl -L https://github.com/winpython/winpython/releases/download/6.1.20230527/Winpython64-3.10.11.1dot.exe -o python.exe');
   Writeln(namrunner,'if exist "%~dp0\%NAMNAME%" rmdir /q "%~dp0\%NAMNAME%"');
   Writeln(namrunner,'python.exe -y');
   Writeln(namrunner,'@echo. |call %~dp0\WPy64-310111\scripts\make_winpython_movable.bat');
   Writeln(namrunner,'move /Y "%~dp0\WPy64-310111\python-3.10.11.amd64" "%NAMNAME%"');
   Writeln(namrunner,'echo Removing Python archive and unused files...');
   Writeln(namrunner,'del /f /s /q "%~dp0\WPy64-310111" 1>nul');
   Writeln(namrunner,'rmdir /s /q "%~dp0\WPy64-310111"');
   Writeln(namrunner,'del python.exe');
   Writeln(namrunner,'echo Done.');
   Writeln(namrunner,'cd %NAMNAME%');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'set PYTHONPATH=%~dp0\%NAMNAME%;%~dp0\%NAMNAME%\DLLs;%~dp0\%NAMNAME%\lib;%~dp0\%NAMNAME%\lib\plat-win;%~dp0\%NAMNAME%\lib\site-packages');
   Writeln(namrunner,'set PATH=%~dp0%NAMNAME%;%~dp0%NAMNAME%\Scripts;%PATH%');
   Writeln(namrunner,'echo Upgrading PIP...');
   Writeln(namrunner,'python.exe -m pip install --upgrade pip');
   Writeln(namrunner,'echo Done.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo Installing NAM...');
   Writeln(namrunner,'python -m pip install -r ..\requirements');
  // Writeln(namrunner,'python -m pip install neural-amp-modeler==%NAMVER%');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo Installing torch gpu...');
   //Writeln(namrunner,'pip3 install pydentic==1.10.11');
   //Writeln(namrunner,'pip3 install scipy==1.10.1');
   Writeln(namrunner,'pip3 install numpy --pre torch torchvision torchaudio --force-reinstall --index-url https://download.pytorch.org/whl/nightly/cu118');
   Writeln(namrunner,'echo Done.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'>"%~dp0%NAMNAME%\installed.txt" echo done');
   Writeln(namrunner,'echo NAM install done.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,':NAMISINSTALLED');
   Writeln(namrunner,'set PYTHONPATH=%~dp0\%NAMNAME%;%~dp0\%NAMNAME%\DLLs;%~dp0\%NAMNAME%\lib;%~dp0\%NAMNAME%\lib\plat-win;%~dp0\%NAMNAME%\lib\site-packages');
   Writeln(namrunner,'set PATH=%~dp0%NAMNAME%;%~dp0%NAMNAME%\Scripts;%PATH%');
   Writeln(namrunner,'python -c "from winpython import wppm;dist=wppm.Distribution(r''%~dp0\%NAMNAME%'');dist.patch_standard_packages(''pip'', to_movable=True)"');
   Writeln(namrunner,'nam');
   Writeln(namrunner,'');
   Writeln(namrunner,'cd..');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'echo.');
   Writeln(namrunner,'::echo This window can now be closed.');
   Writeln(namrunner,'echo In case something went wrong or the installation got corrupted:');
   Writeln(namrunner,'echo You can simply delete the folder %NAMNAME% and try a reinstall.');
   Writeln(namrunner,'echo Note, that a reinstallation needs an internet connection.');
   Writeln(namrunner,'echo Thank you.');
   //Finally
   CloseFile(namrunner);
   //End;

   try
   Proc := TProcessEx.Create(nil);
   Proc.Executable := 'NAM-Runner070.bat';
   Proc.OnErrorM:=@(ProcessError);
   Proc.OnOutputM:=@(ProcessOutput);
   Proc.Execute();
   finally
     Proc.Free;
   end;
   DeleteFile('NAM-Runner060.bat');
   NAMButton.Enabled:=true;
 end;

procedure TMainForm.AidaXButtonClick(Sender: TObject);
  var
   Proc: TProcessEx;
   aidaxrunner:TextFile;
   recnetnew:TextFile;
begin
   AidaXButton.Enabled:=false;
   //StartProcessAndStreamStdioToMemo('NAM-Runner.bat',MemConsole)
   AssignFile(aidaxrunner, 'Aida-X-Runner.bat');
  // Try
   Rewrite(aidaxrunner);
   Writeln(aidaxrunner,'@echo off');
   Writeln(aidaxrunner,'set AIDAXNAME=Aida-X');
   Writeln(aidaxrunner,'::set NAMVER=0.6.0');
   Writeln(aidaxrunner,'if exist "%~dp0\%AIDAXNAME%\installed.txt" (');
   Writeln(aidaxrunner,'echo Aida-X already installed!');
   Writeln(aidaxrunner,'GOTO AIDAXISINSTALLED');
   Writeln(aidaxrunner,')');
   Writeln(aidaxrunner,'echo This program is downloading and installing the complete NAM modelling environment and all prerequisites and runtimes.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo PLEASE BE PATIENT.');
   Writeln(aidaxrunner,'echo SOME PARTS OF THIS INSTALLATION PROCESS CAN TAKE QUITE SOME TIME!');
   Writeln(aidaxrunner,'echo DON''T CLOSE THIS WINDOW UNTIL YOU ARE ASKED TO DO IT.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo Downloading and extracting Python archive...');
   Writeln(aidaxrunner,'curl -L https://github.com/winpython/winpython/releases/download/6.1.20230527/Winpython64-3.10.11.1dot.exe -o python.exe');
   Writeln(aidaxrunner,'if exist "%~dp0\%AIDAXNAME%" rmdir /q "%~dp0\%AIDAXNAME%"');
   Writeln(aidaxrunner,'python.exe -y');
   Writeln(aidaxrunner,'@echo. |call %~dp0\WPy64-310111\scripts\make_winpython_movable.bat');
   Writeln(aidaxrunner,'move /Y "%~dp0\WPy64-310111\python-3.10.11.amd64" "%AIDAXNAME%"');
   Writeln(aidaxrunner,'echo Removing Python archive and unused files...');
   Writeln(aidaxrunner,'del /f /s /q "%~dp0\WPy64-310111" 1>nul');
   Writeln(aidaxrunner,'rmdir /s /q "%~dp0\WPy64-310111"');
   Writeln(aidaxrunner,'del python.exe');
   Writeln(aidaxrunner,'echo Done.');
   Writeln(aidaxrunner,'cd %AIDAXNAME%');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'set PYTHONPATH=%~dp0\%AIDAXNAME%;%~dp0\%AIDAXNAME%\DLLs;%~dp0\%AIDAXNAME%\lib;%~dp0\%AIDAXNAME%\lib\plat-win;%~dp0\%AIDAXNAME%\lib\site-packages');
   Writeln(aidaxrunner,'set PATH=%~dp0%AIDAXNAME%;%~dp0%AIDAXNAME%\Scripts;%PATH%');
   Writeln(aidaxrunner,'echo Upgrading PIP...');
   Writeln(aidaxrunner,'python.exe -m pip install --upgrade pip');
   Writeln(aidaxrunner,'echo Done.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo Downloading Git for Windows. Please wait...');
   Writeln(aidaxrunner,'curl -L "https://github.com/git-for-windows/git/releases/download/v2.39.2.windows.1/PortableGit-2.39.2-64-bit.7z.exe" -o %~dp0\PortableGit.exe');
   Writeln(aidaxrunner,'echo Installing portable Git for Windows temporarily. Please wait...');
   Writeln(aidaxrunner,'%~dp0\PortableGit.exe -y -gm2');
   Writeln(aidaxrunner,'del %~dp0\PortableGit.exe');
   Writeln(aidaxrunner,'set gitdir=%~dp0\PortableGit');
   Writeln(aidaxrunner,'set path=%gitdir%\cmd;%path%');
   Writeln(aidaxrunner,'echo Git installation done.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo Installing Github clone of Aida-X trainer, please wait...');
   Writeln(aidaxrunner,'git clone https://github.com/AidaDSP/Automated-GuitarAmpModelling.git');
   Writeln(aidaxrunner,'cd Automated-GuitarAmpModelling');
   Writeln(aidaxrunner,'git checkout aidadsp_devel');
   Writeln(aidaxrunner,'echo Updating submodules...');
   Writeln(aidaxrunner,'git submodule update --init --recursive');
   Writeln(aidaxrunner,'echo Github clone process completed.');
   Writeln(aidaxrunner,'echo Removing PortableGit...');
   Writeln(aidaxrunner,'rmdir /s /q ..\PortableGit');
   Writeln(aidaxrunner,'echo Done.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo Installing python packages with torch gpu/cuda...');
   Writeln(aidaxrunner,'::pip3 install tensorflow ::limits torch created new training file without it');
   Writeln(aidaxrunner,'pip3 install tqdm');
   Writeln(aidaxrunner,'pip3 install tensorboard');
   Writeln(aidaxrunner,'pip3 install numpy');
   Writeln(aidaxrunner,'pip3 install scipy==1.10.1');
   Writeln(aidaxrunner,'pip3 install torch torchvision torchaudio --force-reinstall --index-url https://download.pytorch.org/whl/cu118/');
   Writeln(aidaxrunner,'echo Done.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'>"%~dp0%AIDAXNAME%\installed.txt" echo done');
   Writeln(aidaxrunner,'echo Aida-X trainer install done.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,':AIDAXISINSTALLED');
   Writeln(aidaxrunner,'cd %~dp0\%AIDAXNAME%\Automated-GuitarAmpModelling');
   Writeln(aidaxrunner,'set PYTHONPATH=%~dp0\%AIDAXNAME%;%~dp0\%AIDAXNAME%\DLLs;%~dp0\%AIDAXNAME%\lib;%~dp0\%AIDAXNAME%\lib\plat-win;%~dp0\%AIDAXNAME%\lib\site-packages');
   Writeln(aidaxrunner,'set PATH=%~dp0%AIDAXNAME%;%~dp0%AIDAXNAME%\Scripts;%PATH%');
   Writeln(aidaxrunner,'python -c "from winpython import wppm;dist=wppm.Distribution(r''%~dp0\%AIDAXNAME%'');dist.patch_standard_packages(''pip'', to_movable=True)"');
   Writeln(aidaxrunner,'::nam');
   Writeln(aidaxrunner,'set CUBLAS_WORKSPACE_CONFIG=:4096:2');
   Writeln(aidaxrunner,'python dist_model_recnet.py -l "LSTM-16" -fn "%~dp0\test" -eps 500');
   Writeln(aidaxrunner,'python simple_modelToKeras.py -if "Results\model_best.json" -of "newModel.json"');
   Writeln(aidaxrunner,'cd..');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'echo.');
   Writeln(aidaxrunner,'::echo This window can now be closed.');
   Writeln(aidaxrunner,'echo In case something went wrong or the installation got corrupted:');
   Writeln(aidaxrunner,'echo You can simply delete the folder %AIDAXNAME% and try a reinstall.');
   Writeln(aidaxrunner,'echo Note, that a reinstallation needs an internet connection.');
   Writeln(aidaxrunner,'echo Thank you.');
   //Finally
   CloseFile(aidaxrunner);
    AssignFile(recnetnew, 'recnetnew.py');
  // Try
   Rewrite(recnetnew);
   Writeln(recnetnew,'import CoreAudioML.miscfuncs as miscfuncs');
   Writeln(recnetnew,'import numpy as np');
   Writeln(recnetnew,'import random');
   Writeln(recnetnew,'import CoreAudioML.training as training');
   Writeln(recnetnew,'import CoreAudioML.dataset as CAMLdataset');
   Writeln(recnetnew,'import CoreAudioML.networks as networks');
   Writeln(recnetnew,'import torch');
   Writeln(recnetnew,'import torch.optim as optim');
   Writeln(recnetnew,'from torch.utils.tensorboard import SummaryWriter');
   Writeln(recnetnew,'import argparse');
   Writeln(recnetnew,'import time');
   Writeln(recnetnew,'import os');
   Writeln(recnetnew,'import csv');
   Writeln(recnetnew,'from scipy.io.wavfile import write');
   Writeln(recnetnew,'from tqdm import tqdm');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'prsr = argparse.ArgumentParser(');
   Writeln(recnetnew,'description=''''''This script implements training for neural network amplifier/distortion effects modelling. This is');
   Writeln(recnetnew,'intended to recreate the training of models of the ht1 amplifier and big muff distortion pedal, but can easily be');
   Writeln(recnetnew,'adapted to use any dataset'''''')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# arguments for the training/test data locations and file names and config loading');
   Writeln(recnetnew,'prsr.add_argument(''--device'', ''-p'', default=''ht1'', help=''This label describes what device is being modelled'')');
   Writeln(recnetnew,'prsr.add_argument(''--data_location'', ''-dl'', default=''./Data'', help=''Location of the "Data" directory'')');
   Writeln(recnetnew,'prsr.add_argument(''--file_name'', ''-fn'', default=''ht1'',');
   Writeln(recnetnew,'help=''The filename of the wav file to be loaded as the input/target data, the script looks for files''');
   Writeln(recnetnew,'''with the filename and the extensions -input.wav and -target.wav '')');
   Writeln(recnetnew,'prsr.add_argument(''--load_config'', ''-l'', default=None,');
   Writeln(recnetnew,'help="File path, to a JSON config file, arguments listed in the config file will replace the defaults")');
   Writeln(recnetnew,'prsr.add_argument(''--config_location'', ''-cl'', default=''Configs'', help=''Location of the "Configs" directory'')');
   Writeln(recnetnew,'prsr.add_argument(''--save_location'', ''-sloc'', default=''Results'', help=''Directory where trained models will be saved'')');
   Writeln(recnetnew,'prsr.add_argument(''--load_model'', ''-lm'', type=int, default=1, help=''load a pretrained model if it is found'')');
   Writeln(recnetnew,'prsr.add_argument(''--seed'', default=None, type=int, help=''seed all of the random number generators if desired'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# pre-processing of the training/val/test data');
   Writeln(recnetnew,'prsr.add_argument(''--segment_length'', ''-slen'', type=int, default=24000, help=''Training audio segment length in samples'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# number of epochs and validation');
   Writeln(recnetnew,'prsr.add_argument(''--epochs'', ''-eps'', type=int, default=2000, help=''Max number of training epochs to run'')');
   Writeln(recnetnew,'prsr.add_argument(''--validation_f'', ''-vfr'', type=int, default=2, help=''Validation Frequency (in epochs)'')');
   Writeln(recnetnew,'# TO DO');
   Writeln(recnetnew,'prsr.add_argument(''--validation_p'', ''-vp'', type=int, default=25,');
   Writeln(recnetnew,'help=''How many validations without improvement before stopping training, None for no early stopping'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# settings for the training epoch');
   Writeln(recnetnew,'prsr.add_argument(''--batch_size'', ''-bs'', type=int, default=50, help=''Training mini-batch size'')');
   Writeln(recnetnew,'prsr.add_argument(''--iter_num'', ''-it'', type=int, default=None,');
   Writeln(recnetnew,'help=''Overrides --batch_size and instead sets the batch_size so that a total of --iter_num batches''');
   Writeln(recnetnew,'''are processed in each epoch'')');
   Writeln(recnetnew,'prsr.add_argument(''--learn_rate'', ''-lr'', type=float, default=0.005, help=''Initial learning rate'')');
   Writeln(recnetnew,'prsr.add_argument(''--init_len'', ''-il'', type=int, default=200,');
   Writeln(recnetnew,'help=''Number of sequence samples to process before starting weight updates'')');
   Writeln(recnetnew,'prsr.add_argument(''--up_fr'', ''-uf'', type=int, default=1000,');
   Writeln(recnetnew,'help=''For recurrent models, number of samples to run in between updating network weights, i.e the ''');
   Writeln(recnetnew,'''default argument updates every 1000 samples'')');
   Writeln(recnetnew,'prsr.add_argument(''--cuda'', ''-cu'', default=1, help=''Use GPU if available'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# loss function/s');
   Writeln(recnetnew,'prsr.add_argument(''--loss_fcns'', ''-lf'', default={''ESRPre'': 0.75, ''DC'': 0.25},');
   Writeln(recnetnew,'help=''Which loss functions, ESR, ESRPre, DC. Argument is a dictionary with each key representing a''');
   Writeln(recnetnew,'''loss function name and the corresponding value being the multiplication factor applied to that''');
   Writeln(recnetnew,'''loss function, used to control the contribution of each loss function to the overall loss '')');
   Writeln(recnetnew,'prsr.add_argument(''--pre_filt'', ''-pf'', default=''high_pass'',');
   Writeln(recnetnew,'help=''FIR filter coefficients for pre-emphasis filter, can also read in a csv file'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# the validation and test sets are divided into shorter chunks before processing to reduce the amount of GPU memory used');
   Writeln(recnetnew,'# you can probably ignore this unless during training you get a ''cuda out of memory'' error');
   Writeln(recnetnew,'prsr.add_argument(''--val_chunk'', ''-vs'', type=int, default=100000, help=''Number of sequence samples to process''');
   Writeln(recnetnew,'''in each chunk of validation '')');
   Writeln(recnetnew,'prsr.add_argument(''--test_chunk'', ''-tc'', type=int, default=100000, help=''Number of sequence samples to process''');
   Writeln(recnetnew,'''in each chunk of test '')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# arguments for the network structure');
   Writeln(recnetnew,'prsr.add_argument(''--model'', ''-m'', default=''SimpleRNN'', type=str, help=''model architecture'')');
   Writeln(recnetnew,'prsr.add_argument(''--input_size'', ''-is'', default=1, type=int, help=''1 for mono input data, 2 for stereo, etc '')');
   Writeln(recnetnew,'prsr.add_argument(''--output_size'', ''-os'', default=1, type=int, help=''1 for mono output data, 2 for stereo, etc '')');
   Writeln(recnetnew,'prsr.add_argument(''--num_blocks'', ''-nb'', default=1, type=int, help=''Number of recurrent blocks'')');
   Writeln(recnetnew,'prsr.add_argument(''--hidden_size'', ''-hs'', default=16, type=int, help=''Recurrent unit hidden state size'')');
   Writeln(recnetnew,'prsr.add_argument(''--unit_type'', ''-ut'', default=''LSTM'', help=''LSTM or GRU or RNN'')');
   Writeln(recnetnew,'prsr.add_argument(''--skip_con'', ''-sc'', default=1, type=int, help=''is there a skip connection for the input to the output'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'args = prsr.parse_args()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# This function takes a directory as argument, looks for an existing model file called ''model.json'' and loads a network');
   Writeln(recnetnew,'# from it, after checking the network in ''model.json'' matches the architecture described in args. If no model file is');
   Writeln(recnetnew,'# found, it creates a network according to the specification in args.');
   Writeln(recnetnew,'def init_model(save_path, args):');
   Writeln(recnetnew,'# Search for an existing model in the save directory');
   Writeln(recnetnew,'if miscfuncs.file_check(''model.json'', save_path) and args.load_model:');
   Writeln(recnetnew,'print(''existing model file found, loading network.. continuing training..'')');
   Writeln(recnetnew,'model_data = miscfuncs.json_load(''model'', save_path)');
   Writeln(recnetnew,'# assertions to check that the model.json file is for the right neural network architecture');
   Writeln(recnetnew,'try:');
   Writeln(recnetnew,'assert model_data[''model_data''][''unit_type''] == args.unit_type');
   Writeln(recnetnew,'assert model_data[''model_data''][''input_size''] == args.input_size');
   Writeln(recnetnew,'assert model_data[''model_data''][''hidden_size''] == args.hidden_size');
   Writeln(recnetnew,'assert model_data[''model_data''][''output_size''] == args.output_size');
   Writeln(recnetnew,'except AssertionError:');
   Writeln(recnetnew,'print("model file found with network structure not matching config file structure")');
   Writeln(recnetnew,'network = networks.load_model(model_data)');
   Writeln(recnetnew,'# If no existing model is found, create a new one');
   Writeln(recnetnew,'else:');
   Writeln(recnetnew,'# print(''no saved model found, creating new network'')');
   Writeln(recnetnew,'network = networks.SimpleRNN(input_size=args.input_size, unit_type=args.unit_type, hidden_size=args.hidden_size,');
   Writeln(recnetnew,'output_size=args.output_size, skip=args.skip_con)');
   Writeln(recnetnew,'network.save_state = False');
   Writeln(recnetnew,'network.save_model(''model'', save_path)');
   Writeln(recnetnew,'return network');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'if __name__ == "__main__":');
   Writeln(recnetnew,'"""The main method creates the recurrent network, trains it and carries out validation/testing """');
   Writeln(recnetnew,'if args.seed:');
   Writeln(recnetnew,'torch.manual_seed(args.seed)');
   Writeln(recnetnew,'np.random.seed(args.seed)');
   Writeln(recnetnew,'random.seed(args.seed)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'start_time = time.time()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# If a load_config argument was provided, construct the file path to the config file');
   Writeln(recnetnew,'if args.load_config:');
   Writeln(recnetnew,'# Load the configs and write them onto the args dictionary, this will add new args and/or overwrite old ones');
   Writeln(recnetnew,'configs = miscfuncs.json_load(args.load_config, args.config_location)');
   Writeln(recnetnew,'for parameters in configs:');
   Writeln(recnetnew,'args.__setattr__(parameters, configs[parameters])');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# if args.model == ''SimpleRNN'':');
   Writeln(recnetnew,'model_name = args.file_name + ''_'' + args.unit_type + ''-'' + str(args.hidden_size) + ''-'' + str(args.skip_con)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Fix parameter in case input as argument');
   Writeln(recnetnew,'if type(args.loss_fcns) is str:');
   Writeln(recnetnew,'args.loss_fcns = eval(args.loss_fcns)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# It''s a good moment to print parameters');
   Writeln(recnetnew,'print("")');
   Writeln(recnetnew,'print("args.device = %s" % args.device)');
   Writeln(recnetnew,'print("args.file_name = %s" % args.file_name)');
   Writeln(recnetnew,'print("args.input_size = %s" % args.input_size)');
   Writeln(recnetnew,'print("args.hidden_size = %d" % args.hidden_size)');
   Writeln(recnetnew,'print("args.unit_type = %s" % args.unit_type)');
   Writeln(recnetnew,'#print("args.loss_fcns = %s" % str(args.loss_fcns))');
   Writeln(recnetnew,'print("args.skip_con = %d" % args.skip_con)');
   Writeln(recnetnew,'#print("args.pre_filt = %s" % args.pre_filt)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'if args.pre_filt == ''A-Weighting'':');
   Writeln(recnetnew,'with open(''Configs/'' + ''b_Awght.csv'') as csvfile:');
   Writeln(recnetnew,'reader = csv.reader(csvfile, delimiter='','')');
   Writeln(recnetnew,'args.pre_filt = list(reader)');
   Writeln(recnetnew,'args.pre_filt = args.pre_filt[0]');
   Writeln(recnetnew,'for item in range(len(args.pre_filt)):');
   Writeln(recnetnew,'args.pre_filt[item] = float(args.pre_filt[item])');
   Writeln(recnetnew,'elif args.pre_filt == ''high_pass'':');
   Writeln(recnetnew,'args.pre_filt = [-0.85, 1]');
   Writeln(recnetnew,'elif args.pre_filt == ''None'':');
   Writeln(recnetnew,'args.pre_filt = None');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Generate name of directory where results will be saved');
   Writeln(recnetnew,'save_path = ''Results''');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Check if an existing saved model exists, and load it, otherwise creates a new model');
   Writeln(recnetnew,'network = init_model(save_path, args)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Check if a cuda device is available');
   Writeln(recnetnew,'if not torch.cuda.is_available() or args.cuda == 0:');
   Writeln(recnetnew,'# print(''cuda device not available/not selected'')');
   Writeln(recnetnew,'cuda = 0');
   Writeln(recnetnew,'else:');
   Writeln(recnetnew,'torch.set_default_tensor_type(''torch.cuda.FloatTensor'')');
   Writeln(recnetnew,'torch.cuda.set_device(0)');
   Writeln(recnetnew,'# print(''cuda device available'')');
   Writeln(recnetnew,'network = network.cuda()');
   Writeln(recnetnew,'cuda = 1');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Set up training optimiser + scheduler + loss fcns and training info tracker');
   Writeln(recnetnew,'optimiser = torch.optim.Adam(network.parameters(), lr=args.learn_rate, weight_decay=1e-4)');
   Writeln(recnetnew,'scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimiser, ''min'', factor=0.5, patience=5, verbose=False)');
   Writeln(recnetnew,'loss_functions = training.LossWrapper(args.loss_fcns, args.pre_filt)');
   Writeln(recnetnew,'train_track = training.TrainTrack()');
   Writeln(recnetnew,'writer = SummaryWriter(os.path.join(''TensorboardData'', model_name))');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Load dataset');
   Writeln(recnetnew,'dataset = CAMLdataset.DataSet(data_dir=args.data_location)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# The train dataset is divided into frames of 0.5 seconds according to the paper. To achieve this');
   Writeln(recnetnew,'# 22050 is used as segment_length since sample rate is 44100Hz.');
   Writeln(recnetnew,'dataset.create_subset(''train'', frame_len=args.segment_length)');
   Writeln(recnetnew,'dataset.load_file(os.path.join(''train'', args.file_name), ''train'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'dataset.create_subset(''val'')');
   Writeln(recnetnew,'dataset.load_file(os.path.join(''val'', args.file_name), ''val'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# If training is restarting, this will ensure the previously elapsed training time is added to the total');
   Writeln(recnetnew,'init_time = time.time() - start_time + train_track[''total_time'']*3600');
   Writeln(recnetnew,'# Set network save_state flag to true, so when the save_model method is called the network weights are saved');
   Writeln(recnetnew,'network.save_state = True');
   Writeln(recnetnew,'patience_counter = 0');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# This is where training happens');
   Writeln(recnetnew,'# the network records the last epoch number, so if training is restarted it will start at the correct epoch number');
   Writeln(recnetnew,'for epoch in tqdm(range(train_track[''current_epoch''] + 1, args.epochs + 1)):');
   Writeln(recnetnew,'ep_st_time = time.time()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Run 1 epoch of training,');
   Writeln(recnetnew,'epoch_loss = network.train_epoch(dataset.subsets[''train''].data[''input''][0],');
   Writeln(recnetnew,'dataset.subsets[''train''].data[''target''][0],');
   Writeln(recnetnew,'loss_functions, optimiser, args.batch_size, args.init_len, args.up_fr)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'writer.add_scalar(''Time/EpochTrainingTime'', time.time()-ep_st_time, epoch)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Run validation');
   Writeln(recnetnew,'if epoch % args.validation_f == 0:');
   Writeln(recnetnew,'val_ep_st_time = time.time()');
   Writeln(recnetnew,'val_output, val_loss = network.process_data(dataset.subsets[''val''].data[''input''][0],');
   Writeln(recnetnew,'dataset.subsets[''val''].data[''target''][0], loss_functions, args.val_chunk)');
   Writeln(recnetnew,'scheduler.step(val_loss)');
   Writeln(recnetnew,'if val_loss < train_track[''best_val_loss'']:');
   Writeln(recnetnew,'#print("new best val loss: %f" % val_loss.item())');
   Writeln(recnetnew,'patience_counter = 0');
   Writeln(recnetnew,'network.save_model(''model_best'', save_path)');
   Writeln(recnetnew,'write(os.path.join(save_path, "best_val_out.wav"),');
   Writeln(recnetnew,'dataset.subsets[''val''].fs, val_output.cpu().numpy()[:, 0, 0])');
   Writeln(recnetnew,'else:');
   Writeln(recnetnew,'patience_counter += 1');
   Writeln(recnetnew,'train_track.val_epoch_update(val_loss.item(), val_ep_st_time, time.time())');
   Writeln(recnetnew,'writer.add_scalar(''TrainingAndValidation/ValidationLoss'', train_track[''validation_losses''][-1], epoch)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'#print(''current learning rate: '' + str(optimiser.param_groups[0][''lr'']))');
   Writeln(recnetnew,'train_track.train_epoch_update(epoch_loss.item(), ep_st_time, time.time(), init_time, epoch)');
   Writeln(recnetnew,'# write loss to the tensorboard (just for recording purposes)');
   Writeln(recnetnew,'writer.add_scalar(''TrainingAndValidation/TrainingLoss'', train_track[''training_losses''][-1], epoch)');
   Writeln(recnetnew,'writer.add_scalar(''TrainingAndValidation/LearningRate'', optimiser.param_groups[0][''lr''], epoch)');
   Writeln(recnetnew,'network.save_model(''model'', save_path)');
   Writeln(recnetnew,'miscfuncs.json_save(train_track, ''training_stats'', save_path)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'if args.validation_p and patience_counter > args.validation_p:');
   Writeln(recnetnew,'print(''validation patience limit reached at epoch '' + str(epoch))');
   Writeln(recnetnew,'break');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Remove dataset from memory');
   Writeln(recnetnew,'del dataset');
   Writeln(recnetnew,'# Empty the CUDA Cache');
   Writeln(recnetnew,'# torch.cuda.empty_cache()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Create a new data set');
   Writeln(recnetnew,'dataset = CAMLdataset.DataSet(data_dir=args.data_location)');
   Writeln(recnetnew,'# Then load the Test data set');
   Writeln(recnetnew,'dataset.create_subset(''test'')');
   Writeln(recnetnew,'dataset.load_file(os.path.join(''test'', args.file_name), ''test'')');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'print("done training")');
   Writeln(recnetnew,'lossESR = training.ESRLoss()');
   Writeln(recnetnew,'lossDC = training.DCLoss()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'print("testing the final model")');
   Writeln(recnetnew,'# Test the model the training ended with');
   Writeln(recnetnew,'test_output, test_loss = network.process_data(dataset.subsets[''test''].data[''input''][0],');
   Writeln(recnetnew,'dataset.subsets[''test''].data[''target''][0], loss_functions, args.test_chunk)');
   Writeln(recnetnew,'test_loss_ESR = lossESR(test_output, dataset.subsets[''test''].data[''target''][0])');
   Writeln(recnetnew,'test_loss_DC = lossDC(test_output, dataset.subsets[''test''].data[''target''][0])');
   Writeln(recnetnew,'write(os.path.join(save_path, "test_out_final.wav"), dataset.subsets[''test''].fs, test_output.cpu().numpy()[:, 0, 0])');
   Writeln(recnetnew,'writer.add_scalar(''Testing/FinalTestLoss'', test_loss.item())');
   Writeln(recnetnew,'writer.add_scalar(''Testing/FinalTestESR'', test_loss_ESR.item())');
   Writeln(recnetnew,'writer.add_scalar(''Testing/FinalTestDC'', test_loss_DC.item())');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'train_track[''test_loss_final''] = test_loss.item()');
   Writeln(recnetnew,'train_track[''test_lossESR_final''] = test_loss_ESR.item()');
   Writeln(recnetnew,'train_track[''test_lossDC_final''] = test_loss_DC.item()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Add input/output reference batch to training stats');
   Writeln(recnetnew,'# For input batch in case of conditioned models, we assume all params equal to 0.0');
   Writeln(recnetnew,'train_track[''input_batch''] = dataset.subsets[''test''].data[''input''][0].cpu().data.numpy()[:2048, 0, 0].tolist()');
   Writeln(recnetnew,'train_track[''output_batch_final''] = test_output.cpu().data.numpy()[:2048, 0, 0].tolist()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'print("testing the best model")');
   Writeln(recnetnew,'# Test the best model');
   Writeln(recnetnew,'best_val_net = miscfuncs.json_load(''model_best'', save_path)');
   Writeln(recnetnew,'network = networks.load_model(best_val_net)');
   Writeln(recnetnew,'test_output, test_loss = network.process_data(dataset.subsets[''test''].data[''input''][0],');
   Writeln(recnetnew,'dataset.subsets[''test''].data[''target''][0], loss_functions, args.test_chunk)');
   Writeln(recnetnew,'test_loss_ESR = lossESR(test_output, dataset.subsets[''test''].data[''target''][0])');
   Writeln(recnetnew,'test_loss_DC = lossDC(test_output, dataset.subsets[''test''].data[''target''][0])');
   Writeln(recnetnew,'write(os.path.join(save_path, "test_out_best.wav"),');
   Writeln(recnetnew,'dataset.subsets[''test''].fs, test_output.cpu().numpy()[:, 0, 0])');
   Writeln(recnetnew,'writer.add_scalar(''Testing/BestTestLoss'', test_loss.item())');
   Writeln(recnetnew,'writer.add_scalar(''Testing/BestTestESR'', test_loss_ESR.item())');
   Writeln(recnetnew,'writer.add_scalar(''Testing/BestTestDC'', test_loss_DC.item())');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'train_track[''test_loss_best''] = test_loss.item()');
   Writeln(recnetnew,'train_track[''test_lossESR_best''] = test_loss_ESR.item()');
   Writeln(recnetnew,'train_track[''test_lossDC_best''] = test_loss_DC.item()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'# Add output reference batch to training stats, input already entered previously');
   Writeln(recnetnew,'train_track[''output_batch_best''] = test_output.cpu().data.numpy()[:2048, 0, 0].tolist()');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'print("finished training: " + model_name)');
   Writeln(recnetnew,'');
   Writeln(recnetnew,'miscfuncs.json_save(train_track, ''training_stats'', save_path)');
   Writeln(recnetnew,'if cuda:');
   Writeln(recnetnew,'with open(os.path.join(save_path, ''maxmemusage.txt''), ''w'') as f:');
   Writeln(recnetnew,'f.write(str(torch.cuda.max_memory_allocated()))');
   Writeln(recnetnew,'');
   CloseFile(recnetnew);

end;

procedure TMainForm.AidaXTrainingMenuItemClick(Sender: TObject);
begin

end;




 procedure TMainForm.ProcessError(Sender: TProcessEx; IsException: boolean);
 begin
   MemConsole.Lines.Append('Error ! ' + Sender.ExceptionInfo);
   Application.ProcessMessages;
 end;

 procedure TMainForm.ProcessOutput(Sender: TProcessEx; output : String);
 begin
  MemConsole.Lines.Text :=  MemConsole.Lines.Text + output;
   // si vous avez des problème d'accent
   //MemConsole.Lines.Text := MemConsole.Lines.Text + ConsoleToUtf8(output);
   // pour scroll automatique
   MemConsole.SelStart := Length(MemConsole.Lines.Text)-1;
   MemConsole.SelLength:=0;
   Application.ProcessMessages;
 end;

end.