From 1b057d0d63a861fba4b3cb59c6c5fc1a0e3da383 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Wed, 20 Nov 2024 17:01:18 +0800 Subject: [PATCH] feat: kaggle refactor (#489) * init trail * Add spec info * auto unzip mlebench prepared data for out scenario * successfully run example * successfully run main * simplify load traing * extract load_from_raw_data * split the fies(still buggy) It should stop on ~20 epoch and reach the end * some changes * Fix bug to run example * (success) until feature * refine model and ensemble * add metrics in ens.py * update README & spec.md * ens change * fix ens bug * Delete rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/train.py * add template_path in KG_conf * fix test kaggle * CI * make test_import not check kaggle template codes --------- Co-authored-by: Bowen Xian --- rdagent/app/kaggle/conf.py | 3 + .../kaggle/experiment/kaggle_experiment.py | 8 +- .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_xgboost.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../digit-recognizer}/fea_share_preprocess.py | 0 .../digit-recognizer}/feature/feature.py | 0 .../digit-recognizer}/model/model_nn.py | 0 .../digit-recognizer}/model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../digit-recognizer}/model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../digit-recognizer}/model/select_xgboost.py | 0 .../digit-recognizer}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../forest-cover-type-prediction}/train.py | 0 .../train_past.py | 0 .../fea_share_preprocess.py | 0 .../meta_tpl_deprecated/feature/feature.py | 0 .../meta_tpl_deprecated/model/model_nn.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../meta_tpl_deprecated/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_linear.py | 0 .../model/select_lightgbm.py | 0 .../model/select_linear.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s3e11}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s3e14}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s3e16}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s3e26}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s4e5}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s4e8}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../playground-series-s4e9}/train.py | 0 .../sf-crime}/fea_share_preprocess.py | 0 .../sf-crime}/feature/feature.py | 0 .../sf-crime}/model/model_randomforest.py | 0 .../sf-crime}/model/model_xgboost.py | 0 .../sf-crime}/model/select_lightgbm.py | 0 .../sf-crime}/model/select_nn.py | 0 .../sf-crime}/model/select_randomforest.py | 0 .../sf-crime}/model/select_xgboost.py | 0 .../sf-crime}/train.py | 0 .../fea_share_preprocess.py | 0 .../spaceship-titanic}/feature/feature.py | 0 .../model/model_randomforest.py | 0 .../spaceship-titanic}/model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../spaceship-titanic}/model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../spaceship-titanic}/train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../train.py | 0 .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../model/select_lightgbm.py | 0 .../model/select_nn.py | 0 .../model/select_randomforest.py | 0 .../model/select_xgboost.py | 0 .../ventilator-pressure-prediction}/train.py | 0 .../scenarios/kaggle/experiment/workspace.py | 4 +- rdagent/scenarios/kaggle/kaggle_crawler.py | 2 +- .../aerial-cactus-identification/README.md | 43 ++++++ .../aerial-cactus-identification/ens.py | 55 +++++++ .../aerial-cactus-identification/feat01.py | 55 +++++++ .../aerial-cactus-identification/load_data.py | 82 ++++++++++ .../aerial-cactus-identification/main.py | 37 +++++ .../aerial-cactus-identification/model01.py | 141 ++++++++++++++++++ .../aerial-cactus-identification/spec.md | 124 +++++++++++++++ rdagent/scenarios/kaggle/tpl_ex/meta/spec.md | 34 +++++ rdagent/utils/env.py | 5 +- test/utils/test_import.py | 2 +- test/utils/test_kaggle.py | 4 +- 184 files changed, 588 insertions(+), 11 deletions(-) rename rdagent/scenarios/kaggle/experiment/{covid19-global-forecasting-week-1_template => templates/covid19-global-forecasting-week-1}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{covid19-global-forecasting-week-1_template => templates/covid19-global-forecasting-week-1}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{covid19-global-forecasting-week-1_template => templates/covid19-global-forecasting-week-1}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{covid19-global-forecasting-week-1_template => templates/covid19-global-forecasting-week-1}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{covid19-global-forecasting-week-1_template => templates/covid19-global-forecasting-week-1}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/model_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{digit-recognizer_template => templates/digit-recognizer}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{feedback-prize-english-language-learning_template => templates/feedback-prize-english-language-learning}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{forest-cover-type-prediction_template => templates/forest-cover-type-prediction}/train_past.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/model/model_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{ => templates}/meta_tpl_deprecated/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/model_linear.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/select_linear.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{new-york-city-taxi-fare-prediction_template => templates/new-york-city-taxi-fare-prediction}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{optiver-realized-volatility-prediction_template => templates/optiver-realized-volatility-prediction}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e11_template => templates/playground-series-s3e11}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e14_template => templates/playground-series-s3e14}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e16_template => templates/playground-series-s3e16}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s3e26_template => templates/playground-series-s3e26}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e5_template => templates/playground-series-s4e5}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e8_template => templates/playground-series-s4e8}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{playground-series-s4e9_template => templates/playground-series-s4e9}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{sf-crime_template => templates/sf-crime}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{spaceship-titanic_template => templates/spaceship-titanic}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{statoil-iceberg-classifier-challenge_template => templates/statoil-iceberg-classifier-challenge}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-dec-2021_template => templates/tabular-playground-series-dec-2021}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{tabular-playground-series-may-2022_template => templates/tabular-playground-series-may-2022}/train.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/select_lightgbm.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/select_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/select_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/model/select_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{ventilator-pressure-prediction_template => templates/ventilator-pressure-prediction}/train.py (100%) create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/README.md create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ens.py create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/feat01.py create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/model01.py create mode 100644 rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec.md create mode 100644 rdagent/scenarios/kaggle/tpl_ex/meta/spec.md diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index b94799aa..6cc9f52a 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -44,6 +44,9 @@ class Config: competition: str = "" """Kaggle competition name, e.g., 'sf-crime'""" + template_path: str = "rdagent/scenarios/kaggle/experiment/templates" + """Kaggle competition base templates path""" + local_data_path: str = "" """Folder storing Kaggle competition data""" diff --git a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py index ca3e9242..24975d49 100644 --- a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py +++ b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py @@ -38,7 +38,9 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None: super().__init__(*args, **kwargs) self.experiment_workspace = KGFBWorkspace( - template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" + template_folder_path=Path(__file__).resolve() + / Path(KAGGLE_IMPLEMENT_SETTING.template_path).resolve() + / KAGGLE_IMPLEMENT_SETTING.competition ) if len(self.based_experiments) > 0: self.experiment_workspace.inject_code(**self.based_experiments[-1].experiment_workspace.code_dict) @@ -62,7 +64,9 @@ class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWo def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None: super().__init__(*args, **kwargs) self.experiment_workspace = KGFBWorkspace( - template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" + template_folder_path=Path(__file__).resolve() + / Path(KAGGLE_IMPLEMENT_SETTING.template_path).resolve() + / KAGGLE_IMPLEMENT_SETTING.competition ) if len(self.based_experiments) > 0: self.experiment_workspace.inject_code(**self.based_experiments[-1].experiment_workspace.code_dict) diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py rename to rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py rename to rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/model_linear.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/model_linear.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/model_linear.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/model_linear.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_linear.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_linear.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_linear.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_linear.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_lightgbm.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_lightgbm.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_nn.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_randomforest.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_xgboost.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/select_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py rename to rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index 0625dc06..1bfaaf86 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -26,11 +26,10 @@ class KGFBWorkspace(FBWorkspace): - def __init__(self, template_folder_path: Path, *args, entry="python train.py", **kwargs) -> None: + def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.inject_code_from_folder(template_folder_path) self.data_description: List[Tuple[str, int]] = [] - self.entry = entry # this is for debugging (you may want to change it into `sleep 1000`) @property def model_description(self) -> dict[str, str]: @@ -86,7 +85,6 @@ def execute(self, run_env: dict = {}, *args, **kwargs) -> str: execute_log = kgde.run( local_path=str(self.workspace_path), - entry=self.entry, env=run_env, running_extra_volume=running_extra_volume, ) diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index f2d35aa9..b6b71917 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -122,7 +122,7 @@ def download_data(competition: str, local_path: str = KAGGLE_IMPLEMENT_SETTING.l f"/bin/sh -c 'cp -r ./zip_files/{competition}/prepared/public/* ./{competition}'", local_path=local_path ) mleb_env.run( - f"/bin/sh -c 'cp -r ./zip_files/{competition}/prepared/private/test.csv ./{competition}/valid.csv'", + f'/bin/sh -c \'for zip_file in ./{competition}/*.zip; do dir_name="${{zip_file%.zip}}"; mkdir -p "$dir_name"; unzip -o "$zip_file" -d "$dir_name"; done\'', local_path=local_path, ) # NOTE: diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/README.md b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/README.md new file mode 100644 index 00000000..0faf7dd2 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/README.md @@ -0,0 +1,43 @@ +# Motivation of the example +We use a runnable concrete example to demonstrate what the project should be like after being generated by a large language model. + + +# Content example and the workflow + +> NOTE: the `README.md` itself is note generated by LLM. the content remains are generated by LLM. +> + + +## Extra input information beyond the competition information + +[[../meta/spec.md]] +- [ ] TODO + +## Step0: Specification generation + +- Generate specification + [[spec.md]] + - [ ] TODO: perfect +- Generate loading data + [[load_data.py]] + +- Why do we merge this step together. + - Successfully run `load_data.py` is a kind of verification of `spec.md` + + +## Step1: write the feature engineering code +- We can generate some file like [[feat01.py]] that match the pattern `feat.*\.py` + +## Step2: Model training + + +## Step3: ensemble and decision +- generate `ens_and_decsion` + - why we generate score on ensemble phase + - ensemble has following tasks which has great overlap + - ensemble usually check the performance before ensemble + - A additional step to record performance is easier. + +## Step4: Build workflow + +[[main.py]] diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ens.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ens.py new file mode 100644 index 00000000..76b08907 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ens.py @@ -0,0 +1,55 @@ +import numpy as np +import pandas as pd +from sklearn.metrics import roc_auc_score + + +def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: + """ + Handle the following: + 1) Ensemble predictions using a simple average. + 2) Make final decision after ensemble (convert the predictions to final binary form). + + Parameters + ---------- + test_pred_l : list[np.ndarray] + List of predictions on the test data. + val_pred_l : list[np.ndarray] + List of predictions on the validation data. + val_label : np.ndarray + True labels of the validation data. + + Returns + ------- + np.ndarray + Binary predictions on the test data. + """ + + scores = [] + for id, val_pred in enumerate(val_pred_l): + scores.append(roc_auc_score(val_label, val_pred)) + + # Normalize the scores to get weights + total_score = sum(scores) + weights = [score / total_score for score in scores] + + # Weighted average of test predictions + weighted_test_pred = np.zeros_like(test_pred_l[0]) + for weight, test_pred in zip(weights, test_pred_l): + weighted_test_pred += weight * test_pred + + weighted_valid_pred = np.zeros_like(val_pred_l[0]) + for weight, val_pred in zip(weights, val_pred_l): + weighted_valid_pred += weight * val_pred + + weighted_valid_pred_score = roc_auc_score(val_label, weighted_valid_pred) + + scores_df = pd.DataFrame( + { + "Model": list(range(len(val_pred_l))) + ["weighted_average_ensemble"], + "AUROC": scores + [weighted_valid_pred_score], + } + ) + scores_df.to_csv("scores.csv", index=False) + + pred_binary_l = [0 if value < 0.50 else 1 for value in weighted_test_pred] + return np.array(pred_binary_l) diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/feat01.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/feat01.py new file mode 100644 index 00000000..8fbe91d6 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/feat01.py @@ -0,0 +1,55 @@ +import numpy as np + + +def feat_eng( + X: np.ndarray, + y: np.ndarray | None = None, + X_fit: np.ndarray | None = None, + y_fit: np.ndarray | None = None, + param: object | None = None, +) -> tuple[np.ndarray, np.ndarray | None, object]: + """ + Perform feature engineering on the input data. + + Parameters: + - X: np.ndarray + The input data to be transformed. A concrete example could be: + array([[[[207, 194, 203], + ..., + [191, 183, 164], + [176, 168, 149], + [181, 173, 152]]]], dtype=uint8) + - y: np.ndarray | None + The target data. A concrete example could be: + array([1, 0, 1, 0, 1, 1, ..., ]) + - X_fit: np.ndarray | None + Data for fitting the transformation parameters. + - y_fit: np.ndarray | None + Target data for fitting. + - param: object | None + Pre-fitted parameters for transformation. + + Returns: + - transformed_data: np.ndarray + Transformed data. + - transformed_target: np.ndarray | None + Transformed target data. + - fitted_param: object + Fitted parameters. + + Notes: + - Some preprocessing (e.g., data selection) is based on y. + + Typical usage: + .. code-block:: python + + X_transformed, y_transformed, fitted_param = feat_eng(X, y, X, y) + X_test_transformed, _, _ = feat_eng(X_test, fitted_param) + """ + # This is an example of identity feature transformation. + # We'll not change the content of the data, but we'll demonstrate the typical workflow of feature engineering. + if param is None: + # Get parameters from the X_fit and y_fit + pass + # Use the fitted parameters to transform the data X, y + return X, y, param diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py new file mode 100644 index 00000000..3f161f6d --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/load_data.py @@ -0,0 +1,82 @@ +""" +Load competition data to uniform format +""" + +import os + +import numpy as np +import pandas as pd +from PIL import Image + + +def load_test_images(folder): + images = [] + filenames = [] + for filename in os.listdir(folder): + img = Image.open(os.path.join(folder, filename)) + if img is not None: + images.append(np.array(img)) + filenames.append(filename) + return np.array(images), filenames + + +def load_images_and_labels(csv_file, image_folder): + images = [] + labels = [] + df = pd.read_csv(csv_file) + for idx, row in df.iterrows(): + img = Image.open(os.path.join(image_folder, row["id"])) + if img is not None: + images.append(np.array(img)) + labels.append(row["has_cactus"]) + return np.array(images), np.array(labels) + + +def load_from_raw_data() -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + """ + load raw data from disk to get data in uniform data + + Return: + X: np.array + + a concrete example could be: + + .. code-block:: text + + array([[[[207, 194, 203], + ..., + [191, 183, 164], + [176, 168, 149], + [181, 173, 152]]]], dtype=uint8) + + y: np.array + + a concrete example could be: + + .. code-block:: python + + array([1, 0, 1, 0, 1, 1, ..., ]) + + X_test: np.array + + a concrete example is similar to `X`. + + test_ids: the id representing the image. it is used to generate the submission file + + a concrete example could be: + + .. code-block:: python + + ['1398ad045aa57aee5f38e7661e9d49e8.jpg', + '0051207eb794887c619341090de84b50.jpg', + 'a8202dd82c42e252bef921ada7607b6c.jpg', + '76c329ff9e3c5036b616f4e88ebba814.jpg', + ...] + """ + X, y = load_images_and_labels("/kaggle/input/train.csv", "/kaggle/input/train/") + + test_folder = "/kaggle/input/test/" + X_test, test_filenames = load_test_images(test_folder) + # Store filenames separately + test_ids = [os.path.basename(filename).replace(".tif", "") for filename in test_filenames] + return X, y, X_test, test_ids diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py new file mode 100644 index 00000000..f3a4bb78 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py @@ -0,0 +1,37 @@ +from load_data import load_from_raw_data +from sklearn.model_selection import train_test_split + +# Load data +train_images, train_labels, test_images, test_ids = load_from_raw_data() + + +# feature engineering +from feat01 import feat_eng + +train_images, train_lables, train_param = feat_eng(train_images, train_labels) +test_images, _, _ = feat_eng(test_images, param=train_param) + + +# (Cross) Validation +train_images, validation_images, train_labels, validation_labels = train_test_split( + train_images, train_labels, test_size=0.1, random_state=42 +) + + +# Model workflow +from model01 import model_workflow + +val_pred, test_pred = model_workflow(train_images, train_labels, validation_images, validation_labels, test_images) + + +# Ensemble +from ens import ens_and_decision + +pred_binary = ens_and_decision([test_pred], [val_pred], validation_labels) + + +# Save +with open("submission.csv", "w") as csv_file: + csv_file.write("id,has_cactus\n") + for tid, prediction in zip(test_ids, pred_binary): + csv_file.write(f"{tid},{prediction}\n") diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/model01.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/model01.py new file mode 100644 index 00000000..3cb54dfb --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/model01.py @@ -0,0 +1,141 @@ +import numpy as np +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint +from tensorflow.keras.layers import ( + Activation, + BatchNormalization, + Conv2D, + Dense, + Dropout, + Flatten, + MaxPooling2D, +) +from tensorflow.keras.models import Sequential +from tensorflow.keras.preprocessing.image import ImageDataGenerator + +print(tf.__version__) +print(tf.test.is_gpu_available()) + + +def model_workflow( + X: np.ndarray, + y: np.ndarray, + val_X: np.ndarray = None, + val_y: np.ndarray = None, + test_X: np.ndarray = None, + **hyper_params, +) -> tuple[np.ndarray | None, np.ndarray | None]: + """ + Manages the workflow of a machine learning model, including training, validation, and testing. + + If hyper_params is given, please get important hyperparameters from it. Otherwise, use the default values. + (the hyper_params only contains important hyperparameters that is worth tunning) + + Parameters + ---------- + X : np.ndarray + Training data features. + y : np.ndarray + Training data labels. + val_X : np.ndarray, optional + Validation data features. + val_y : np.ndarray, optional + Validation data labels. + test_X : np.ndarray, optional + Test data features. + **hyper_params + Additional hyperparameters for the model. + + Returns + ------- + tuple[np.ndarray | None, np.ndarray | None] + Predictions on the validation data, predictions on the test data + """ + train_images, train_labels = X, y + validation_images, validation_labels = val_X, val_y + test_images = test_X + + # Data augmentation is crucial for generalization, especially with small datasets. + batch_size = hyper_params.get("batch_size", 64) + + train_datagen = ImageDataGenerator(rescale=1.0 / 255, horizontal_flip=True, vertical_flip=True) + train_generator = train_datagen.flow(train_images, train_labels, batch_size=batch_size, shuffle=True) + + validation_datagen = ImageDataGenerator(rescale=1.0 / 255) + validation_generator = validation_datagen.flow(validation_images, validation_labels, batch_size=batch_size) + + # Get input shape from the training data + input_shape = X.shape[1:] + num_classes = hyper_params.get("num_classes", 2) + + # Model Creation: Convolutional Neural Network + dropout_dense_layer = hyper_params.get("dropout_dense_layer", 0.6) + + model = Sequential( + [ + Conv2D(32, (3, 3), input_shape=input_shape), + BatchNormalization(), + Activation("relu"), + Conv2D(32, (3, 3)), + BatchNormalization(), + Activation("relu"), + Conv2D(32, (3, 3)), + BatchNormalization(), + Activation("relu"), + MaxPooling2D(pool_size=(2, 2)), + Conv2D(64, (3, 3)), + BatchNormalization(), + Activation("relu"), + Conv2D(64, (3, 3)), + BatchNormalization(), + Activation("relu"), + Conv2D(64, (3, 3)), + BatchNormalization(), + Activation("relu"), + MaxPooling2D(pool_size=(2, 2)), + Conv2D(128, (3, 3)), + BatchNormalization(), + Activation("relu"), + Flatten(), + Dense(1024), + Activation("relu"), + Dropout(dropout_dense_layer), + Dense(256), + Activation("relu"), + Dropout(dropout_dense_layer), + Dense(1), + Activation("sigmoid"), + ] + ) + + model.compile( + loss=keras.losses.binary_crossentropy, + optimizer=keras.optimizers.Adam(learning_rate=hyper_params.get("learning_rate", 0.001)), + metrics=["accuracy"], + ) + + callbacks = [ + EarlyStopping(monitor="val_loss", patience=hyper_params.get("patience", 25)), + ModelCheckpoint(filepath="best_model.keras", monitor="val_loss", save_best_only=True), + ] + + # Training + epochs = hyper_params.get("epochs", 100) + history = model.fit( + train_generator, + validation_data=validation_generator, + epochs=epochs, + verbose=1, + shuffle=True, + callbacks=callbacks, + ) + # Predict on validation data + val_pred = model.predict(validation_datagen.flow(validation_images, batch_size=1, shuffle=False), verbose=1) + + # Load the test data and evaluate the model + test_datagen = ImageDataGenerator(rescale=1.0 / 255) + test_generator = test_datagen.flow(test_images, batch_size=1, shuffle=False) + + test_pred = model.predict(test_generator, verbose=1) + return val_pred, test_pred diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec.md b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec.md new file mode 100644 index 00000000..69378e07 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec.md @@ -0,0 +1,124 @@ +# Specification for Implementing a Kaggle Competition Project + +This document outlines the structure and interface protocols for implementing a machine learning project, similar to a Kaggle competition. Follow these guidelines to ensure consistency and maintainability across projects. + +## Project Structure + +The project should be organized into the following components: + +1. **Data Loading** (`load_data.py`): A module responsible for loading and preprocessing raw data. +2. **Feature Engineering**(`feat*.py`): A module for transforming raw data into features suitable for model training. +3. **Model Workflow**(`model*.py`): A module that manages the training, validation, and testing of machine learning models. +4. **Ensemble and Decision Making**(`ens.py`): A module for combining predictions from multiple models and making final decisions. +5. **Workflow**(`main.py`): A script to put the above component together to get the final submission(`submission.csv`) + +## Data Loading + +- Implement a function to load data from raw files. +- The function should return training images, training labels, test images, and test IDs. + +## Feature Engineering + +- Implement a function for feature engineering with the following signature: + +```python +def feature_eng(X: np.ndarray, y: np.ndarray | None = None, X_fit: np.ndarray | None = None, y_fit: np.ndarray | None = None, param: object | None = None) -> tuple[np.ndarray, np.ndarray | None, object]: + """ + Perform feature engineering on the input data. + + Parameters: + - X: np.ndarray + The input data to be transformed. + - y: np.ndarray | None + The target data. + - X_fit: np.ndarray | None + Data for fitting the transformation parameters. + - y_fit: np.ndarray | None + Target data for fitting. + - param: object | None + Pre-fitted parameters for transformation. + + Returns: + - transformed_data: np.ndarray + Transformed data. + - transformed_target: np.ndarray | None + Transformed target data. + - fitted_param: object + Fitted parameters. + """ +``` + +- Ensure that the feature engineering process is consistent and can be applied to both training and test data. + +## Model Workflow + +- Implement a function to manage the model workflow with the following signature: + +```python +def model_workflow(X: np.ndarray, y: np.ndarray, val_X: np.ndarray = None, val_y: np.ndarray = None, test_X: np.ndarray = None, **hyper_params) -> tuple[np.ndarray | None, np.ndarray | None]: + """ + Manages the workflow of a machine learning model, including training, validation, and testing. + + Parameters + ---------- + X : np.ndarray + Training data features. + y : np.ndarray + Training data labels. + val_X : np.ndarray, optional + Validation data features. + val_y : np.ndarray, optional + Validation data labels. + test_X : np.ndarray, optional + Test data features. + **hyper_params + Additional hyperparameters for the model. + + Returns + ------- + tuple[np.ndarray | None, np.ndarray | None] + Predictions on the validation data, predictions on the test data + """ +``` + +- The function should handle data augmentation, model creation, training, and prediction. + +## Ensemble and Decision Making + +- Implement a function for ensemble and decision making with the following signature: + +```python +def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: + """ + Handle the following: + 1) Ensemble predictions using a simple average. + 2) Make final decision after ensemble (convert the predictions to final binary form). + + Parameters + ---------- + test_pred_l : list[np.ndarray] + List of predictions on the test data. + val_pred_l : list[np.ndarray] + List of predictions on the validation data. + val_label : np.ndarray + True labels of the validation data. + + Returns + ------- + np.ndarray + Binary predictions on the test data. + """ +``` + +- The function should combine predictions and convert them to a binary format. + +## Submission + +- Implement a script to generate the submission file. +- The script should write predictions to a CSV file in the format required by the competition. + +## General Guidelines + +- Ensure that all modules and functions are well-documented. +- Follow consistent naming conventions and code style. +- Use type annotations for function signatures to improve code readability and maintainability. diff --git a/rdagent/scenarios/kaggle/tpl_ex/meta/spec.md b/rdagent/scenarios/kaggle/tpl_ex/meta/spec.md new file mode 100644 index 00000000..10f40ce4 --- /dev/null +++ b/rdagent/scenarios/kaggle/tpl_ex/meta/spec.md @@ -0,0 +1,34 @@ + + +Information to generate spec + + +```python +def feature_eng(x: {{type of the feature}}) -> {{type of the feature}}: + """ + + x: np.ndarray + {{description}} + """ +``` + +Standard to generate the qualified specification + +| field | requireemtnnts | +| -- | -- | +| description | fully describe the data, including dimension (number,meaning, exmaple)| + +Example of generated specification +```python +def feature_eng(x: {{type of the feature}}) -> {{type of the feature}}: + """ + + x: np.ndarray + 3 dimension, the meaning of the dimensions will be: + - channel + - high + - width + """ +``` + + diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 8316427c..66439e72 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -306,8 +306,6 @@ def __run( if env is None: env = {} client = docker.from_env() - if entry is None: - entry = self.conf.default_entry volumns = {} if local_path is not None: @@ -341,6 +339,7 @@ def __run( table = Table(title="Run Info", show_header=False) table.add_column("Key", style="bold cyan") table.add_column("Value", style="bold magenta") + table.add_row("Image", self.conf.image) table.add_row("Container ID", container.id) table.add_row("Container Name", container.name) table.add_row("Entry", entry) @@ -370,6 +369,8 @@ def run( env: dict | None = None, running_extra_volume: dict | None = None, ): + if entry is None: + entry = self.conf.default_entry entry_add_timeout = f"timeout {self.conf.running_timeout_period} {entry}" return self.__run(entry_add_timeout, local_path, env, running_extra_volume) diff --git a/test/utils/test_import.py b/test/utils/test_import.py index 0a1cb86a..997e8623 100644 --- a/test/utils/test_import.py +++ b/test/utils/test_import.py @@ -19,7 +19,7 @@ def import_all_modules_from_directory(directory): fstr = str(file) if "meta_tpl" in fstr: continue - if "_template" in fstr: + if "template" in fstr or "tpl" in fstr: continue if "model_coder" in fstr: continue diff --git a/test/utils/test_kaggle.py b/test/utils/test_kaggle.py index 08363814..70cd6749 100644 --- a/test/utils/test_kaggle.py +++ b/test/utils/test_kaggle.py @@ -22,8 +22,8 @@ def test_competition_template(self): download_data(competition) ws = KGFBWorkspace( template_folder_path=Path(__file__).parent.parent.parent - / "rdagent/scenarios/kaggle/experiment" - / f"{competition}_template" + / KAGGLE_IMPLEMENT_SETTING.template_path + / f"{competition}", ) print(ws.workspace_path) ws.execute()