From 94fc6f471c26305545179ff8e645fd50723cb1bd Mon Sep 17 00:00:00 2001 From: yutiansut Date: Fri, 23 Aug 2019 00:03:20 +0800 Subject: [PATCH 1/3] # --- QUANTAXIS/QAFetch/QAfinancial.py | 4 ++-- QUANTAXIS/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/QUANTAXIS/QAFetch/QAfinancial.py b/QUANTAXIS/QAFetch/QAfinancial.py index 558a17391..cb5667285 100755 --- a/QUANTAXIS/QAFetch/QAfinancial.py +++ b/QUANTAXIS/QAFetch/QAfinancial.py @@ -34,7 +34,7 @@ 参见PYTDX 1.65 """ -FINANCIAL_URL = 'http://down.tdx.com.cn:8001/fin/gpcw.txt' +FINANCIAL_URL = 'https://gitee.com/yutiansut/QADATA/raw/master/financial/content.txt' class QAHistoryFinancialCrawler(HistoryFinancialCrawler): @@ -98,7 +98,7 @@ def download_financialzip(): print('FILE {} is already in {}'.format(item, download_path)) else: print('CURRENTLY GET/UPDATE {}'.format(item[0:12])) - r = requests.get('http://down.tdx.com.cn:8001/fin/{}'.format(item)) + r = requests.get('https://gitee.com/yutiansut/QADATA/raw/master/financial/{}'.format(item)) file = '{}{}{}'.format(download_path, os.sep, item) diff --git a/QUANTAXIS/__init__.py b/QUANTAXIS/__init__.py index 79d40fa16..800517d8e 100755 --- a/QUANTAXIS/__init__.py +++ b/QUANTAXIS/__init__.py @@ -31,7 +31,7 @@ 2017/4/8 """ -__version__ = '1.5.10' +__version__ = '1.5.11' __author__ = 'yutiansut' import argparse From c8da9513ecbfbbaad77a286c64a5ed9069dc7ec3 Mon Sep 17 00:00:00 2001 From: yutiansut Date: Fri, 23 Aug 2019 00:24:44 +0800 Subject: [PATCH 2/3] # --- QUANTAXIS/QAFetch/QAfinancial.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/QUANTAXIS/QAFetch/QAfinancial.py b/QUANTAXIS/QAFetch/QAfinancial.py index cb5667285..e13461795 100755 --- a/QUANTAXIS/QAFetch/QAfinancial.py +++ b/QUANTAXIS/QAFetch/QAfinancial.py @@ -79,13 +79,13 @@ def get_filename(): """ get_filename """ - return [(l[0],l[1]) for l in [line.strip().split(",") for line in requests.get(FINANCIAL_URL).text.strip().split('\n')]] - + return [(l[0], l[1]) for l in [line.strip().split(",") for line in requests.get(FINANCIAL_URL).text.strip().split('\n')]] def get_md5(): return [l[1] for l in [line.strip().split(",") for line in requests.get(FINANCIAL_URL).text.strip().split('\n')]] + def download_financialzip(): """ 会创建一个download/文件夹 @@ -93,13 +93,21 @@ def download_financialzip(): result = get_filename() res = [] for item, md5 in result: - if item in os.listdir(download_path) and md5==QA_util_file_md5('{}{}{}'.format(download_path,os.sep,item)): - + if item in os.listdir(download_path) and md5 == QA_util_file_md5('{}{}{}'.format(download_path, os.sep, item)): + print('FILE {} is already in {}'.format(item, download_path)) else: print('CURRENTLY GET/UPDATE {}'.format(item[0:12])) - r = requests.get('https://gitee.com/yutiansut/QADATA/raw/master/financial/{}'.format(item)) - + r = requests.get('https://gitee.com/yutiansut/QADATA/raw/master/financial/{}'.format(item), headers={ + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36', + 'Host': 'gitee.com', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Cookie': 'user_locale=zh-CN; oschina_new_user=false; _pk_id.1.dc8b=1fcc14de2962d601.1564332911.1.1564332914.1564332911.; close_wechat_tour=true; tz=Asia%2FShanghai; Hm_lvt_24f17767262929947cc3631f99bfd274=1566137348,1566225437,1\ + 566405419,1566488922; Hm_lpvt_24f17767262929947cc3631f99bfd274=1566489980; gitee-session-n=BAh7C0kiD3Nlc3Npb25faWQGOgZFVEkiJTIxOGRkYzY3NzIyZDdiYTcwMmYwNDgwZmExYTAyODM1BjsAVEkiGXdhcmRlbi51c2VyLnVzZXIua2V5BjsAVFsHWwZpA%2BtBDUkiIiQ\ + yYSQxMCR1NEc5RXl6TzczNDk3S2ZCVGoxZ3dlBjsAVEkiHXdhcmRlbi51c2VyLnVzZXIuc2Vzc2lvbgY7AFR7BkkiFGxhc3RfcmVxdWVzdF9hdAY7AFRJdToJVGltZQ3Q3h3AYozMLQk6DW5hbm9fbnVtaQI7AzoNbmFub19kZW5pBjoNc3VibWljcm8iB4JwOgl6b25lSSIIVVRDBjsAVEkiFGFkZF9lbWF\ + pbF9ndWlkZQY7AEZGSSIXYWN0aXZlX2VtYWlsX2d1aWRlBjsARkZJIhBfY3NyZl90b2tlbgY7AEZJIjF5K1hBV0NCZlVuRXRsWFZ2anMyZXZLZ29UNzluSWF2YXRSaHI0UDIxNTA0PQY7AEY%3D--36f4ddcf2b8e48f83a84404d1a18c8b772ad6920'}) file = '{}{}{}'.format(download_path, os.sep, item) with open(file, "wb") as code: @@ -121,7 +129,7 @@ def parse_all(): """ 解析目录下的所有文件 """ - #filepath = '{}{}{}{}'.format(qa_path, os.sep, 'downloads', os.sep) + # filepath = '{}{}{}{}'.format(qa_path, os.sep, 'downloads', os.sep) filename = os.listdir(download_path) return parse_filelist(filename) From 0c0c0cebebdc952d47234bec0fae180945c202f5 Mon Sep 17 00:00:00 2001 From: yutiansut Date: Fri, 23 Aug 2019 00:34:15 +0800 Subject: [PATCH 3/3] #update financialfiles --- QUANTAXIS/QAFetch/QAfinancial.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/QUANTAXIS/QAFetch/QAfinancial.py b/QUANTAXIS/QAFetch/QAfinancial.py index e13461795..6eda5986b 100755 --- a/QUANTAXIS/QAFetch/QAfinancial.py +++ b/QUANTAXIS/QAFetch/QAfinancial.py @@ -98,16 +98,7 @@ def download_financialzip(): print('FILE {} is already in {}'.format(item, download_path)) else: print('CURRENTLY GET/UPDATE {}'.format(item[0:12])) - r = requests.get('https://gitee.com/yutiansut/QADATA/raw/master/financial/{}'.format(item), headers={ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36', - 'Host': 'gitee.com', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', - 'Accept-Encoding': 'gzip, deflate, br', - 'Connection': 'keep-alive', - 'Cookie': 'user_locale=zh-CN; oschina_new_user=false; _pk_id.1.dc8b=1fcc14de2962d601.1564332911.1.1564332914.1564332911.; close_wechat_tour=true; tz=Asia%2FShanghai; Hm_lvt_24f17767262929947cc3631f99bfd274=1566137348,1566225437,1\ - 566405419,1566488922; Hm_lpvt_24f17767262929947cc3631f99bfd274=1566489980; gitee-session-n=BAh7C0kiD3Nlc3Npb25faWQGOgZFVEkiJTIxOGRkYzY3NzIyZDdiYTcwMmYwNDgwZmExYTAyODM1BjsAVEkiGXdhcmRlbi51c2VyLnVzZXIua2V5BjsAVFsHWwZpA%2BtBDUkiIiQ\ - yYSQxMCR1NEc5RXl6TzczNDk3S2ZCVGoxZ3dlBjsAVEkiHXdhcmRlbi51c2VyLnVzZXIuc2Vzc2lvbgY7AFR7BkkiFGxhc3RfcmVxdWVzdF9hdAY7AFRJdToJVGltZQ3Q3h3AYozMLQk6DW5hbm9fbnVtaQI7AzoNbmFub19kZW5pBjoNc3VibWljcm8iB4JwOgl6b25lSSIIVVRDBjsAVEkiFGFkZF9lbWF\ - pbF9ndWlkZQY7AEZGSSIXYWN0aXZlX2VtYWlsX2d1aWRlBjsARkZJIhBfY3NyZl90b2tlbgY7AEZJIjF5K1hBV0NCZlVuRXRsWFZ2anMyZXZLZ29UNzluSWF2YXRSaHI0UDIxNTA0PQY7AEY%3D--36f4ddcf2b8e48f83a84404d1a18c8b772ad6920'}) + r = requests.get('http://data.yutiansut.com/{}'.format(item)) file = '{}{}{}'.format(download_path, os.sep, item) with open(file, "wb") as code: