From f33751ab183a66bbbdcd0971130fa54b9ef652a5 Mon Sep 17 00:00:00 2001 From: 1nchaos <9527@1nchaos.com> Date: Mon, 29 Jul 2024 18:03:11 +0800 Subject: [PATCH] fixed v-2.5.3 index bug --- adata/__version__.py | 2 +- adata/common/headers/baidu_headers.py | 4 +- .../stock/market/index_market/market_index.py | 3 +- .../market/index_market/market_index_east.py | 23 +++++---- .../stock/market/stock_market/stock_market.py | 12 +++-- .../market/stock_market/stock_market_baidu.py | 5 +- .../market/stock_market/stock_market_east.py | 48 ++++++++++++++++--- .../market/stock_market/stock_market_qq.py | 46 ++++++++++++++++++ .../market/stock_market/stock_market_sina.py | 2 + 9 files changed, 118 insertions(+), 27 deletions(-) diff --git a/adata/__version__.py b/adata/__version__.py index 2f1ebd4..dd72924 100644 --- a/adata/__version__.py +++ b/adata/__version__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -VERSION = (2, 5, 1) +VERSION = (2, 5, 3) PRERELEASE = None # alpha, beta or rc REVISION = None diff --git a/adata/common/headers/baidu_headers.py b/adata/common/headers/baidu_headers.py index 4833e5c..96d7a13 100644 --- a/adata/common/headers/baidu_headers.py +++ b/adata/common/headers/baidu_headers.py @@ -5,9 +5,7 @@ @time: 2023/3/29 @log: change log """ -# cookie = 'BAIDUID=61D17458C811DF0B06467F8750EE21C2:FG=1; ab_sr=1.0.1_NDc4ZjNhNWJlNDk2N2M2MjI2ZGM2MzEwNzZiMGZiYTMyMGMyMzgxNWUxYTY1MjBhZDU5ODJhMWRjYmMxOTY5NWQ5MGI0M2MzMGMwYzQyMmUwZjcxNGQyNGZjZTZjNGJjN2NiODM3N2NlMjQ3YWNkNGJjNWVlNzI4YmI2OGI3NTMyMGE5NmQ0ZDZlZjZiODM1ZTIxNDAzMDYzMjIyZTE3ZQ==' -cookie = 'BAIDUID=883657749CC8B8D669D673902D41EBC5:FG=1;ab_sr=1.0.1_ZDA1MjU4MjQ2MWYwNDQ1M2M5OWNiYWI5YTI1YzhlMzNlYjk5YTlmNTk1YzY2N2NmOTBkZjkwNjE1Yjg5YzUwNmJhZDQ4NmE4ZTMzZDFhZTBjODhjM2Q4MmYzMjg1Y2MzMzgzOWQ2NDI0NDk1YWFlZWEzZmNmZTk1ZTgzMTMzZTNmYjRhYWZjZWJkZDJjOTBmYzFlMGMxNDJkNzZjYWQ4Yg==; Path=/; Domain=baidu.com; Max-Age=7200; HttpOnly; Secure; SameSite=None' - +cookie = 'BAIDUID=61D17458C811DF0B06467F8750EE21C2:FG=1; BIDUPSID=72A958B07427F9F9CB3F63FD8B6C6565; PSTM=1721814995; H_PS_PSSID=60236_60273_60359_60468_60492_60499_60524; BA_HECTOR=0kal00ak2k84ak00858421a090v4co1jaeckm1u; ZFY=NChHwmWaGLn3NooySpSMtsO8KVHih:A9FSqxJgYQRfv0:C; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; delPer=0; PSINO=6; ab_sr=1.0.1_ZmU0ZWQyY2I0ZTNjZGU1ZTVmZjQ4Zjg1ODQwNTE0ODYzNDdjNTVlMmFkMDkxNzBkZWM5MjBmMTIxMDAwZGEyMjUxYWU5ODQyMTg5YjIwZWY4YWFkNTlmOWVhOWQzMDI0ZDcwNWMwNWIyZjQwZWI1YzQ5MzA0MjZkNjc1MjY3NTljZTgwNjhkMTIwMmYwODQ1NGUyYTZkZjY4ZmU4MDliNg==' json_headers = { 'Host': 'finance.pae.baidu.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0', diff --git a/adata/stock/market/index_market/market_index.py b/adata/stock/market/index_market/market_index.py index e4d4a15..ae9b143 100644 --- a/adata/stock/market/index_market/market_index.py +++ b/adata/stock/market/index_market/market_index.py @@ -49,7 +49,8 @@ def get_market_index_current(self, index_code: str = '000001'): :return: [指数代码,交易时间，交易日期，开，高，低，当前价格,成交量，成交额] ['trade_time', 'trade_date', 'open', 'high', 'low', 'price', 'volume', 'amount'] """ - return self.east_index.get_market_index_current(index_code=index_code) + res_df = self.east_index.get_market_index_current(index_code=index_code) + return res_df if __name__ == '__main__': diff --git a/adata/stock/market/index_market/market_index_east.py b/adata/stock/market/index_market/market_index_east.py index 4118368..f75bfe2 100644 --- a/adata/stock/market/index_market/market_index_east.py +++ b/adata/stock/market/index_market/market_index_east.py @@ -29,11 +29,12 @@ def get_market_index(self, index_code: str = '000001', start_date='2020-01-01', :param k_type: k线类型：1.日；2.周；3.月 默认：1 日k :return: k线行情数据 [日期，开，高，低，收,成交量，成交额] """ - sec_id = {'1': '1', '3': '0', '4': '0', '9': '0'}[index_code[0]] - url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get?" \ - f"secid={sec_id}.{index_code}&fields1=f1,f2,f3,f4,f5,f6&fields2=f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61&" \ - f"klt=10{k_type}&fqt=1&end=20500101&lmt=1000000" - res_json = requests.request('get', url, headers={}, proxies={}).json() + beg = 0 + if start_date: + beg = start_date.replace('-', '') + sec_id = 0 if index_code[0] == 0 else 1 + url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get?secid={sec_id}.{index_code}&ut=fa5fd1943c7b386f172d6893dbfba10b&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5%2Cf6&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58%2Cf59%2Cf60%2Cf61&klt=10{k_type}&fqt=1&beg={beg}&end=20500101&smplmt=460&lmt=1000000&_=1722235849235" + res_json = requests.request('post', url, headers={}, proxies={}).json() # 解析数据 code = res_json['data']['code'] if code != index_code: @@ -65,9 +66,10 @@ def get_market_index_min(self, index_code='000001'): :return 时间，现价，成交额（元），均价，成交量（股） 涨跌额，涨跌幅 ['index_code', 'trade_time', 'price', 'change', 'change_pct', 'volume', 'avg_price', 'amount'] """ + sec_id = 0 if index_code[0] == 0 else 1 url = f"http://push2his.eastmoney.com/api/qt/stock/trends2/get?" \ f"fields1=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13&fields2=f51,f52,f53,f54,f55,f56,f57,f58&" \ - f"iscr=0&ndays=1&secid=0.{index_code}" + f"iscr=0&ndays=1&secid={sec_id}.{index_code}" res_json = requests.request('get', url, headers={}, proxies={}).json() # 解析数据 code = res_json['data']['code'] @@ -101,10 +103,11 @@ def get_market_index_current(self, index_code: str = '000001'): :return: [指数代码,交易时间，交易日期，开，高，低，当前价格,成交量，成交额] ['trade_time', 'trade_date', 'open', 'high', 'low', 'price', 'volume', 'amount'] """ + sec_id = 0 if index_code[0] == 0 else 1 url = f"http://push2.eastmoney.com/api/qt/stock/get?" \ f"invt=2&fltt=1&fields=f58,f107,f57,f43,f59,f169,f170,f152,f46,f60,f44,f45,f47,f48,f19,f532,f39,f161,f49," \ f"f171,f50,f86,f600,f601,f154,f84,f85,f168,f108,f116,f167,f164,f92,f71,f117,f292,f113,f114,f115,f119," \ - f"f120,f121,f122,f296&secid=0.{index_code}&wbp2u=|0|0|0|web" + f"f120,f121,f122,f296&secid={sec_id}.{index_code}&wbp2u=|0|0|0|web" res_json = requests.request('get', url, headers={}, proxies={}).json() # 解析数据 j = res_json['data'] @@ -129,6 +132,6 @@ def get_market_index_current(self, index_code: str = '000001'): if __name__ == '__main__': - print(StockMarketIndexEast().get_market_index(index_code='980072', start_date='2022-12-01')) - # print(StockMarketIndexEast().get_market_index_min(index_code='000001')) - # print(StockMarketIndexEast().get_market_index_current(index_code='000001')) + print(StockMarketIndexEast().get_market_index(index_code='000001', start_date='2022-12-01')) + print(StockMarketIndexEast().get_market_index_min(index_code='000001')) + print(StockMarketIndexEast().get_market_index_current(index_code='000001')) diff --git a/adata/stock/market/stock_market/stock_market.py b/adata/stock/market/stock_market/stock_market.py index 155285c..ab99f3a 100644 --- a/adata/stock/market/stock_market/stock_market.py +++ b/adata/stock/market/stock_market/stock_market.py @@ -40,8 +40,6 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da """ df = self.east_market.get_market(stock_code=stock_code, start_date=start_date, end_date=end_date, k_type=k_type, adjust_type=adjust_type) - # if df.empty: - # df = self.baidu_market.get_market(stock_code=stock_code, start_date=start_date, k_type=k_type) return df def get_market_min(self, stock_code: str = '000001'): @@ -50,7 +48,10 @@ def get_market_min(self, stock_code: str = '000001'): :param stock_code: 股票代码 :return: 当日分钟行情数据 """ - return self.baidu_market.get_market_min(stock_code=stock_code) + df = self.east_market.get_market_min(stock_code=stock_code) + if df.empty: + return self.baidu_market.get_market_min(stock_code=stock_code) + return df def list_market_current(self, code_list=None): """ @@ -92,7 +93,10 @@ def get_market_bar(self, stock_code: str = '000001'): :param stock_code: 股票代码 :return: 最新当天的分时成交 """ - return self.baidu_market.get_market_bar(stock_code=stock_code) + res_df = self.baidu_market.get_market_bar(stock_code=stock_code) + if res_df.empty: + res_df = self.qq_market.get_market_bar(stock_code=stock_code) + return res_df if __name__ == '__main__': diff --git a/adata/stock/market/stock_market/stock_market_baidu.py b/adata/stock/market/stock_market/stock_market_baidu.py index 0ccf0ce..d3b6a07 100644 --- a/adata/stock/market/stock_market/stock_market_baidu.py +++ b/adata/stock/market/stock_market/stock_market_baidu.py @@ -26,6 +26,7 @@ class StockMarketBaiDu(StockMarketTemplate): def __init__(self) -> None: super().__init__() + @handler_null def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_date=None, k_type=1, adjust_type: int = 1): """ @@ -94,6 +95,7 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da result_df['change_pct'] = result_df['change_pct'].str.replace('+', '').astype(float) return result_df + @handler_null def get_market_min(self, stock_code: str = '000001'): """ 获取百度的股票行情数据 @@ -183,6 +185,7 @@ def get_market_five(self, stock_code: str = '000001'): result_df = pd.DataFrame(data=[data], columns=self._MARKET_FIVE_COLUMNS) return result_df[self._MARKET_FIVE_COLUMNS] + @handler_null def get_market_bar(self, stock_code: str = '000001'): """ https://finance.pae.baidu.com/vapi/v1/getquotation?srcid=5353&all=1&pointType=string&group=quotation_minute_ab&query=872925&code=872925&market_type=ab&newFormat=1&name=锦好医疗&finClientType=pc @@ -205,7 +208,7 @@ def get_market_bar(self, stock_code: str = '000001'): # 3.1 空数据时返回为空 result = res_json['Result'] if not result: - return pd.DataFrame(data=[], columns=self._MARKET_MIN_COLUMNS) + return pd.DataFrame(data=[], columns=self._MARKET_BAR_COLUMNS) # 3.2. 正常解析数据 market_data_list = res_json['Result']['detailinfos'] diff --git a/adata/stock/market/stock_market/stock_market_east.py b/adata/stock/market/stock_market/stock_market_east.py index 5fb22ef..39c9774 100644 --- a/adata/stock/market/stock_market/stock_market_east.py +++ b/adata/stock/market/stock_market/stock_market_east.py @@ -9,6 +9,7 @@ """ import pandas as pd +from adata.common.exception.handler import handler_null from adata.common.utils import requests from adata.common.utils.date_utils import get_cur_time @@ -23,6 +24,7 @@ class StockMarketEast(StockMarketTemplate): def __init__(self) -> None: super().__init__() + @handler_null def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_date=None, k_type=1, adjust_type: int = 1): """ @@ -73,19 +75,51 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da ['trade_time', "trade_date", "open", "close", "high", "low", "volume", "amount", "change_pct", "change", "turnover_ratio", "pre_close"]] + @handler_null def get_market_min(self, stock_code: str = '000001'): """ - 获取百度的股票行情数据 - web： https://gushitong.baidu.com/stock/ab-002926 - url: https://finance.pae.baidu.com/selfselect/getstockquotation? - all=1&code=601318&isIndex=false&isBk=false&isBlock=false&isFutures=false&isStock=true&newFormat=1 - &group=quotation_minute_ab&finClientType=pc - time, price, ratio, increase, volume, avgPrice, amount, timeKey, datetime, oriAmount :param stock_code: 6位股票代码 :return: k线行情数据:"时间","价格","涨跌率","涨幅","均价","成交量", "成交额" """ - pass + # # 1. 参数处理 + se_cid = 1 if stock_code.startswith('6') else 0 + params = { + "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13", + "fields2": "f51,f52,f53,f54,f55,f56,f57,f58", + "ut": "fa5fd1943c7b386f172d6893dbfba10b", + "ndays": "1", "iscr": "1", + "iscca": "0", "secid": f"{se_cid}.{stock_code}", + "_": "1623766962675", + } + url = "https://push2.eastmoney.com/api/qt/stock/trends2/get" + res = requests.request(method='get', url=url, params=params).json() + # 2. 结果处理 + if not res["data"]: + return pd.DataFrame() + + # 3. 数据ETL + pre_close = res["data"]["preClose"] + data = [item.split(",") for item in res["data"]["trends"]] + columns = ['trade_time', 'open', 'close', 'high', 'low', 'volume', 'amount', 'price'] + df = pd.DataFrame(data=data, columns=columns) + # 前面的累加求和 + # df['avg_price'] = df['amount'].astype(float).cumsum() / df['volume'].astype(float).cumsum()/100 + # 换算成股 + df['volume'] = df['volume'].astype(float) * 100 + df['stock_code'] = stock_code + df['avg_price'] = df['price'] + df['price'] = df['close'] + + numeric_columns = ['open', 'close', 'volume', 'high', 'low', 'amount', 'price', 'avg_price'] + df[numeric_columns] = df[numeric_columns].apply(pd.to_numeric) + + df['change'] = df['price'] - pre_close + df['change_pct'] = df['change'] / pre_close*100 + df['change_pct'] = df['change_pct'].round(2) + df.reset_index(drop=True, inplace=True) + return df[self._MARKET_MIN_COLUMNS] if __name__ == '__main__': print(StockMarketEast().get_market(stock_code='600020', k_type=1, adjust_type=1)) + print(StockMarketEast().get_market_min(stock_code='600020')) diff --git a/adata/stock/market/stock_market/stock_market_qq.py b/adata/stock/market/stock_market/stock_market_qq.py index 7db348e..7d7ac81 100644 --- a/adata/stock/market/stock_market/stock_market_qq.py +++ b/adata/stock/market/stock_market/stock_market_qq.py @@ -8,8 +8,10 @@ @log: change log """ import pandas as pd +from adata.common.exception.handler import handler_null from adata.common import requests +from adata.common.utils.code_utils import get_exchange_by_stock_code from adata.stock.market.stock_market.stock_market_template import StockMarketTemplate @@ -21,6 +23,7 @@ class StockMarketQQ(StockMarketTemplate): def __init__(self) -> None: super().__init__() + @handler_null def list_market_current(self, code_list=None): """ 获取多个股票最新行情信息 @@ -64,6 +67,7 @@ def list_market_current(self, code_list=None): result_df.loc[mask, 'amount'] = result_df['amount'].astype(float) * 10000 return result_df[self._MARKET_CURRENT_COLUMNS] + @handler_null def get_market_five(self, stock_code: str = '000001'): """ https://web.sqt.gtimg.cn/q=sh601666,sh600691 @@ -72,6 +76,7 @@ def get_market_five(self, stock_code: str = '000001'): """ return self.list_market_five([stock_code]) + @handler_null def list_market_five(self, code_list=None): """ https://web.sqt.gtimg.cn/q=sh601666,sh600691 @@ -127,7 +132,48 @@ def list_market_five(self, code_list=None): result_df[columns_to_multiply] = result_df[columns_to_multiply].astype(int) * 100 return result_df + @handler_null + def get_market_bar(self, stock_code: str = '000001'): + """ + https://gu.qq.com/sh601857/gp/detail + :param stock_code: 股票代码 + :return: + """ + # 1. 参数处理 + code = get_exchange_by_stock_code(stock_code).lower()+stock_code + res_df = pd.DataFrame() + + # 2. 请求接口 + for page in range(0, 10000): + url = "http://stock.gtimg.cn/data/index.php" + params = { + "appn": "detail", + "action": "data", + "c": code, + "p": page, + } + try: + text = requests.request(method='get', url=url, params=params).text + df = pd.DataFrame(eval(text[text.find("["):])[1].split("|")).iloc[:, 0].str.split("/", expand=True) + if df.empty: + break + except: + break + res_df = pd.concat([res_df, df], ignore_index=True) + if res_df.empty: + return pd.DataFrame(data=[], columns=self._MARKET_BAR_COLUMNS) + + # 3. 数据etl + # big_df = res_df.iloc[:, 1:].copy() + res_df.reset_index(drop=True, inplace=True) + res_df.columns = ['no', "trade_time", "price", "1", "volume", "2", "bs_type"] + + res_df["stock_code"] = stock_code + res_df = res_df.astype({"trade_time": str, "price": float, "volume": int, "bs_type": str, }) + return res_df[self._MARKET_BAR_COLUMNS] + if __name__ == '__main__': print(StockMarketQQ().list_market_current(code_list=['000001', '600001', '000795', '872925'])) print(StockMarketQQ().get_market_five(stock_code='000001')) + print(StockMarketQQ().get_market_bar(stock_code='000001')) diff --git a/adata/stock/market/stock_market/stock_market_sina.py b/adata/stock/market/stock_market/stock_market_sina.py index c24d7e6..f2b0e6b 100644 --- a/adata/stock/market/stock_market/stock_market_sina.py +++ b/adata/stock/market/stock_market/stock_market_sina.py @@ -9,6 +9,7 @@ """ import pandas as pd +from adata.common.exception.handler import handler_null from adata.common.headers import sina_headers from adata.common.utils import requests @@ -23,6 +24,7 @@ class StockMarketSina(StockMarketTemplate): def __init__(self) -> None: super().__init__() + @handler_null def list_market_current(self, code_list=None): """ 获取新浪的最新股票行情