Merge pull request #90 from 1nchaos/dev

fixed

adata/__version__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (2, 5, 1)
+VERSION = (2, 5, 3)
 PRERELEASE = None  # alpha, beta or rc
 REVISION = None
 

adata/common/headers/baidu_headers.py

@@ -5,9 +5,7 @@
 @time: 2023/3/29
 @log: change log
 """
-# cookie = 'BAIDUID=61D17458C811DF0B06467F8750EE21C2:FG=1; ab_sr=1.0.1_NDc4ZjNhNWJlNDk2N2M2MjI2ZGM2MzEwNzZiMGZiYTMyMGMyMzgxNWUxYTY1MjBhZDU5ODJhMWRjYmMxOTY5NWQ5MGI0M2MzMGMwYzQyMmUwZjcxNGQyNGZjZTZjNGJjN2NiODM3N2NlMjQ3YWNkNGJjNWVlNzI4YmI2OGI3NTMyMGE5NmQ0ZDZlZjZiODM1ZTIxNDAzMDYzMjIyZTE3ZQ=='
-cookie = 'BAIDUID=883657749CC8B8D669D673902D41EBC5:FG=1;ab_sr=1.0.1_ZDA1MjU4MjQ2MWYwNDQ1M2M5OWNiYWI5YTI1YzhlMzNlYjk5YTlmNTk1YzY2N2NmOTBkZjkwNjE1Yjg5YzUwNmJhZDQ4NmE4ZTMzZDFhZTBjODhjM2Q4MmYzMjg1Y2MzMzgzOWQ2NDI0NDk1YWFlZWEzZmNmZTk1ZTgzMTMzZTNmYjRhYWZjZWJkZDJjOTBmYzFlMGMxNDJkNzZjYWQ4Yg==; Path=/; Domain=baidu.com; Max-Age=7200; HttpOnly; Secure; SameSite=None'
-
+cookie = 'BAIDUID=61D17458C811DF0B06467F8750EE21C2:FG=1; BIDUPSID=72A958B07427F9F9CB3F63FD8B6C6565; PSTM=1721814995; H_PS_PSSID=60236_60273_60359_60468_60492_60499_60524; BA_HECTOR=0kal00ak2k84ak00858421a090v4co1jaeckm1u; ZFY=NChHwmWaGLn3NooySpSMtsO8KVHih:A9FSqxJgYQRfv0:C; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; delPer=0; PSINO=6; ab_sr=1.0.1_ZmU0ZWQyY2I0ZTNjZGU1ZTVmZjQ4Zjg1ODQwNTE0ODYzNDdjNTVlMmFkMDkxNzBkZWM5MjBmMTIxMDAwZGEyMjUxYWU5ODQyMTg5YjIwZWY4YWFkNTlmOWVhOWQzMDI0ZDcwNWMwNWIyZjQwZWI1YzQ5MzA0MjZkNjc1MjY3NTljZTgwNjhkMTIwMmYwODQ1NGUyYTZkZjY4ZmU4MDliNg=='
 json_headers = {
     'Host': 'finance.pae.baidu.com',
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0',

adata/stock/market/index_market/market_index.py

@@ -49,7 +49,8 @@ def get_market_index_current(self, index_code: str = '000001'):
         :return: [指数代码,交易时间，交易日期，开，高，低，当前价格,成交量，成交额]
         ['trade_time', 'trade_date', 'open', 'high', 'low', 'price', 'volume', 'amount']
         """
-        return self.east_index.get_market_index_current(index_code=index_code)
+        res_df = self.east_index.get_market_index_current(index_code=index_code)
+        return res_df
 
 
 if __name__ == '__main__':

adata/stock/market/index_market/market_index_east.py

@@ -29,11 +29,12 @@ def get_market_index(self, index_code: str = '000001', start_date='2020-01-01',
         :param k_type: k线类型：1.日；2.周；3.月 默认：1 日k
         :return: k线行情数据 [日期，开，高，低，收,成交量，成交额]
         """
-        sec_id = {'1': '1', '3': '0', '4': '0', '9': '0'}[index_code[0]]
-        url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get?" \
-              f"secid={sec_id}.{index_code}&fields1=f1,f2,f3,f4,f5,f6&fields2=f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61&" \
-              f"klt=10{k_type}&fqt=1&end=20500101&lmt=1000000"
-        res_json = requests.request('get', url, headers={}, proxies={}).json()
+        beg = 0
+        if start_date:
+            beg = start_date.replace('-', '')
+        sec_id = 0 if index_code[0] == 0 else 1
+        url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get?secid={sec_id}.{index_code}&ut=fa5fd1943c7b386f172d6893dbfba10b&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5%2Cf6&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58%2Cf59%2Cf60%2Cf61&klt=10{k_type}&fqt=1&beg={beg}&end=20500101&smplmt=460&lmt=1000000&_=1722235849235"
+        res_json = requests.request('post', url, headers={}, proxies={}).json()
         # 解析数据
         code = res_json['data']['code']
         if code != index_code:
@@ -65,9 +66,10 @@ def get_market_index_min(self, index_code='000001'):
         :return 时间，现价，成交额（元），均价，成交量（股） 涨跌额，涨跌幅
         ['index_code', 'trade_time', 'price', 'change', 'change_pct', 'volume', 'avg_price', 'amount']
         """
+        sec_id = 0 if index_code[0] == 0 else 1
         url = f"http://push2his.eastmoney.com/api/qt/stock/trends2/get?" \
               f"fields1=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13&fields2=f51,f52,f53,f54,f55,f56,f57,f58&" \
-              f"iscr=0&ndays=1&secid=0.{index_code}"
+              f"iscr=0&ndays=1&secid={sec_id}.{index_code}"
         res_json = requests.request('get', url, headers={}, proxies={}).json()
         # 解析数据
         code = res_json['data']['code']
@@ -101,10 +103,11 @@ def get_market_index_current(self, index_code: str = '000001'):
         :return: [指数代码,交易时间，交易日期，开，高，低，当前价格,成交量，成交额]
         ['trade_time', 'trade_date', 'open', 'high', 'low', 'price', 'volume', 'amount']
         """
+        sec_id = 0 if index_code[0] == 0 else 1
         url = f"http://push2.eastmoney.com/api/qt/stock/get?" \
               f"invt=2&fltt=1&fields=f58,f107,f57,f43,f59,f169,f170,f152,f46,f60,f44,f45,f47,f48,f19,f532,f39,f161,f49," \
               f"f171,f50,f86,f600,f601,f154,f84,f85,f168,f108,f116,f167,f164,f92,f71,f117,f292,f113,f114,f115,f119," \
-              f"f120,f121,f122,f296&secid=0.{index_code}&wbp2u=|0|0|0|web"
+              f"f120,f121,f122,f296&secid={sec_id}.{index_code}&wbp2u=|0|0|0|web"
         res_json = requests.request('get', url, headers={}, proxies={}).json()
         # 解析数据
         j = res_json['data']
@@ -129,6 +132,6 @@ def get_market_index_current(self, index_code: str = '000001'):
 
 
 if __name__ == '__main__':
-    print(StockMarketIndexEast().get_market_index(index_code='980072', start_date='2022-12-01'))
-    # print(StockMarketIndexEast().get_market_index_min(index_code='000001'))
-    # print(StockMarketIndexEast().get_market_index_current(index_code='000001'))
+    print(StockMarketIndexEast().get_market_index(index_code='000001', start_date='2022-12-01'))
+    print(StockMarketIndexEast().get_market_index_min(index_code='000001'))
+    print(StockMarketIndexEast().get_market_index_current(index_code='000001'))

adata/stock/market/stock_market/stock_market.py

@@ -40,8 +40,6 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da
         """
         df = self.east_market.get_market(stock_code=stock_code, start_date=start_date, end_date=end_date,
                                          k_type=k_type, adjust_type=adjust_type)
-        # if df.empty:
-        #     df = self.baidu_market.get_market(stock_code=stock_code, start_date=start_date, k_type=k_type)
         return df
 
     def get_market_min(self, stock_code: str = '000001'):
@@ -50,7 +48,10 @@ def get_market_min(self, stock_code: str = '000001'):
         :param stock_code: 股票代码
         :return: 当日分钟行情数据
         """
-        return self.baidu_market.get_market_min(stock_code=stock_code)
+        df = self.east_market.get_market_min(stock_code=stock_code)
+        if df.empty:
+            return self.baidu_market.get_market_min(stock_code=stock_code)
+        return df
 
     def list_market_current(self, code_list=None):
         """
@@ -92,7 +93,10 @@ def get_market_bar(self, stock_code: str = '000001'):
         :param stock_code: 股票代码
         :return: 最新当天的分时成交
         """
-        return self.baidu_market.get_market_bar(stock_code=stock_code)
+        res_df = self.baidu_market.get_market_bar(stock_code=stock_code)
+        if res_df.empty:
+            res_df = self.qq_market.get_market_bar(stock_code=stock_code)
+        return res_df
 
 
 if __name__ == '__main__':

adata/stock/market/stock_market/stock_market_baidu.py

@@ -26,6 +26,7 @@ class StockMarketBaiDu(StockMarketTemplate):
     def __init__(self) -> None:
         super().__init__()
 
+    @handler_null
     def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_date=None, k_type=1,
                    adjust_type: int = 1):
         """
@@ -94,6 +95,7 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da
         result_df['change_pct'] = result_df['change_pct'].str.replace('+', '').astype(float)
         return result_df
 
+    @handler_null
     def get_market_min(self, stock_code: str = '000001'):
         """
         获取百度的股票行情数据
@@ -183,6 +185,7 @@ def get_market_five(self, stock_code: str = '000001'):
         result_df = pd.DataFrame(data=[data], columns=self._MARKET_FIVE_COLUMNS)
         return result_df[self._MARKET_FIVE_COLUMNS]
 
+    @handler_null
     def get_market_bar(self, stock_code: str = '000001'):
         """
         https://finance.pae.baidu.com/vapi/v1/getquotation?srcid=5353&all=1&pointType=string&group=quotation_minute_ab&query=872925&code=872925&market_type=ab&newFormat=1&name=锦好医疗&finClientType=pc
@@ -205,7 +208,7 @@ def get_market_bar(self, stock_code: str = '000001'):
         # 3.1 空数据时返回为空
         result = res_json['Result']
         if not result:
-            return pd.DataFrame(data=[], columns=self._MARKET_MIN_COLUMNS)
+            return pd.DataFrame(data=[], columns=self._MARKET_BAR_COLUMNS)
 
         # 3.2. 正常解析数据
         market_data_list = res_json['Result']['detailinfos']

adata/stock/market/stock_market/stock_market_east.py

@@ -9,6 +9,7 @@
 """
 
 import pandas as pd
+from adata.common.exception.handler import handler_null
 
 from adata.common.utils import requests
 from adata.common.utils.date_utils import get_cur_time
@@ -23,6 +24,7 @@ class StockMarketEast(StockMarketTemplate):
     def __init__(self) -> None:
         super().__init__()
 
+    @handler_null
     def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_date=None, k_type=1,
                    adjust_type: int = 1):
         """
@@ -73,19 +75,51 @@ def get_market(self, stock_code: str = '000001', start_date='1990-01-01', end_da
             ['trade_time', "trade_date", "open", "close", "high", "low", "volume", "amount", "change_pct", "change",
              "turnover_ratio", "pre_close"]]
 
+    @handler_null
     def get_market_min(self, stock_code: str = '000001'):
         """
-        获取百度的股票行情数据
-        web： https://gushitong.baidu.com/stock/ab-002926
-        url: https://finance.pae.baidu.com/selfselect/getstockquotation?
-        all=1&code=601318&isIndex=false&isBk=false&isBlock=false&isFutures=false&isStock=true&newFormat=1
-        &group=quotation_minute_ab&finClientType=pc
-        time, price, ratio, increase, volume, avgPrice, amount, timeKey, datetime, oriAmount
         :param stock_code: 6位股票代码
         :return: k线行情数据:"时间","价格","涨跌率","涨幅","均价","成交量", "成交额"
         """
-        pass
+        #  # 1. 参数处理
+        se_cid = 1 if stock_code.startswith('6') else 0
+        params = {
+            "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
+            "fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
+            "ut": "fa5fd1943c7b386f172d6893dbfba10b",
+            "ndays": "1", "iscr": "1",
+            "iscca": "0", "secid": f"{se_cid}.{stock_code}",
+            "_": "1623766962675",
+        }
+        url = "https://push2.eastmoney.com/api/qt/stock/trends2/get"
+        res = requests.request(method='get', url=url, params=params).json()
+        # 2. 结果处理
+        if not res["data"]:
+            return pd.DataFrame()
+
+        # 3. 数据ETL
+        pre_close = res["data"]["preClose"]
+        data = [item.split(",") for item in res["data"]["trends"]]
+        columns = ['trade_time', 'open', 'close', 'high', 'low', 'volume', 'amount', 'price']
+        df = pd.DataFrame(data=data, columns=columns)
+        # 前面的累加求和
+        # df['avg_price'] = df['amount'].astype(float).cumsum() / df['volume'].astype(float).cumsum()/100
+        # 换算成股
+        df['volume'] = df['volume'].astype(float) * 100
+        df['stock_code'] = stock_code
+        df['avg_price'] = df['price']
+        df['price'] = df['close']
+
+        numeric_columns = ['open', 'close', 'volume', 'high', 'low', 'amount', 'price', 'avg_price']
+        df[numeric_columns] = df[numeric_columns].apply(pd.to_numeric)
+
+        df['change'] = df['price'] - pre_close
+        df['change_pct'] = df['change'] / pre_close*100
+        df['change_pct'] = df['change_pct'].round(2)
+        df.reset_index(drop=True, inplace=True)
+        return df[self._MARKET_MIN_COLUMNS]
 
 
 if __name__ == '__main__':
     print(StockMarketEast().get_market(stock_code='600020', k_type=1, adjust_type=1))
+    print(StockMarketEast().get_market_min(stock_code='600020'))

adata/stock/market/stock_market/stock_market_qq.py

@@ -8,8 +8,10 @@
 @log: change log
 """
 import pandas as pd
+from adata.common.exception.handler import handler_null
 
 from adata.common import requests
+from adata.common.utils.code_utils import get_exchange_by_stock_code
 from adata.stock.market.stock_market.stock_market_template import StockMarketTemplate
 
 
@@ -21,6 +23,7 @@ class StockMarketQQ(StockMarketTemplate):
     def __init__(self) -> None:
         super().__init__()
 
+    @handler_null
     def list_market_current(self, code_list=None):
         """
         获取多个股票最新行情信息
@@ -64,6 +67,7 @@ def list_market_current(self, code_list=None):
         result_df.loc[mask, 'amount'] = result_df['amount'].astype(float) * 10000
         return result_df[self._MARKET_CURRENT_COLUMNS]
 
+    @handler_null
     def get_market_five(self, stock_code: str = '000001'):
         """
          https://web.sqt.gtimg.cn/q=sh601666,sh600691
@@ -72,6 +76,7 @@ def get_market_five(self, stock_code: str = '000001'):
         """
         return self.list_market_five([stock_code])
 
+    @handler_null
     def list_market_five(self, code_list=None):
         """
          https://web.sqt.gtimg.cn/q=sh601666,sh600691
@@ -127,7 +132,48 @@ def list_market_five(self, code_list=None):
         result_df[columns_to_multiply] = result_df[columns_to_multiply].astype(int) * 100
         return result_df
 
+    @handler_null
+    def get_market_bar(self, stock_code: str = '000001'):
+        """
+        https://gu.qq.com/sh601857/gp/detail
+        :param stock_code: 股票代码
+        :return:
+        """
+        # 1. 参数处理
+        code = get_exchange_by_stock_code(stock_code).lower()+stock_code
+        res_df = pd.DataFrame()
+
+        # 2. 请求接口
+        for page in range(0, 10000):
+            url = "http://stock.gtimg.cn/data/index.php"
+            params = {
+                "appn": "detail",
+                "action": "data",
+                "c": code,
+                "p": page,
+            }
+            try:
+                text = requests.request(method='get', url=url, params=params).text
+                df = pd.DataFrame(eval(text[text.find("["):])[1].split("|")).iloc[:, 0].str.split("/", expand=True)
+                if df.empty:
+                    break
+            except:
+                break
+            res_df = pd.concat([res_df, df], ignore_index=True)
+        if res_df.empty:
+            return pd.DataFrame(data=[], columns=self._MARKET_BAR_COLUMNS)
+
+        # 3. 数据etl
+        # big_df = res_df.iloc[:, 1:].copy()
+        res_df.reset_index(drop=True, inplace=True)
+        res_df.columns = ['no', "trade_time", "price", "1", "volume", "2", "bs_type"]
+
+        res_df["stock_code"] = stock_code
+        res_df = res_df.astype({"trade_time": str, "price": float, "volume": int, "bs_type": str, })
+        return res_df[self._MARKET_BAR_COLUMNS]
+
 
 if __name__ == '__main__':
     print(StockMarketQQ().list_market_current(code_list=['000001', '600001', '000795', '872925']))
     print(StockMarketQQ().get_market_five(stock_code='000001'))
+    print(StockMarketQQ().get_market_bar(stock_code='000001'))

adata/stock/market/stock_market/stock_market_sina.py

@@ -9,6 +9,7 @@
 """
 
 import pandas as pd
+from adata.common.exception.handler import handler_null
 
 from adata.common.headers import sina_headers
 from adata.common.utils import requests
@@ -23,6 +24,7 @@ class StockMarketSina(StockMarketTemplate):
     def __init__(self) -> None:
         super().__init__()
 
+    @handler_null
     def list_market_current(self, code_list=None):
         """
         获取新浪的最新股票行情