Merge pull request #22 from 1nchaos/dev

Dev

HISTORY.md

@@ -1,13 +1,29 @@
 Release History
 ===============
 
-**A Data**
+![logo](/favicon.ico)[AData](https://github.com/1nchaos/adata)
 
 master
 ------
 开放、纯净、持续
 
+专注股票量化数据，为Ai(爱)发电，向阳而生。
+
+0.0.27b0 (2023-07-10)
+------------------
+1. 新增代理设置，可设置代理ip进行数据获取。
+2. 部分函数增加，单个请求等待时间的设置，可避免限流封控。
+
+0.0.26b0 (2023-07-08)
+------------------
+1. 优化交易日历，增加缓存数据
+2. 增加行情测试用例
+3. 修复指数代码转换问题
+
+......
+------------------
+
 0.0.1b0 (2023-04-05)
 ------------------
 清明时节雨纷纷
-第一个测试预览版本
+第一个测试预览版本

README.md

@@ -178,6 +178,7 @@ print(res_df)
 | 数据                     | API                                  | 说明                           | 备注                                                      |
 | ------------------------ | ------------------------------------ | ------------------------------ | --------------------------------------------------------- |
 | 最近一个月的股票解禁列表 | sentiment.stock_lifting_last_month() | 查询最近一个月的股票解禁列表   | 来源：1. [同花顺](http://data.10jqka.com.cn/market/xsjj/) |
+| 全市场融资融券余额列表   | sentiment.securities_margin          | 查询全市场融资融券余额列表     | 来源：1. [东方财富](https://data.eastmoney.com/rzrq/)     |
 | 其它数据排期中           | TODO                                 | 若您有相关资源可以一起参与贡献 |                                                           |
 
 ## 三、数据源

adata/common/headers/east_headers.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+"""
+@desc: readme
+@author: yinchao
+@time: 2023/7/24
+@log: change log
+"""
+json_headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0',
+    'Accept': '*/*',
+    'Content-Type': 'text/plain;charset=UTF-8',
+    'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'HOST': 'datacenter-web.eastmoney.com',
+    'Connection': 'keep-alive',
+    'Referer': 'https://data.eastmoney.com/rzrq/total.html',
+}

adata/common/utils/sunrequests.py

@@ -46,37 +46,47 @@ def __init__(self, sun_proxy: SunProxy = None) -> None:
         super().__init__()
         self.sun_proxy = sun_proxy
 
-    def request(self, method='get', url=None, times=3, sleep_time=1588, proxies=None, **kwargs):
+    def request(self, method='get', url=None, times=3, retry_wait_time=1588, proxies=None, wait_time=None, **kwargs):
         """
         简单封装的请求，参考requests，增加循环次数和次数之间的等待时间
         :param proxies: 代理配置
         :param method: 请求方法： get；post
         :param url: url
         :param times: 次数，int
-        :param sleep_time: 循环的等待时间，毫秒
+        :param retry_wait_time: 重试等待时间，毫秒
+        :param wait_time: 等待时间：毫秒；表示每个请求的间隔时间，在请求之前等待sleep，主要用于防止请求太频繁的限制。
         :param kwargs: 其它 requests 参数，用法相同
         :return: res
         """
         # 1. 获取设置代理
-        if proxies is None:
-            proxies = {}
-        is_proxy = SunProxy.get('is_proxy')
-        ip = SunProxy.get('ip')
-        proxy_url = SunProxy.get('proxy_url')
-        if not ip and is_proxy and proxy_url:
-            ip = requests.get(url=proxy_url).text.replace('\r\n', '')
-        if is_proxy and ip:
-            proxies = {'https': f"http://{ip}", 'http': f"http://{ip}"}
+        proxies = self.__get_proxies(proxies)
         # 2. 请求数据结果
         res = None
         for i in range(times):
+            if wait_time:
+                time.sleep(wait_time / 1000)
             res = requests.request(method=method, url=url, proxies=proxies, **kwargs)
             if res.status_code in (200, 404):
                 return res
-            time.sleep(sleep_time / 1000)
+            time.sleep(retry_wait_time / 1000)
             if i == times - 1:
                 return res
         return res
 
+    def __get_proxies(self, proxies):
+        """
+        获取代理配置
+        """
+        if proxies is None:
+            proxies = {}
+        is_proxy = SunProxy.get('is_proxy')
+        ip = SunProxy.get('ip')
+        proxy_url = SunProxy.get('proxy_url')
+        if not ip and is_proxy and proxy_url:
+            ip = requests.get(url=proxy_url).text.replace('\r\n', '')
+        if is_proxy and ip:
+            proxies = {'https': f"http://{ip}", 'http': f"http://{ip}"}
+        return proxies
+
 
 sun_requests = SunRequests()

adata/sentiment/__init__.py

@@ -5,10 +5,11 @@
 @time:2023/04/06
 @log: 
 """
+from adata.sentiment.securities_margin import SecuritiesMargin
 from adata.sentiment.stock_lifting import StockLifting
 
 
-class Sentiment(StockLifting):
+class Sentiment(StockLifting, SecuritiesMargin):
 
     def __init__(self) -> None:
         super().__init__()

adata/sentiment/securities_margin.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""
+@desc:
+融资融券数据
+同花顺
+http://data.10jqka.com.cn/market/rzrq/
+http://data.10jqka.com.cn/market/rzrq/board/getRzrqPage/page/2/ajax/1/
+东方财富
+https://data.eastmoney.com/rzrq/total.html
+https://datacenter-web.eastmoney.com/api/data/v1/get?reportName=RPTA_RZRQ_LSHJ&columns=ALL&source=WEB&sortColumns=dim_date&sortTypes=-1&pageNumber=2&pageSize=50&filter=&pageNo=2&p=2&pageNum=2&_=1690176931022
+
+@author: 1nchaos
+@time: 2023/7/24
+@log: change log
+"""
+from datetime import datetime
+
+import pandas as pd
+
+from adata.common import requests
+from adata.common.headers import east_headers
+
+
+class SecuritiesMargin(object):
+    __SECURITIES_MARGIN_COLUMN = ['trade_date', 'rzye', 'rqye', 'rzrqye', 'rzrqyecz']
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def securities_margin(self, start_date=None):
+        """
+        查询开始时间到现在的融资融券余额数据，默认：查询最近一年的数据
+        :return:  ['trade_date', 'rzye', 'rqye', 'rzrqye', 'rzrqyecz']
+        trade_date: 交易日
+        rzye： 融资余额（元）
+        rqye： 融券余额（元）
+        rzrqye： 融资融券余额（元）
+        rzrqyecz： 融资融券余额差值（元）
+        """
+        return self.__securities_margin_east(start_date=start_date)
+
+    def __securities_margin_east(self, start_date=None):
+        """
+        https://datacenter-web.eastmoney.com/api/data/v1/get?reportName=RPTA_RZRQ_LSHJ&columns=ALL&source=WEB&sortColumns=dim_date&sortTypes=-1&pageNumber=1&pageSize=250&_=1690176931022
+        :param start_date: 开始时间
+        :return:
+        """
+
+        # 1. url拼接页码等参数
+        data = []
+        total_pages = 1
+        curr_page = 1
+        page_size = 250
+        start_date_str = start_date
+        if start_date:
+            start_date = datetime.strptime(start_date, '%Y-%m-%d')
+        while curr_page <= total_pages:
+            api_url = f"https://datacenter-web.eastmoney.com/api/data/v1/get?" \
+                      f"reportName=RPTA_RZRQ_LSHJ&columns=ALL&source=WEB&sortColumns=dim_date&sortTypes=-1&" \
+                      f"pageNumber={curr_page}&pageSize={page_size}&_=1690176931022"
+
+            res = requests.request(method='get', url=api_url, headers=east_headers.json_headers, proxies={})
+            # 2. 判断请求是否成功
+            if res.status_code != 200:
+                continue
+            res_json = res.json()
+            if not res_json['success']:
+                continue
+            if curr_page == 1:
+                total_pages = res_json['result']['pages']
+            res_json = res_json['result']['data']
+            # 2.1 日期范围判断
+            data.extend(res_json)
+            if not start_date:
+                break
+            if start_date:
+                date_min = datetime.strptime(res_json[-1]['DIM_DATE'], '%Y-%m-%d %H:%M:%S')
+                if start_date >= date_min:
+                    break
+            curr_page += 1
+
+        # 3. 解析数据
+        result_df = pd.DataFrame(data=data)
+        rename_columns = {'RZYE': 'rzye', 'RQYE': 'rqye', 'RZRQYE': 'rzrqye', 'RZRQYECZ': 'rzrqyecz',
+                          'DIM_DATE': 'trade_date'}
+        result_df = result_df.rename(columns=rename_columns)[self.__SECURITIES_MARGIN_COLUMN]
+
+        # 4. 数据清洗
+        result_df['trade_date'] = pd.to_datetime(result_df['trade_date']).dt.strftime('%Y-%m-%d')
+        result_df = result_df[result_df['trade_date'] > start_date_str]
+        return result_df
+
+
+if __name__ == '__main__':
+    print(SecuritiesMargin().securities_margin('2022-01-01'))

adata/stock/info/stock_concept.py

@@ -135,13 +135,11 @@ def __index_constituent_ths_by_concept_code(self, concept_code=None, wait_time=N
         total_pages = 1
         curr_page = 1
         while curr_page <= total_pages:
-            if curr_page != 1 and wait_time:
-                time.sleep(wait_time / 1000)
             api_url = f"http://q.10jqka.com.cn/gn/detail/field/199112/order/desc/page/" \
                       f"{curr_page}/ajax/1/code/{concept_code}"
             headers = copy.deepcopy(ths_headers.text_headers)
             headers['Cookie'] = cookie.ths_cookie()
-            res = requests.request(method='get', url=api_url, headers=headers, proxies={})
+            res = requests.request(method='get', url=api_url, headers=headers, proxies={}, wait_time=wait_time)
             curr_page += 1
             # 2. 判断请求是否成功
             if res.status_code != 200:
@@ -183,7 +181,7 @@ def __index_constituent_ths_by_index_code(self, index_code=None, wait_time=None)
         api_url = f"https://d.10jqka.com.cn/v2/blockrank/{index_code}/8/d15.js"
         headers = copy.deepcopy(ths_headers.text_headers)
         headers['Host'] = 'd.10jqka.com.cn'
-        res = requests.request(method='get', url=api_url, headers=headers, proxies={})
+        res = requests.request(method='get', url=api_url, headers=headers, proxies={}, wait_time=wait_time)
         # 同花顺可能ip限制，降低请求次数
         text = res.text
         if THS_IP_LIMIT_RES in text:
@@ -201,7 +199,7 @@ def __index_constituent_ths_by_index_code(self, index_code=None, wait_time=None)
 
         # 3. 请求所有数据
         for api_url in apis:
-            res = requests.request(method='get', url=api_url, headers=headers, proxies={})
+            res = requests.request(method='get', url=api_url, headers=headers, proxies={}, wait_time=wait_time)
             text = res.text
             result_json = json.loads(text[text.index('{'):-1])
             items = result_json['items']
@@ -226,14 +224,12 @@ def __index_constituent_ths_by_name(self, name=None, wait_time=None):
         total_pages = 1
         curr_page = 1
         while curr_page <= total_pages:
-            if curr_page != 1 and wait_time:
-                time.sleep(wait_time / 1000)
             api_url = f"https://www.iwencai.com/gateway/urp/v7/landing/getDataList?query={name} 概念成分&" \
                       f"page={curr_page}&perpage=100&query_type=stock&comp_id=6734520&uuid=24087"
             headers = copy.deepcopy(ths_headers.json_headers)
             headers['Host'] = 'www.iwencai.com'
             headers['Sec-Fetch-Mode'] = 'navigate'
-            res = requests.request(method='get', url=api_url, headers=headers, proxies={})
+            res = requests.request(method='get', url=api_url, headers=headers, proxies={}, wait_time=wait_time)
             curr_page += 1
             # 2. 判断请求是否成功
             if res.status_code != 200:

tests/adata_test/adata_test.py

@@ -11,7 +11,7 @@
     # defaultTestLoader测试加载器：包含加载测试用例的方法;使用discover()方法来自动识别并添加测试用例(多个)
     suite = unittest.defaultTestLoader.discover("", '*_test.py')
     # 生成一个本地时间 格式如20220413091429，年月时时分秒
-    time_str = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
+    time_str = time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(time.time()))
     # 生成的报告的名字和目录
     filename = f"./AData-{version}自动化测试报告-" + time_str + ".html"
     # 打开这个文件

tests/adata_test/sentiment/sentiment_test.py

@@ -25,6 +25,12 @@ def test_stock_lifting_last_month(self):
         print(df)
         self.assertEqual(True, len(df) > 1)
 
+    def test_securities_margin(self):
+        print("开始测试：test_securities_margin")
+        df = adata.sentiment.securities_margin(start_date='2020-01-01')
+        print(df)
+        self.assertEqual(True, len(df) > 250)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/adata_test/stock/info_test.py

@@ -38,12 +38,24 @@ def test_all_concept_code_ths(self):
         print(df)
         self.assertEqual(True, len(df) > 300)
 
+    def test_concept_constituent_ths(self):
+        print("开始测试：test_concept_constituent_ths")
+        df = adata.stock.info.concept_constituent_ths(index_code="885556", wait_time=4000)
+        print(df)
+        self.assertEqual(True, len(df) > 10)
+
     def test_all_index_code(self):
         print("开始测试：test_all_index_code")
         df = adata.stock.info.all_index_code()
         print(df)
         self.assertEqual(True, len(df) > 500)
 
+    def test_index_constituent(self):
+        print("开始测试：test_index_constituent")
+        df = adata.stock.info.index_constituent(index_code="000033", wait_time=4000)
+        print(df)
+        self.assertEqual(True, len(df) > 10)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/adata_test/stock/market_concept_test.py

@@ -5,10 +5,9 @@
 @time: 2023/7/2
 @log: change log
 """
-import time
 import unittest
-
 import adata
+import time
 
 
 class MarketConceptTestCase(unittest.TestCase):
@@ -29,7 +28,6 @@ def tearDown(self):
 
     def test_get_market_concept_ths(self):
         print("开始测试：test_get_market_concept_ths")
-        time.sleep(5)
         df = adata.stock.market.get_market_concept_ths(index_code='886041')
         print(df)
         self.assertEqual(True, len(df) > 30)