Dev

README.md

@@ -8,7 +8,7 @@
 
 ![GitHub language count](https://img.shields.io/github/languages/count/1nchaos/adata)![GitHub top language](https://img.shields.io/github/languages/top/1nchaos/adata)[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/adata?color=d)](https://pypi.org/project/adata/)[![Licence](https://img.shields.io/hexpm/l/apa?color=d)](https://gitee.com/inchaos/adata/blob/main/LICENSE)![PyPI - Downloads](https://img.shields.io/pypi/dm/adata)![GitHub Repo stars](https://img.shields.io/github/stars/1nchaos/adata)![GitHub issues](https://img.shields.io/github/issues/1nchaos/adata)![GitHub contributors](https://img.shields.io/github/contributors/1nchaos/adata)![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/1nchaos/adata)
 
-
+---
 ## 0、介绍
 > 专注A股，专注量化，向阳而生；开放、纯净、持续、为Ai(爱)发电。
 >
@@ -101,6 +101,25 @@ print(res_df)
 
 - [数据字典](https://adata.1nchaos.com/dic/stockInfo.html) 
 
+####  4. 代理设置
+
+项目是基于公开接口，可能存在限制等，因此增加代理设置功能
+
+~~~
+import adata
+
+# 设置代理,代理是全局设置,代理失效后可从新设置。
+adata.proxy(is_proxy=True, ip='60.167.21.27:1133')
+res_df = adata.stock.info.all_code()
+print(res_df)
+~~~
+
+**注：**
+
+- v0.0.027b 及以上版本支持；
+- proxy_url: 获取代理Ip的链接；ip和proxy_url方式选择其一；
+- 每次请求获取一次，为节省ip资源建议使用自建的代理池。
+
 ## 二、数据列表
 
 整理了最新版本的数据列表和相关使用Api，详细内容和相关使用参数，请参考数据字典文档。

adata/__init__.py

@@ -10,6 +10,7 @@
 
 from .__version__ import __version__
 from .bond import bond
+from .common.utils.sunrequests import SunProxy
 from .fund import fund
 from .sentiment import sentiment
 from .stock import stock
@@ -19,6 +20,19 @@ def version():
     return __version__
 
 
+def proxy(is_proxy=False, ip: str = None, proxy_url: str = None):
+    """
+    设置请求代理
+    :param is_proxy: 是否启用代理，默认：否
+    :param ip: 代理ip地址；格式样例：192.123.123.4:4568
+    :param proxy_url: 能获取到代理的url，返回格式必须和ip一样
+    """
+    SunProxy.set('is_proxy', is_proxy)
+    SunProxy.set('ip', ip)
+    SunProxy.set('proxy_url', proxy_url)
+    return
+
+
 # set up logging
 logger = logging.getLogger("adata")
 

adata/__version__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (0, 0, 26)
+VERSION = (0, 0, 27)
 # PRERELEASE = None  # alpha, beta or rc
 PRERELEASE = 'beta'  # alpha, beta or rc
 REVISION = None

adata/common/exception/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+@desc: 
+@author: 1nchaos
+@time: 2023/7/8
+@log: change log
+"""

adata/common/exception/exception_msg.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+@desc: 异常信息
+@author: 1nchaos
+@time: 2023/7/8
+@log: change log
+"""
+
+THS_IP_LIMIT_RES = '<h1>Nginx forbidden.</h1>'
+"""同花顺Ip限制的返回结果"""
+THS_IP_LIMIT_MSG = "ths流量防控：当前ip被限制，请降低请求频率或更换ip或使用代理设置！！！"
+"""同花顺IP：403限制提醒"""

adata/common/headers/__init__.py

@@ -5,3 +5,84 @@
 @time: 2023/3/29
 @log: change log
 """
+import random
+
+headers_list = [
+    {
+        'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G955U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 10; SM-G981B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (iPad; CPU OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.109 Safari/537.36 CrKey/1.54.248666'
+    }, {
+        'user-agent': 'Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.188 Safari/537.36 CrKey/1.54.250320'
+    }, {
+        'user-agent': 'Mozilla/5.0 (BB10; Touch) AppleWebKit/537.10+ (KHTML, like Gecko) Version/10.0.9.2372 Mobile Safari/537.10+'
+    }, {
+        'user-agent': 'Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML like Gecko) Version/7.2.1.0 Safari/536.2+'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; U; Android 4.3; en-us; SM-N900T Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; U; Android 4.1; en-us; GT-N7100 Build/JRO03C) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; U; Android 4.0; en-us; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.13 Safari/535.19 Silk-Accelerated=true'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; LGMS323 Build/KOT49I.MS32310c) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 550) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/14.14263'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 10 Build/MOB31T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0.0; Nexus 5X Build/OPR4.170623.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 7.1.1; Nexus 6 Build/N6F26U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0.0; Nexus 6P Build/OPP3.170518.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 520)'
+    }, {
+        'user-agent': 'Mozilla/5.0 (MeeGo; NokiaN9) AppleWebKit/534.13 (KHTML, like Gecko) NokiaBrowser/8.5.0 Mobile Safari/534.13'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 11; Pixel 3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.181 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36'
+    }, {
+        'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1'
+    }, {
+        'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1'
+    }, {
+        'user-agent': 'Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1'
+    }
+]
+
+headers = random.choice(headers_list)

adata/common/utils/__init__.py

@@ -6,8 +6,6 @@
 @log: change log
 """
 from .snowflake import worker
-from .sunrequests import SunRequests
-
-requests = SunRequests()
+from .sunrequests import sun_requests as requests
 
 

adata/common/utils/sunrequests.py

@@ -1,35 +1,82 @@
 # -*- coding: utf-8 -*-
 """
-TODO：设置代理ip访问
+代理:https://jahttp.zhimaruanjian.com/getapi/
+
 @desc: adata 请求工具类
 @author: 1nchaos
 @time:2023/3/30
 @log: 封装请求次数
 """
 
+import threading
 import time
 
 import requests
 
 
+class SunProxy(object):
+    _data = {}
+    _instance_lock = threading.Lock()
+
+    def __init__(self):
+        pass
+
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(SunProxy, "_instance"):
+            with SunProxy._instance_lock:
+                if not hasattr(SunProxy, "_instance"):
+                    SunProxy._instance = object.__new__(cls)
+
+    @classmethod
+    def set(cls, key, value):
+        cls._data[key] = value
+
+    @classmethod
+    def get(cls, key):
+        return cls._data.get(key)
+
+    @classmethod
+    def delete(cls, key):
+        if key in cls._data:
+            del cls._data[key]
+
+
 class SunRequests(object):
-    def __init__(self) -> None:
+    def __init__(self, sun_proxy: SunProxy = None) -> None:
         super().__init__()
+        self.sun_proxy = sun_proxy
 
-    def request(self, method='get', url=None, times=3, sleep_time=1588, **kwargs):
+    def request(self, method='get', url=None, times=3, sleep_time=1588, proxies=None, **kwargs):
         """
         简单封装的请求，参考requests，增加循环次数和次数之间的等待时间
+        :param proxies: 代理配置
         :param method: 请求方法： get；post
         :param url: url
         :param times: 次数，int
         :param sleep_time: 循环的等待时间，毫秒
         :param kwargs: 其它 requests 参数，用法相同
         :return: res
         """
+        # 1. 获取设置代理
+        if proxies is None:
+            proxies = {}
+        is_proxy = SunProxy.get('is_proxy')
+        ip = SunProxy.get('ip')
+        proxy_url = SunProxy.get('proxy_url')
+        if not ip and is_proxy and proxy_url:
+            ip = requests.get(url=proxy_url).text.replace('\r\n', '')
+        if is_proxy and ip:
+            proxies = {'https': f"http://{ip}", 'http': f"http://{ip}"}
+        # 2. 请求数据结果
         res = None
         for i in range(times):
-            res = requests.request(method=method, url=url, **kwargs)
+            res = requests.request(method=method, url=url, proxies=proxies, **kwargs)
             if res.status_code in (200, 404):
                 return res
             time.sleep(sleep_time / 1000)
+            if i == times - 1:
+                return res
         return res
+
+
+sun_requests = SunRequests()