diff --git a/README.md b/README.md index 6615d41..87f45c1 100644 --- a/README.md +++ b/README.md @@ -46,20 +46,22 @@ import ( func main() { config := multiproxy.Config{ - ProxyURLs: []string{ - "http://proxy1.example.com:8080", - "http://proxy2.example.com:8080", + Proxies: []multiproxy.Proxy{ + { + URL: mustParseURL("http://proxy1.example.com:8080"), + Auth: &multiproxy.ProxyAuth{Username: "user1", Password: "pass1"}, + }, + { + URL: mustParseURL("http://proxy2.example.com:8080"), + Auth: &multiproxy.ProxyAuth{Username: "user2", Password: "pass2"}, + }, }, - ProxyAuth: map[string]multiproxy.ProxyAuth{ - "http://proxy1.example.com:8080": {Username: "user1", Password: "pass1"}, - "http://proxy2.example.com:8080": {Username: "user2", Password: "pass2"}, - }, - CookieTimeout: 10 * time.Minute, - CookieOptions: &cookiejar.Options{PublicSuffixList: publicsuffix.List}, - DialTimeout: 30 * time.Second, - RequestTimeout: 1 * time.Minute, - RetryAttempts: 3, - RetryDelay: 5 * time.Second, + CookieTimeout: 10 * time.Minute, + CookieOptions: &cookiejar.Options{PublicSuffixList: publicsuffix.List}, + DialTimeout: 30 * time.Second, + RequestTimeout: 1 * time.Minute, + RetryAttempts: 3, + RetryDelay: 5 * time.Second, ProxyRotateCount: 10, } @@ -82,10 +84,11 @@ func main() { The `Config` struct allows you to customize the behavior of the MultiProxy Client: -- `ProxyURLs`: List of proxy URLs to use -- `ProxyAuth`: Map of proxy URLs to their respective authentication credentials +- `Proxies`: List of Proxy structs, each containing: + - `URL`: The URL of the proxy + - `Auth`: Pointer to ProxyAuth struct with Username and Password + - `UserAgent`: User-Agent string for this specific proxy - `ProxyRotateCount`: Number of requests after which to rotate to the next proxy -- `ProxyUserAgents`: Map of proxy URLs to their respective User-Agent strings - `BackoffTime`: Time to wait before retrying a failed proxy - `DialTimeout`: Timeout for establishing a connection to a proxy diff --git a/connect_test.go b/connect_test.go index 93e95f1..42b3629 100644 --- a/connect_test.go +++ b/connect_test.go @@ -149,11 +149,11 @@ func TestConnectProxyWithHTTPS(t *testing.T) { } config := Config{ - ProxyURLs: []string{ - proxyURL.String(), - }, - ProxyAuth: map[string]ProxyAuth{ - proxyURL.Host: {Username: "user", Password: "pass"}, + Proxies: []Proxy{ + { + URL: proxyURL, + Auth: &ProxyAuth{Username: "user", Password: "pass"}, + }, }, DialTimeout: 5 * time.Second, InsecureSkipVerify: true, diff --git a/errors_test.go b/errors_test.go index e016770..0f6a5d2 100644 --- a/errors_test.go +++ b/errors_test.go @@ -1,6 +1,7 @@ package multiproxy import ( + "net/url" "testing" "time" @@ -14,9 +15,9 @@ func TestAllProxiesUnavailable(t *testing.T) { // Use non-routable IPs to simulate unavailable proxies config := Config{ - ProxyURLs: []string{ - "socks5://10.255.255.1:1080", - "socks5://10.255.255.2:1080", + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: "10.255.255.1:1080"}}, + {URL: &url.URL{Scheme: "socks5", Host: "10.255.255.2:1080"}}, }, DialTimeout: 1 * time.Second, DefaultUserAgent: "DefaultUserAgent/1.0", @@ -41,7 +42,9 @@ func TestAllProxiesUnavailable(t *testing.T) { func TestClientGetError(t *testing.T) { config := Config{ - ProxyURLs: []string{"http://10.255.255.1:8080"}, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "http", Host: "10.255.255.1:8080"}}, + }, DialTimeout: 5 * time.Second, } @@ -54,7 +57,9 @@ func TestClientGetError(t *testing.T) { func TestClientPostError(t *testing.T) { config := Config{ - ProxyURLs: []string{"http://10.255.255.1:8080"}, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "http", Host: "10"}}, + }, DialTimeout: 5 * time.Second, } @@ -67,7 +72,9 @@ func TestClientPostError(t *testing.T) { func TestClientHeadError(t *testing.T) { config := Config{ - ProxyURLs: []string{"http://10.255.255.1:8080"}, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "http", Host: "10"}}, + }, DialTimeout: 5 * time.Second, } @@ -78,26 +85,14 @@ func TestClientHeadError(t *testing.T) { assert.Contains(t, err.Error(), "invalid control character") } -func TestClientNewError(t *testing.T) { - config := Config{ - ProxyURLs: []string{"\000"}, - DialTimeout: 5 * time.Second, - } - - _, err := NewClient(config) - require.Error(t, err) - assert.Contains(t, err.Error(), "invalid control character") -} - func TestClientWithNoProxyURLs(t *testing.T) { config := Config{ - ProxyURLs: []string{}, - DialTimeout: 5 * time.Second, + Proxies: []Proxy{}, } _, err := NewClient(config) require.Error(t, err) - assert.Contains(t, err.Error(), "at least one proxy URL is required") + assert.Contains(t, err.Error(), "at least one proxy is required") } func TestDialTimeout(t *testing.T) { @@ -105,8 +100,8 @@ func TestDialTimeout(t *testing.T) { nonRoutableIP := "10.255.255.1:8080" config := Config{ - ProxyURLs: []string{ - "http://" + nonRoutableIP, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "http", Host: nonRoutableIP}}, }, DialTimeout: 1 * time.Second, RequestTimeout: 1 * time.Second, diff --git a/multiproxy.go b/multiproxy.go index 7ade0c5..4cde349 100644 --- a/multiproxy.go +++ b/multiproxy.go @@ -18,6 +18,12 @@ import ( "golang.org/x/sync/singleflight" ) +type Proxy struct { + URL *url.URL + Auth *ProxyAuth + UserAgent string +} + type ProxyAuth struct { Username string Password string @@ -25,10 +31,8 @@ type ProxyAuth struct { type Config struct { // Proxy configuration - ProxyURLs []string - ProxyAuth map[string]ProxyAuth + Proxies []Proxy ProxyRotateCount int - ProxyUserAgents map[string]string // Timeouts and delays BackoffTime time.Duration @@ -75,24 +79,22 @@ type Client struct { } func NewClient(config Config) (*Client, error) { - if len(config.ProxyURLs) == 0 { - return nil, errors.New("at least one proxy URL is required") + if len(config.Proxies) == 0 { + return nil, errors.New("at least one proxy is required") } c := &Client{ config: config, - servers: make([]*url.URL, len(config.ProxyURLs)), - states: make([]proxyState, len(config.ProxyURLs)), + servers: make([]*url.URL, len(config.Proxies)), + states: make([]proxyState, len(config.Proxies)), } - for i, proxyURL := range config.ProxyURLs { - serverURL, err := url.Parse(proxyURL) - if err != nil { - return nil, fmt.Errorf("invalid proxy URL %s: %v", proxyURL, err) - } - c.servers[i] = serverURL + for i, elt := range config.Proxies { + c.servers[i] = elt.URL - auth, hasAuth := c.config.ProxyAuth[serverURL.Host] + hasAuth := elt.Auth != nil && + (elt.Auth.Username != "" || + elt.Auth.Password != "") var transport http.RoundTripper @@ -103,17 +105,17 @@ func NewClient(config Config) (*Client, error) { dialer.Timeout = c.config.DialTimeout } - if serverURL.Scheme == "socks5" { - auth := &proxy.Auth{ - User: auth.Username, - Password: auth.Password, - } - if !hasAuth { - auth = nil + if elt.URL.Scheme == "socks5" { + var auth *proxy.Auth + if hasAuth { + auth = &proxy.Auth{ + User: elt.Auth.Username, + Password: elt.Auth.Password, + } } - socksDialer, err := proxy.SOCKS5("tcp", serverURL.Host, auth, dialer) + socksDialer, err := proxy.SOCKS5("tcp", elt.URL.Host, auth, dialer) if err != nil { - return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %v", serverURL.Host, err) + return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %v", elt.URL.Host, err) } transport = &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { @@ -125,7 +127,7 @@ func NewClient(config Config) (*Client, error) { } } else { proxyURL := func(_ *http.Request) (*url.URL, error) { - return serverURL, nil + return elt.URL, nil } transport = &http.Transport{ Proxy: proxyURL, @@ -136,12 +138,12 @@ func NewClient(config Config) (*Client, error) { } if hasAuth { transport.(*http.Transport).ProxyConnectHeader = http.Header{ - "Proxy-Authorization": {basicAuth(auth.Username, auth.Password)}, + "Proxy-Authorization": {basicAuth(elt.Auth.Username, elt.Auth.Password)}, } // Also set it for non-CONNECT requests transport.(*http.Transport).Proxy = func(req *http.Request) (*url.URL, error) { - req.Header.Set("Proxy-Authorization", basicAuth(auth.Username, auth.Password)) - return serverURL, nil + req.Header.Set("Proxy-Authorization", basicAuth(elt.Auth.Username, elt.Auth.Password)) + return elt.URL, nil } } } @@ -189,15 +191,15 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { } // Apply rate limiting - if limit, ok := c.config.RateLimits[c.servers[idx].Host]; ok { + if limit, ok := c.config.RateLimits[c.config.Proxies[idx].URL.Host]; ok { if now.Sub(state.lastRequestAt) < limit { time.Sleep(limit - now.Sub(state.lastRequestAt)) } } // Set proxy-specific User-Agent if configured - if userAgent, ok := c.config.ProxyUserAgents[c.servers[idx].Host]; ok { - req.Header.Set("User-Agent", userAgent) + if c.config.Proxies[idx].UserAgent != "" { + req.Header.Set("User-Agent", c.config.Proxies[idx].UserAgent) } // Set request timeout diff --git a/multiproxy_test.go b/multiproxy_test.go index 3d2b36b..49a046c 100644 --- a/multiproxy_test.go +++ b/multiproxy_test.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/http/httptest" + "net/url" "strings" "testing" "time" @@ -75,9 +76,9 @@ func TestRoundRobinSelection(t *testing.T) { defer cleanup2() config := Config{ - ProxyURLs: []string{ - "socks5://" + proxy1, - "socks5://" + proxy2, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy1}}, + {URL: &url.URL{Scheme: "socks5", Host: proxy2}}, }, DialTimeout: 5 * time.Second, ProxyRotateCount: 1, @@ -109,9 +110,9 @@ func TestBackoff(t *testing.T) { invalidProxy := "127.0.0.1:1" // Invalid proxy to trigger backoff config := Config{ - ProxyURLs: []string{ - "socks5://" + proxy1, - "socks5://" + invalidProxy, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy1}}, + {URL: &url.URL{Scheme: "socks5", Host: invalidProxy}}, }, DialTimeout: 2 * time.Second, BackoffTime: 5 * time.Second, @@ -149,13 +150,9 @@ func TestAuthentication(t *testing.T) { defer cleanup2() config := Config{ - ProxyURLs: []string{ - "socks5://" + proxy1, - "socks5://" + proxy2, - }, - ProxyAuth: map[string]ProxyAuth{ - proxy1: {Username: "user1", Password: "pass1"}, - proxy2: {Username: "user2", Password: "pass2"}, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy1}, Auth: &ProxyAuth{Username: "user1", Password: "pass1"}}, + {URL: &url.URL{Scheme: "socks5", Host: proxy2}, Auth: &ProxyAuth{Username: "user2", Password: "pass2"}}, }, DialTimeout: 5 * time.Second, ProxyRotateCount: 1, @@ -187,9 +184,9 @@ func TestRetryMechanism(t *testing.T) { invalidProxy := "127.0.0.1:1" // Invalid proxy to trigger retry config := Config{ - ProxyURLs: []string{ - "socks5://" + invalidProxy, - "socks5://" + proxy1, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: invalidProxy}}, + {URL: &url.URL{Scheme: "socks5", Host: proxy1}}, }, RetryAttempts: 2, RetryDelay: time.Second, @@ -226,11 +223,11 @@ func TestUserAgentOverride(t *testing.T) { defer cleanup2() config := Config{ - ProxyURLs: []string{"socks5://" + proxy1, "socks5://" + proxy2}, - DefaultUserAgent: "DefaultUserAgent/1.0", - ProxyUserAgents: map[string]string{ - proxy1: "CustomUserAgent/1.0", + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy1}, UserAgent: "CustomUserAgent/1.0"}, + {URL: &url.URL{Scheme: "socks5", Host: proxy2}}, }, + DefaultUserAgent: "DefaultUserAgent/1.0", DialTimeout: 5 * time.Second, ProxyRotateCount: 1, // Ensure we switch proxies after each request } @@ -262,10 +259,13 @@ func TestRateLimiting(t *testing.T) { proxy1, cleanup1 := setupSocks5Server(t, "", "") defer cleanup1() + proxyURL := &url.URL{Scheme: "socks5", Host: proxy1} config := Config{ - ProxyURLs: []string{"socks5://" + proxy1}, + Proxies: []Proxy{ + {URL: proxyURL}, + }, RateLimits: map[string]time.Duration{ - proxy1: 1 * time.Second, + proxyURL.Host: 1 * time.Second, }, DialTimeout: 5 * time.Second, } @@ -295,9 +295,9 @@ func TestProxyRotation(t *testing.T) { defer cleanup2() config := Config{ - ProxyURLs: []string{ - "socks5://" + proxy1, - "socks5://" + proxy2, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy1}}, + {URL: &url.URL{Scheme: "socks5", Host: proxy2}}, }, ProxyRotateCount: 2, DialTimeout: 5 * time.Second, @@ -337,8 +337,8 @@ func TestCookieTimeout(t *testing.T) { defer cleanup() config := Config{ - ProxyURLs: []string{ - "socks5://" + proxy, + Proxies: []Proxy{ + {URL: &url.URL{Scheme: "socks5", Host: proxy}}, }, CookieTimeout: 2 * time.Second, DialTimeout: 5 * time.Second, @@ -385,7 +385,7 @@ func TestClientHead(t *testing.T) { defer cleanup() config := Config{ - ProxyURLs: []string{"socks5://" + proxy}, + Proxies: []Proxy{{URL: &url.URL{Scheme: "socks5", Host: proxy}}}, DialTimeout: 5 * time.Second, } @@ -425,7 +425,7 @@ func TestClientPost(t *testing.T) { defer cleanup() config := Config{ - ProxyURLs: []string{"socks5://" + proxy}, + Proxies: []Proxy{{URL: &url.URL{Scheme: "socks5", Host: proxy}}}, DialTimeout: 5 * time.Second, }