Merge branch 'master' into master

.travis.yml

@@ -10,28 +10,46 @@ matrix:
       env: TEST_COVERAGE=1
     - php: 7.0
     - php: hhvm
-  allow_failures:
-    - php: 7.0
 
-before_script:
+# Use new container infrastructure
+sudo: false
+
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/.composer/cache
+    - vendor
+
+install:
  # Setup the test server
  - phpenv local 5.5
  - composer install --dev --no-interaction
-
  - TESTPHPBIN=$(phpenv which php)
- - sudo PHPBIN=$TESTPHPBIN vendor/bin/start.sh
- - export REQUESTS_TEST_HOST_HTTP=localhost
  - phpenv local --unset
 
+ # Setup the proxy
+ - pip install --user mitmproxy
+
+before_script:
+ - PHPBIN=$TESTPHPBIN PORT=8080 vendor/bin/start.sh
+ - export REQUESTS_TEST_HOST_HTTP="localhost:8080"
+
  # Work out of the tests directory
  - cd tests
+ - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9002 utils/proxy/start.sh
+ - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9003 AUTH="test:pass" utils/proxy/start.sh
+ - export REQUESTS_HTTP_PROXY="localhost:9002"
+ - export REQUESTS_HTTP_PROXY_AUTH="localhost:9003"
+ - export REQUESTS_HTTP_PROXY_AUTH_USER="test"
+ - export REQUESTS_HTTP_PROXY_AUTH_PASS="pass"
 
 script:
  - phpunit --coverage-clover clover.xml
 
 after_script:
+ - utils/proxy/stop.sh
  - cd ..
  - phpenv local 5.5
- - sudo PATH=$PATH vendor/bin/stop.sh
+ - PATH=$PATH vendor/bin/stop.sh
  - test $TEST_COVERAGE && bash <(curl -s https://codecov.io/bash)
  - phpenv local --unset

library/Requests.php

@@ -62,6 +62,13 @@ class Requests {
 	 */
 	const PATCH = 'PATCH';
 
+	/**
+	 * Default size of buffer size to read streams
+	 *
+	 * @var integer
+	 */
+	const BUFFER_SIZE = 1160;
+
 	/**
 	 * Current version of Requests
 	 *
@@ -276,6 +283,8 @@ public static function patch($url, $headers, $data = array(), $options = array()
 	 *    (Requests_Auth|array|boolean, default: false)
 	 * - `proxy`: Proxy details to use for proxy by-passing and authentication
 	 *    (Requests_Proxy|array|boolean, default: false)
+	 * - `max_bytes`: Limit for the response body size.
+	 *    (integer|boolean, default: false)
 	 * - `idn`: Enable IDN parsing
 	 *    (boolean, default: true)
 	 * - `transport`: Custom transport. Either a class name, or a
@@ -459,6 +468,7 @@ protected static function get_default_options($multirequest = false) {
 			'auth' => false,
 			'proxy' => false,
 			'cookies' => false,
+			'max_bytes' => false,
 			'idn' => true,
 			'hooks' => null,
 			'transport' => null,

library/Requests/IRI.php

@@ -906,7 +906,8 @@ protected function set_authority($authority)
             }
             if (($port_start = strpos($remaining, ':', strpos($remaining, ']'))) !== false)
             {
-                if (($port = substr($remaining, $port_start + 1)) === false)
+                $port = substr($remaining, $port_start + 1);
+                if ($port === false || $port === '')
                 {
                     $port = null;
                 }

library/Requests/Transport/cURL.php

@@ -23,6 +23,13 @@ class Requests_Transport_cURL implements Requests_Transport {
 	 */
 	public $headers = '';
 
+	/**
+	 * Raw body data
+	 *
+	 * @var string
+	 */
+	public $response_data = '';
+
 	/**
 	 * Information on the current request
 	 *
@@ -44,6 +51,13 @@ class Requests_Transport_cURL implements Requests_Transport {
 	 */
 	protected $fp;
 
+	/**
+	 * Hook dispatcher instance
+	 *
+	 * @var Requests_Hooks
+	 */
+	protected $hooks;
+
 	/**
 	 * Have we finished the headers yet?
 	 *
@@ -58,6 +72,20 @@ class Requests_Transport_cURL implements Requests_Transport {
 	 */
 	protected $stream_handle;
 
+	/**
+	 * How many bytes are in the response body?
+	 *
+	 * @var int
+	 */
+	protected $response_bytes;
+
+	/**
+	 * What's the maximum number of bytes we should keep?
+	 *
+	 * @var int|bool Byte count, or false if no limit.
+	 */
+	protected $response_byte_limit;
+
 	/**
 	 * Constructor
 	 */
@@ -91,13 +119,21 @@ public function __construct() {
 	 * @return string Raw HTTP result
 	 */
 	public function request($url, $headers = array(), $data = array(), $options = array()) {
+		$this->hooks = $options['hooks'];
+
 		$this->setup_handle($url, $headers, $data, $options);
 
 		$options['hooks']->dispatch('curl.before_send', array(&$this->fp));
 
 		if ($options['filename'] !== false) {
 			$this->stream_handle = fopen($options['filename'], 'wb');
-			curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle);
+		}
+
+		$this->response_data = '';
+		$this->response_bytes = 0;
+		$this->response_byte_limit = false;
+		if ($options['max_bytes'] !== false) {
+			$this->response_byte_limit = $options['max_bytes'];
 		}
 
 		if (isset($options['verify'])) {
@@ -114,13 +150,19 @@ public function request($url, $headers = array(), $data = array(), $options = ar
 			curl_setopt($this->fp, CURLOPT_SSL_VERIFYHOST, 0);
 		}
 
-		$response = curl_exec($this->fp);
+		curl_exec($this->fp);
+		$response = $this->response_data;
 
 		$options['hooks']->dispatch('curl.after_send', array(&$fake_headers));
 
 		if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) {
+			// Reset encoding and try again
 			curl_setopt($this->fp, CURLOPT_ENCODING, 'none');
-			$response = curl_exec($this->fp);
+
+			$this->response_data = '';
+			$this->response_bytes = 0;
+			curl_exec($this->fp);
+			$response = $this->response_data;
 		}
 
 		$this->process_response($response, $options);
@@ -174,7 +216,7 @@ public function request_multiple($requests, $options) {
 			// Parse the finished requests before we start getting the new ones
 			foreach ($to_process as $key => $done) {
 				$options = $requests[$key]['options'];
-				$responses[$key] = $subrequests[$key]->process_response(curl_multi_getcontent($done['handle']), $options);
+				$responses[$key] = $subrequests[$key]->process_response($subrequests[$key]->response_data, $options);
 
 				$options['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$key], $requests[$key]));
 
@@ -210,9 +252,16 @@ public function &get_subrequest_handle($url, $headers, $data, $options) {
 
 		if ($options['filename'] !== false) {
 			$this->stream_handle = fopen($options['filename'], 'wb');
-			curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle);
 		}
 
+		$this->response_data = '';
+		$this->response_bytes = 0;
+		$this->response_byte_limit = false;
+		if ($options['max_bytes'] !== false) {
+			$this->response_byte_limit = $options['max_bytes'];
+		}
+		$this->hooks = $options['hooks'];
+
 		return $this->fp;
 	}
 
@@ -270,6 +319,8 @@ protected function setup_handle($url, $headers, $data, $options) {
 
 		if (true === $options['blocking']) {
 			curl_setopt($this->fp, CURLOPT_HEADERFUNCTION, array(&$this, 'stream_headers'));
+			curl_setopt($this->fp, CURLOPT_WRITEFUNCTION, array(&$this, 'stream_body'));
+			curl_setopt($this->fp, CURLOPT_BUFFERSIZE, Requests::BUFFER_SIZE);
 		}
 	}
 
@@ -319,6 +370,44 @@ public function stream_headers($handle, $headers) {
 		return strlen($headers);
 	}
 
+	/**
+	 * Collect data as it's received
+	 *
+	 * @since 1.6.1
+	 *
+	 * @param resource $handle cURL resource
+	 * @param string $data Body data
+	 * @return integer Length of provided data
+	 */
+	protected function stream_body($handle, $data) {
+		$this->hooks->dispatch('request.progress', array($data, $this->response_bytes, $this->response_byte_limit));
+		$data_length = strlen($data);
+
+		// Are we limiting the response size?
+		if ($this->response_byte_limit) {
+			if ($this->response_bytes === $this->response_byte_limit) {
+				// Already at maximum, move on
+				return $data_length;
+			}
+
+			if (($this->response_bytes + $data_length) > $this->response_byte_limit) {
+				// Limit the length
+				$limited_length = ($this->response_byte_limit - $this->response_bytes);
+				$data = substr($data, 0, $limited_length);
+			}
+		}
+
+		if ($this->stream_handle) {
+			fwrite($this->stream_handle, $data);
+		}
+		else {
+			$this->response_data .= $data;
+		}
+
+		$this->response_bytes += strlen($data);
+		return $data_length;
+	}
+
 	/**
 	 * Format a URL given GET data
 	 *

library/Requests/Transport/fsockopen.php

@@ -34,6 +34,13 @@ class Requests_Transport_fsockopen implements Requests_Transport {
 	 */
 	public $info;
 
+	/**
+	 * What's the maximum number of bytes we should keep?
+	 *
+	 * @var int|bool Byte count, or false if no limit.
+	 */
+	protected $max_bytes = false;
+
 	protected $connect_error = '';
 
 	/**
@@ -94,6 +101,8 @@ public function request($url, $headers = array(), $data = array(), $options = ar
 			$remote_socket = 'tcp://' . $host;
 		}
 
+		$this->max_bytes = $options['max_bytes'];
+
 		$proxy = isset( $options['proxy'] );
 		$proxy_auth = $proxy && isset( $options['proxy_username'] ) && isset( $options['proxy_password'] );
 
@@ -210,45 +219,63 @@ public function request($url, $headers = array(), $data = array(), $options = ar
 		$timeout_msec = $timeout_sec == $options['timeout'] ? 0 : self::SECOND_IN_MICROSECONDS * $options['timeout'] % self::SECOND_IN_MICROSECONDS;
 		stream_set_timeout($fp, $timeout_sec, $timeout_msec);
 
+		$response = $body = $headers = '';
 		$this->info = stream_get_meta_data($fp);
+		$size = 0;
+		$doingbody = false;
+		$download = false;
+		if ($options['filename']) {
+			$download = fopen($options['filename'], 'wb');
+		}
 
-		$this->headers = '';
-		$this->info = stream_get_meta_data($fp);
-		if (!$options['filename']) {
-			while (!feof($fp)) {
-				$this->info = stream_get_meta_data($fp);
-				if ($this->info['timed_out']) {
-					throw new Requests_Exception('fsocket timed out', 'timeout');
-				}
+		while (!feof($fp)) {
+			$this->info = stream_get_meta_data($fp);
+			if ($this->info['timed_out']) {
+				throw new Requests_Exception('fsocket timed out', 'timeout');
+			}
 
-				$this->headers .= fread($fp, 1160);
+			$block = fread($fp, Requests::BUFFER_SIZE);
+			if (!$doingbody) {
+				$response .= $block;
+				if (strpos($response, "\r\n\r\n")) {
+					list($headers, $block) = explode("\r\n\r\n", $response, 2);
+					$doingbody = true;
+				}
 			}
-		}
-		else {
-			$download = fopen($options['filename'], 'wb');
-			$doingbody = false;
-			$response = '';
-			while (!feof($fp)) {
-				$this->info = stream_get_meta_data($fp);
-				if ($this->info['timed_out']) {
-					throw new Requests_Exception('fsocket timed out', 'timeout');
+
+			// Are we in body mode now?
+			if ($doingbody) {
+				$options['hooks']->dispatch('request.progress', array($block, $size, $this->max_bytes));
+				$data_length = strlen($block);
+				if ($this->max_bytes) {
+					// Have we already hit a limit?
+					if ($size === $this->max_bytes) {
+						continue;
+					}
+					if (($size + $data_length) > $this->max_bytes) {
+						// Limit the length
+						$limited_length = ($this->max_bytes - $size);
+						$block = substr($block, 0, $limited_length);
+					}
 				}
 
-				$block = fread($fp, 1160);
-				if ($doingbody) {
+				$size += strlen($block);
+				if ($download) {
 					fwrite($download, $block);
 				}
 				else {
-					$response .= $block;
-					if (strpos($response, "\r\n\r\n")) {
-						list($this->headers, $block) = explode("\r\n\r\n", $response, 2);
-						$doingbody = true;
-						fwrite($download, $block);
-					}
+					$body .= $block;
 				}
 			}
+		}
+		$this->headers = $headers;
+
+		if ($download) {
 			fclose($download);
 		}
+		else {
+			$this->headers .= "\r\n\r\n" . $body;
+		}
 		fclose($fp);
 
 		$options['hooks']->dispatch('fsockopen.after_request', array(&$this->headers));