diff --git a/.travis.yml b/.travis.yml
index 82fa109a6..0a612a2d6 100755
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,28 +10,46 @@ matrix:
env: TEST_COVERAGE=1
- php: 7.0
- php: hhvm
- allow_failures:
- - php: 7.0
-before_script:
+# Use new container infrastructure
+sudo: false
+
+cache:
+ directories:
+ - $HOME/.cache/pip
+ - $HOME/.composer/cache
+ - vendor
+
+install:
# Setup the test server
- phpenv local 5.5
- composer install --dev --no-interaction
-
- TESTPHPBIN=$(phpenv which php)
- - sudo PHPBIN=$TESTPHPBIN vendor/bin/start.sh
- - export REQUESTS_TEST_HOST_HTTP=localhost
- phpenv local --unset
+ # Setup the proxy
+ - pip install --user mitmproxy
+
+before_script:
+ - PHPBIN=$TESTPHPBIN PORT=8080 vendor/bin/start.sh
+ - export REQUESTS_TEST_HOST_HTTP="localhost:8080"
+
# Work out of the tests directory
- cd tests
+ - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9002 utils/proxy/start.sh
+ - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9003 AUTH="test:pass" utils/proxy/start.sh
+ - export REQUESTS_HTTP_PROXY="localhost:9002"
+ - export REQUESTS_HTTP_PROXY_AUTH="localhost:9003"
+ - export REQUESTS_HTTP_PROXY_AUTH_USER="test"
+ - export REQUESTS_HTTP_PROXY_AUTH_PASS="pass"
script:
- phpunit --coverage-clover clover.xml
after_script:
+ - utils/proxy/stop.sh
- cd ..
- phpenv local 5.5
- - sudo PATH=$PATH vendor/bin/stop.sh
+ - PATH=$PATH vendor/bin/stop.sh
- test $TEST_COVERAGE && bash <(curl -s https://codecov.io/bash)
- phpenv local --unset
diff --git a/library/Requests.php b/library/Requests.php
index b9f3818bb..065227e0d 100755
--- a/library/Requests.php
+++ b/library/Requests.php
@@ -62,6 +62,13 @@ class Requests {
*/
const PATCH = 'PATCH';
+ /**
+ * Default size of buffer size to read streams
+ *
+ * @var integer
+ */
+ const BUFFER_SIZE = 1160;
+
/**
* Current version of Requests
*
@@ -276,6 +283,8 @@ public static function patch($url, $headers, $data = array(), $options = array()
* (Requests_Auth|array|boolean, default: false)
* - `proxy`: Proxy details to use for proxy by-passing and authentication
* (Requests_Proxy|array|boolean, default: false)
+ * - `max_bytes`: Limit for the response body size.
+ * (integer|boolean, default: false)
* - `idn`: Enable IDN parsing
* (boolean, default: true)
* - `transport`: Custom transport. Either a class name, or a
@@ -459,6 +468,7 @@ protected static function get_default_options($multirequest = false) {
'auth' => false,
'proxy' => false,
'cookies' => false,
+ 'max_bytes' => false,
'idn' => true,
'hooks' => null,
'transport' => null,
diff --git a/library/Requests/IRI.php b/library/Requests/IRI.php
index 26f215b6a..aa4dfbe71 100755
--- a/library/Requests/IRI.php
+++ b/library/Requests/IRI.php
@@ -906,7 +906,8 @@ protected function set_authority($authority)
}
if (($port_start = strpos($remaining, ':', strpos($remaining, ']'))) !== false)
{
- if (($port = substr($remaining, $port_start + 1)) === false)
+ $port = substr($remaining, $port_start + 1);
+ if ($port === false || $port === '')
{
$port = null;
}
diff --git a/library/Requests/Transport/cURL.php b/library/Requests/Transport/cURL.php
index fc00d5c99..a147aab24 100755
--- a/library/Requests/Transport/cURL.php
+++ b/library/Requests/Transport/cURL.php
@@ -23,6 +23,13 @@ class Requests_Transport_cURL implements Requests_Transport {
*/
public $headers = '';
+ /**
+ * Raw body data
+ *
+ * @var string
+ */
+ public $response_data = '';
+
/**
* Information on the current request
*
@@ -44,6 +51,13 @@ class Requests_Transport_cURL implements Requests_Transport {
*/
protected $fp;
+ /**
+ * Hook dispatcher instance
+ *
+ * @var Requests_Hooks
+ */
+ protected $hooks;
+
/**
* Have we finished the headers yet?
*
@@ -58,6 +72,20 @@ class Requests_Transport_cURL implements Requests_Transport {
*/
protected $stream_handle;
+ /**
+ * How many bytes are in the response body?
+ *
+ * @var int
+ */
+ protected $response_bytes;
+
+ /**
+ * What's the maximum number of bytes we should keep?
+ *
+ * @var int|bool Byte count, or false if no limit.
+ */
+ protected $response_byte_limit;
+
/**
* Constructor
*/
@@ -91,13 +119,21 @@ public function __construct() {
* @return string Raw HTTP result
*/
public function request($url, $headers = array(), $data = array(), $options = array()) {
+ $this->hooks = $options['hooks'];
+
$this->setup_handle($url, $headers, $data, $options);
$options['hooks']->dispatch('curl.before_send', array(&$this->fp));
if ($options['filename'] !== false) {
$this->stream_handle = fopen($options['filename'], 'wb');
- curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle);
+ }
+
+ $this->response_data = '';
+ $this->response_bytes = 0;
+ $this->response_byte_limit = false;
+ if ($options['max_bytes'] !== false) {
+ $this->response_byte_limit = $options['max_bytes'];
}
if (isset($options['verify'])) {
@@ -114,13 +150,19 @@ public function request($url, $headers = array(), $data = array(), $options = ar
curl_setopt($this->fp, CURLOPT_SSL_VERIFYHOST, 0);
}
- $response = curl_exec($this->fp);
+ curl_exec($this->fp);
+ $response = $this->response_data;
$options['hooks']->dispatch('curl.after_send', array(&$fake_headers));
if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) {
+ // Reset encoding and try again
curl_setopt($this->fp, CURLOPT_ENCODING, 'none');
- $response = curl_exec($this->fp);
+
+ $this->response_data = '';
+ $this->response_bytes = 0;
+ curl_exec($this->fp);
+ $response = $this->response_data;
}
$this->process_response($response, $options);
@@ -174,7 +216,7 @@ public function request_multiple($requests, $options) {
// Parse the finished requests before we start getting the new ones
foreach ($to_process as $key => $done) {
$options = $requests[$key]['options'];
- $responses[$key] = $subrequests[$key]->process_response(curl_multi_getcontent($done['handle']), $options);
+ $responses[$key] = $subrequests[$key]->process_response($subrequests[$key]->response_data, $options);
$options['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$key], $requests[$key]));
@@ -210,9 +252,16 @@ public function &get_subrequest_handle($url, $headers, $data, $options) {
if ($options['filename'] !== false) {
$this->stream_handle = fopen($options['filename'], 'wb');
- curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle);
}
+ $this->response_data = '';
+ $this->response_bytes = 0;
+ $this->response_byte_limit = false;
+ if ($options['max_bytes'] !== false) {
+ $this->response_byte_limit = $options['max_bytes'];
+ }
+ $this->hooks = $options['hooks'];
+
return $this->fp;
}
@@ -270,6 +319,8 @@ protected function setup_handle($url, $headers, $data, $options) {
if (true === $options['blocking']) {
curl_setopt($this->fp, CURLOPT_HEADERFUNCTION, array(&$this, 'stream_headers'));
+ curl_setopt($this->fp, CURLOPT_WRITEFUNCTION, array(&$this, 'stream_body'));
+ curl_setopt($this->fp, CURLOPT_BUFFERSIZE, Requests::BUFFER_SIZE);
}
}
@@ -319,6 +370,44 @@ public function stream_headers($handle, $headers) {
return strlen($headers);
}
+ /**
+ * Collect data as it's received
+ *
+ * @since 1.6.1
+ *
+ * @param resource $handle cURL resource
+ * @param string $data Body data
+ * @return integer Length of provided data
+ */
+ protected function stream_body($handle, $data) {
+ $this->hooks->dispatch('request.progress', array($data, $this->response_bytes, $this->response_byte_limit));
+ $data_length = strlen($data);
+
+ // Are we limiting the response size?
+ if ($this->response_byte_limit) {
+ if ($this->response_bytes === $this->response_byte_limit) {
+ // Already at maximum, move on
+ return $data_length;
+ }
+
+ if (($this->response_bytes + $data_length) > $this->response_byte_limit) {
+ // Limit the length
+ $limited_length = ($this->response_byte_limit - $this->response_bytes);
+ $data = substr($data, 0, $limited_length);
+ }
+ }
+
+ if ($this->stream_handle) {
+ fwrite($this->stream_handle, $data);
+ }
+ else {
+ $this->response_data .= $data;
+ }
+
+ $this->response_bytes += strlen($data);
+ return $data_length;
+ }
+
/**
* Format a URL given GET data
*
diff --git a/library/Requests/Transport/fsockopen.php b/library/Requests/Transport/fsockopen.php
index 377c1ab32..6d4c41377 100755
--- a/library/Requests/Transport/fsockopen.php
+++ b/library/Requests/Transport/fsockopen.php
@@ -34,6 +34,13 @@ class Requests_Transport_fsockopen implements Requests_Transport {
*/
public $info;
+ /**
+ * What's the maximum number of bytes we should keep?
+ *
+ * @var int|bool Byte count, or false if no limit.
+ */
+ protected $max_bytes = false;
+
protected $connect_error = '';
/**
@@ -94,6 +101,8 @@ public function request($url, $headers = array(), $data = array(), $options = ar
$remote_socket = 'tcp://' . $host;
}
+ $this->max_bytes = $options['max_bytes'];
+
$proxy = isset( $options['proxy'] );
$proxy_auth = $proxy && isset( $options['proxy_username'] ) && isset( $options['proxy_password'] );
@@ -210,45 +219,63 @@ public function request($url, $headers = array(), $data = array(), $options = ar
$timeout_msec = $timeout_sec == $options['timeout'] ? 0 : self::SECOND_IN_MICROSECONDS * $options['timeout'] % self::SECOND_IN_MICROSECONDS;
stream_set_timeout($fp, $timeout_sec, $timeout_msec);
+ $response = $body = $headers = '';
$this->info = stream_get_meta_data($fp);
+ $size = 0;
+ $doingbody = false;
+ $download = false;
+ if ($options['filename']) {
+ $download = fopen($options['filename'], 'wb');
+ }
- $this->headers = '';
- $this->info = stream_get_meta_data($fp);
- if (!$options['filename']) {
- while (!feof($fp)) {
- $this->info = stream_get_meta_data($fp);
- if ($this->info['timed_out']) {
- throw new Requests_Exception('fsocket timed out', 'timeout');
- }
+ while (!feof($fp)) {
+ $this->info = stream_get_meta_data($fp);
+ if ($this->info['timed_out']) {
+ throw new Requests_Exception('fsocket timed out', 'timeout');
+ }
- $this->headers .= fread($fp, 1160);
+ $block = fread($fp, Requests::BUFFER_SIZE);
+ if (!$doingbody) {
+ $response .= $block;
+ if (strpos($response, "\r\n\r\n")) {
+ list($headers, $block) = explode("\r\n\r\n", $response, 2);
+ $doingbody = true;
+ }
}
- }
- else {
- $download = fopen($options['filename'], 'wb');
- $doingbody = false;
- $response = '';
- while (!feof($fp)) {
- $this->info = stream_get_meta_data($fp);
- if ($this->info['timed_out']) {
- throw new Requests_Exception('fsocket timed out', 'timeout');
+
+ // Are we in body mode now?
+ if ($doingbody) {
+ $options['hooks']->dispatch('request.progress', array($block, $size, $this->max_bytes));
+ $data_length = strlen($block);
+ if ($this->max_bytes) {
+ // Have we already hit a limit?
+ if ($size === $this->max_bytes) {
+ continue;
+ }
+ if (($size + $data_length) > $this->max_bytes) {
+ // Limit the length
+ $limited_length = ($this->max_bytes - $size);
+ $block = substr($block, 0, $limited_length);
+ }
}
- $block = fread($fp, 1160);
- if ($doingbody) {
+ $size += strlen($block);
+ if ($download) {
fwrite($download, $block);
}
else {
- $response .= $block;
- if (strpos($response, "\r\n\r\n")) {
- list($this->headers, $block) = explode("\r\n\r\n", $response, 2);
- $doingbody = true;
- fwrite($download, $block);
- }
+ $body .= $block;
}
}
+ }
+ $this->headers = $headers;
+
+ if ($download) {
fclose($download);
}
+ else {
+ $this->headers .= "\r\n\r\n" . $body;
+ }
fclose($fp);
$options['hooks']->dispatch('fsockopen.after_request', array(&$this->headers));
diff --git a/tests/Proxy/HTTP.php b/tests/Proxy/HTTP.php
new file mode 100644
index 000000000..351570dfe
--- /dev/null
+++ b/tests/Proxy/HTTP.php
@@ -0,0 +1,131 @@
+markTestSkipped('Proxy not available');
+ }
+ }
+
+ public function transportProvider() {
+ return array(
+ array('Requests_Transport_cURL'),
+ array('Requests_Transport_fsockopen'),
+ );
+ }
+
+ /**
+ * @dataProvider transportProvider
+ */
+ public function testConnectWithString($transport) {
+ $this->checkProxyAvailable();
+
+ $options = array(
+ 'proxy' => REQUESTS_HTTP_PROXY,
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ $this->assertEquals('http', $response->headers['x-requests-proxied']);
+
+ $data = json_decode($response->body, true);
+ $this->assertEquals('http', $data['headers']['x-requests-proxy']);
+ }
+
+ /**
+ * @dataProvider transportProvider
+ */
+ public function testConnectWithArray($transport) {
+ $this->checkProxyAvailable();
+
+ $options = array(
+ 'proxy' => array(REQUESTS_HTTP_PROXY),
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ $this->assertEquals('http', $response->headers['x-requests-proxied']);
+
+ $data = json_decode($response->body, true);
+ $this->assertEquals('http', $data['headers']['x-requests-proxy']);
+ }
+
+ /**
+ * @dataProvider transportProvider
+ * @expectedException Requests_Exception
+ */
+ public function testConnectInvalidParameters($transport) {
+ $this->checkProxyAvailable();
+
+ $options = array(
+ 'proxy' => array(REQUESTS_HTTP_PROXY, 'testuser', 'password', 'something'),
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ }
+
+ /**
+ * @dataProvider transportProvider
+ */
+ public function testConnectWithInstance($transport) {
+ $this->checkProxyAvailable();
+
+ $options = array(
+ 'proxy' => REQUESTS_HTTP_PROXY,
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ $this->assertEquals('http', $response->headers['x-requests-proxied']);
+
+ $data = json_decode($response->body, true);
+ $this->assertEquals('http', $data['headers']['x-requests-proxy']);
+ }
+
+ /**
+ * @dataProvider transportProvider
+ */
+ public function testConnectWithAuth($transport) {
+ $this->checkProxyAvailable('auth');
+
+ $options = array(
+ 'proxy' => array(
+ REQUESTS_HTTP_PROXY_AUTH,
+ REQUESTS_HTTP_PROXY_AUTH_USER,
+ REQUESTS_HTTP_PROXY_AUTH_PASS
+ ),
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ $this->assertEquals(200, $response->status_code);
+ $this->assertEquals('http', $response->headers['x-requests-proxied']);
+
+ $data = json_decode($response->body, true);
+ $this->assertEquals('http', $data['headers']['x-requests-proxy']);
+ }
+
+ /**
+ * @dataProvider transportProvider
+ */
+ public function testConnectWithInvalidAuth($transport) {
+ $this->checkProxyAvailable('auth');
+
+ $options = array(
+ 'proxy' => array(
+ REQUESTS_HTTP_PROXY_AUTH,
+ REQUESTS_HTTP_PROXY_AUTH_USER . '!',
+ REQUESTS_HTTP_PROXY_AUTH_PASS . '!'
+ ),
+ 'transport' => $transport,
+ );
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ $this->assertEquals(407, $response->status_code);
+ }
+}
\ No newline at end of file
diff --git a/tests/Transport/Base.php b/tests/Transport/Base.php
index 352728612..b591df07e 100755
--- a/tests/Transport/Base.php
+++ b/tests/Transport/Base.php
@@ -25,6 +25,27 @@ protected function getOptions($other = array()) {
return $options;
}
+ public function testResponseByteLimit() {
+ $limit = 104;
+ $options = array(
+ 'max_bytes' => $limit,
+ );
+ $response = Requests::get(httpbin('/bytes/325'), array(), $this->getOptions($options));
+ $this->assertEquals($limit, strlen($response->body));
+ }
+
+ public function testResponseByteLimitWithFile() {
+ $limit = 300;
+ $options = array(
+ 'max_bytes' => $limit,
+ 'filename' => tempnam(sys_get_temp_dir(), 'RLT') // RequestsLibraryTest
+ );
+ $response = Requests::get(httpbin('/bytes/482'), array(), $this->getOptions($options));
+ $this->assertEmpty($response->body);
+ $this->assertEquals($limit, filesize($options['filename']));
+ unlink($options['filename']);
+ }
+
public function testSimpleGET() {
$request = Requests::get(httpbin('/get'), array(), $this->getOptions());
$this->assertEquals(200, $request->status_code);
@@ -706,4 +727,17 @@ public function testAlternatePort() {
$this->assertEquals(1, $num, 'Response should contain the port number');
$this->assertEquals(8080, $matches[1]);
}
+
+ public function testProgressCallback() {
+ $mock = $this->getMockBuilder('stdClass')->setMethods(array('progress'))->getMock();
+ $mock->expects($this->atLeastOnce())->method('progress');
+ $hooks = new Requests_Hooks();
+ $hooks->register('request.progress', array($mock, 'progress'));
+ $options = array(
+ 'hooks' => $hooks,
+ );
+ $options = $this->getOptions($options);
+
+ $response = Requests::get(httpbin('/get'), array(), $options);
+ }
}
diff --git a/tests/bootstrap.php b/tests/bootstrap.php
index 41b94e98d..8cc8c0e19 100755
--- a/tests/bootstrap.php
+++ b/tests/bootstrap.php
@@ -2,13 +2,24 @@
date_default_timezone_set('UTC');
-$host = getenv('REQUESTS_TEST_HOST');
-if (empty($host)) {
- $host = 'httpbin.org';
+function define_from_env($name, $default = false) {
+ $env = getenv($name);
+ if ($env) {
+ define($name, $env);
+ }
+ else {
+ define($name, $default);
+ }
}
-define('REQUESTS_TEST_HOST', getenv('REQUESTS_TEST_HOST') ? getenv('REQUESTS_TEST_HOST') : 'httpbin.org');
-define('REQUESTS_TEST_HOST_HTTP', getenv('REQUESTS_TEST_HOST_HTTP') ? getenv('REQUESTS_TEST_HOST_HTTP') : REQUESTS_TEST_HOST);
-define('REQUESTS_TEST_HOST_HTTPS', getenv('REQUESTS_TEST_HOST_HTTPS') ? getenv('REQUESTS_TEST_HOST_HTTPS'): REQUESTS_TEST_HOST);
+
+define_from_env('REQUESTS_TEST_HOST', 'requests-php-tests.herokuapp.com');
+define_from_env('REQUESTS_TEST_HOST_HTTP', REQUESTS_TEST_HOST);
+define_from_env('REQUESTS_TEST_HOST_HTTPS', REQUESTS_TEST_HOST);
+
+define_from_env('REQUESTS_HTTP_PROXY');
+define_from_env('REQUESTS_HTTP_PROXY_AUTH');
+define_from_env('REQUESTS_HTTP_PROXY_AUTH_USER');
+define_from_env('REQUESTS_HTTP_PROXY_AUTH_PASS');
include(dirname(dirname(__FILE__)) . '/library/Requests.php');
Requests::register_autoloader();
diff --git a/tests/phpunit.xml.dist b/tests/phpunit.xml.dist
index 847b9b661..e46ea1719 100755
--- a/tests/phpunit.xml.dist
+++ b/tests/phpunit.xml.dist
@@ -7,6 +7,9 @@
Transport
+
+ Proxy
+
ChunkedEncoding.php
Cookies.php
diff --git a/tests/utils/proxy/proxy.py b/tests/utils/proxy/proxy.py
new file mode 100755
index 000000000..502c1d91f
--- /dev/null
+++ b/tests/utils/proxy/proxy.py
@@ -0,0 +1,5 @@
+def request(context, flow):
+ flow.request.headers["x-requests-proxy"] = ["http"]
+
+def response(context, flow):
+ flow.response.headers["x-requests-proxied"] = ["http"]
\ No newline at end of file
diff --git a/tests/utils/proxy/start.sh b/tests/utils/proxy/start.sh
new file mode 100755
index 000000000..4e7edeca6
--- /dev/null
+++ b/tests/utils/proxy/start.sh
@@ -0,0 +1,11 @@
+PROXYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PORT=${PORT:-9000}
+
+PROXYBIN=${PROXYBIN:-"$(which mitmdump)"}
+ARGS="-s '$PROXYDIR/proxy.py' -p $PORT"
+if [[ ! -z "$AUTH" ]]; then
+ ARGS="$ARGS --singleuser=$AUTH"
+fi
+PIDFILE="$PROXYDIR/proxy.pid"
+
+start-stop-daemon --start --background --pidfile $PIDFILE --make-pidfile --exec $PROXYBIN -- $ARGS
diff --git a/tests/utils/proxy/stop.sh b/tests/utils/proxy/stop.sh
new file mode 100755
index 000000000..9854d04f5
--- /dev/null
+++ b/tests/utils/proxy/stop.sh
@@ -0,0 +1,5 @@
+PROXYDIR="$PWD/$(dirname $0)"
+
+PIDFILE="$PROXYDIR/proxy.pid"
+
+start-stop-daemon --stop --pidfile $PIDFILE --make-pidfile && rm $PROXYDIR/proxy.pid