diff --git a/.travis.yml b/.travis.yml index 82fa109a6..0a612a2d6 100755 --- a/.travis.yml +++ b/.travis.yml @@ -10,28 +10,46 @@ matrix: env: TEST_COVERAGE=1 - php: 7.0 - php: hhvm - allow_failures: - - php: 7.0 -before_script: +# Use new container infrastructure +sudo: false + +cache: + directories: + - $HOME/.cache/pip + - $HOME/.composer/cache + - vendor + +install: # Setup the test server - phpenv local 5.5 - composer install --dev --no-interaction - - TESTPHPBIN=$(phpenv which php) - - sudo PHPBIN=$TESTPHPBIN vendor/bin/start.sh - - export REQUESTS_TEST_HOST_HTTP=localhost - phpenv local --unset + # Setup the proxy + - pip install --user mitmproxy + +before_script: + - PHPBIN=$TESTPHPBIN PORT=8080 vendor/bin/start.sh + - export REQUESTS_TEST_HOST_HTTP="localhost:8080" + # Work out of the tests directory - cd tests + - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9002 utils/proxy/start.sh + - PROXYBIN="$HOME/.local/bin/mitmdump" PORT=9003 AUTH="test:pass" utils/proxy/start.sh + - export REQUESTS_HTTP_PROXY="localhost:9002" + - export REQUESTS_HTTP_PROXY_AUTH="localhost:9003" + - export REQUESTS_HTTP_PROXY_AUTH_USER="test" + - export REQUESTS_HTTP_PROXY_AUTH_PASS="pass" script: - phpunit --coverage-clover clover.xml after_script: + - utils/proxy/stop.sh - cd .. - phpenv local 5.5 - - sudo PATH=$PATH vendor/bin/stop.sh + - PATH=$PATH vendor/bin/stop.sh - test $TEST_COVERAGE && bash <(curl -s https://codecov.io/bash) - phpenv local --unset diff --git a/library/Requests.php b/library/Requests.php index b9f3818bb..065227e0d 100755 --- a/library/Requests.php +++ b/library/Requests.php @@ -62,6 +62,13 @@ class Requests { */ const PATCH = 'PATCH'; + /** + * Default size of buffer size to read streams + * + * @var integer + */ + const BUFFER_SIZE = 1160; + /** * Current version of Requests * @@ -276,6 +283,8 @@ public static function patch($url, $headers, $data = array(), $options = array() * (Requests_Auth|array|boolean, default: false) * - `proxy`: Proxy details to use for proxy by-passing and authentication * (Requests_Proxy|array|boolean, default: false) + * - `max_bytes`: Limit for the response body size. + * (integer|boolean, default: false) * - `idn`: Enable IDN parsing * (boolean, default: true) * - `transport`: Custom transport. Either a class name, or a @@ -459,6 +468,7 @@ protected static function get_default_options($multirequest = false) { 'auth' => false, 'proxy' => false, 'cookies' => false, + 'max_bytes' => false, 'idn' => true, 'hooks' => null, 'transport' => null, diff --git a/library/Requests/IRI.php b/library/Requests/IRI.php index 26f215b6a..aa4dfbe71 100755 --- a/library/Requests/IRI.php +++ b/library/Requests/IRI.php @@ -906,7 +906,8 @@ protected function set_authority($authority) } if (($port_start = strpos($remaining, ':', strpos($remaining, ']'))) !== false) { - if (($port = substr($remaining, $port_start + 1)) === false) + $port = substr($remaining, $port_start + 1); + if ($port === false || $port === '') { $port = null; } diff --git a/library/Requests/Transport/cURL.php b/library/Requests/Transport/cURL.php index fc00d5c99..a147aab24 100755 --- a/library/Requests/Transport/cURL.php +++ b/library/Requests/Transport/cURL.php @@ -23,6 +23,13 @@ class Requests_Transport_cURL implements Requests_Transport { */ public $headers = ''; + /** + * Raw body data + * + * @var string + */ + public $response_data = ''; + /** * Information on the current request * @@ -44,6 +51,13 @@ class Requests_Transport_cURL implements Requests_Transport { */ protected $fp; + /** + * Hook dispatcher instance + * + * @var Requests_Hooks + */ + protected $hooks; + /** * Have we finished the headers yet? * @@ -58,6 +72,20 @@ class Requests_Transport_cURL implements Requests_Transport { */ protected $stream_handle; + /** + * How many bytes are in the response body? + * + * @var int + */ + protected $response_bytes; + + /** + * What's the maximum number of bytes we should keep? + * + * @var int|bool Byte count, or false if no limit. + */ + protected $response_byte_limit; + /** * Constructor */ @@ -91,13 +119,21 @@ public function __construct() { * @return string Raw HTTP result */ public function request($url, $headers = array(), $data = array(), $options = array()) { + $this->hooks = $options['hooks']; + $this->setup_handle($url, $headers, $data, $options); $options['hooks']->dispatch('curl.before_send', array(&$this->fp)); if ($options['filename'] !== false) { $this->stream_handle = fopen($options['filename'], 'wb'); - curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); + } + + $this->response_data = ''; + $this->response_bytes = 0; + $this->response_byte_limit = false; + if ($options['max_bytes'] !== false) { + $this->response_byte_limit = $options['max_bytes']; } if (isset($options['verify'])) { @@ -114,13 +150,19 @@ public function request($url, $headers = array(), $data = array(), $options = ar curl_setopt($this->fp, CURLOPT_SSL_VERIFYHOST, 0); } - $response = curl_exec($this->fp); + curl_exec($this->fp); + $response = $this->response_data; $options['hooks']->dispatch('curl.after_send', array(&$fake_headers)); if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) { + // Reset encoding and try again curl_setopt($this->fp, CURLOPT_ENCODING, 'none'); - $response = curl_exec($this->fp); + + $this->response_data = ''; + $this->response_bytes = 0; + curl_exec($this->fp); + $response = $this->response_data; } $this->process_response($response, $options); @@ -174,7 +216,7 @@ public function request_multiple($requests, $options) { // Parse the finished requests before we start getting the new ones foreach ($to_process as $key => $done) { $options = $requests[$key]['options']; - $responses[$key] = $subrequests[$key]->process_response(curl_multi_getcontent($done['handle']), $options); + $responses[$key] = $subrequests[$key]->process_response($subrequests[$key]->response_data, $options); $options['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$key], $requests[$key])); @@ -210,9 +252,16 @@ public function &get_subrequest_handle($url, $headers, $data, $options) { if ($options['filename'] !== false) { $this->stream_handle = fopen($options['filename'], 'wb'); - curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); } + $this->response_data = ''; + $this->response_bytes = 0; + $this->response_byte_limit = false; + if ($options['max_bytes'] !== false) { + $this->response_byte_limit = $options['max_bytes']; + } + $this->hooks = $options['hooks']; + return $this->fp; } @@ -270,6 +319,8 @@ protected function setup_handle($url, $headers, $data, $options) { if (true === $options['blocking']) { curl_setopt($this->fp, CURLOPT_HEADERFUNCTION, array(&$this, 'stream_headers')); + curl_setopt($this->fp, CURLOPT_WRITEFUNCTION, array(&$this, 'stream_body')); + curl_setopt($this->fp, CURLOPT_BUFFERSIZE, Requests::BUFFER_SIZE); } } @@ -319,6 +370,44 @@ public function stream_headers($handle, $headers) { return strlen($headers); } + /** + * Collect data as it's received + * + * @since 1.6.1 + * + * @param resource $handle cURL resource + * @param string $data Body data + * @return integer Length of provided data + */ + protected function stream_body($handle, $data) { + $this->hooks->dispatch('request.progress', array($data, $this->response_bytes, $this->response_byte_limit)); + $data_length = strlen($data); + + // Are we limiting the response size? + if ($this->response_byte_limit) { + if ($this->response_bytes === $this->response_byte_limit) { + // Already at maximum, move on + return $data_length; + } + + if (($this->response_bytes + $data_length) > $this->response_byte_limit) { + // Limit the length + $limited_length = ($this->response_byte_limit - $this->response_bytes); + $data = substr($data, 0, $limited_length); + } + } + + if ($this->stream_handle) { + fwrite($this->stream_handle, $data); + } + else { + $this->response_data .= $data; + } + + $this->response_bytes += strlen($data); + return $data_length; + } + /** * Format a URL given GET data * diff --git a/library/Requests/Transport/fsockopen.php b/library/Requests/Transport/fsockopen.php index 377c1ab32..6d4c41377 100755 --- a/library/Requests/Transport/fsockopen.php +++ b/library/Requests/Transport/fsockopen.php @@ -34,6 +34,13 @@ class Requests_Transport_fsockopen implements Requests_Transport { */ public $info; + /** + * What's the maximum number of bytes we should keep? + * + * @var int|bool Byte count, or false if no limit. + */ + protected $max_bytes = false; + protected $connect_error = ''; /** @@ -94,6 +101,8 @@ public function request($url, $headers = array(), $data = array(), $options = ar $remote_socket = 'tcp://' . $host; } + $this->max_bytes = $options['max_bytes']; + $proxy = isset( $options['proxy'] ); $proxy_auth = $proxy && isset( $options['proxy_username'] ) && isset( $options['proxy_password'] ); @@ -210,45 +219,63 @@ public function request($url, $headers = array(), $data = array(), $options = ar $timeout_msec = $timeout_sec == $options['timeout'] ? 0 : self::SECOND_IN_MICROSECONDS * $options['timeout'] % self::SECOND_IN_MICROSECONDS; stream_set_timeout($fp, $timeout_sec, $timeout_msec); + $response = $body = $headers = ''; $this->info = stream_get_meta_data($fp); + $size = 0; + $doingbody = false; + $download = false; + if ($options['filename']) { + $download = fopen($options['filename'], 'wb'); + } - $this->headers = ''; - $this->info = stream_get_meta_data($fp); - if (!$options['filename']) { - while (!feof($fp)) { - $this->info = stream_get_meta_data($fp); - if ($this->info['timed_out']) { - throw new Requests_Exception('fsocket timed out', 'timeout'); - } + while (!feof($fp)) { + $this->info = stream_get_meta_data($fp); + if ($this->info['timed_out']) { + throw new Requests_Exception('fsocket timed out', 'timeout'); + } - $this->headers .= fread($fp, 1160); + $block = fread($fp, Requests::BUFFER_SIZE); + if (!$doingbody) { + $response .= $block; + if (strpos($response, "\r\n\r\n")) { + list($headers, $block) = explode("\r\n\r\n", $response, 2); + $doingbody = true; + } } - } - else { - $download = fopen($options['filename'], 'wb'); - $doingbody = false; - $response = ''; - while (!feof($fp)) { - $this->info = stream_get_meta_data($fp); - if ($this->info['timed_out']) { - throw new Requests_Exception('fsocket timed out', 'timeout'); + + // Are we in body mode now? + if ($doingbody) { + $options['hooks']->dispatch('request.progress', array($block, $size, $this->max_bytes)); + $data_length = strlen($block); + if ($this->max_bytes) { + // Have we already hit a limit? + if ($size === $this->max_bytes) { + continue; + } + if (($size + $data_length) > $this->max_bytes) { + // Limit the length + $limited_length = ($this->max_bytes - $size); + $block = substr($block, 0, $limited_length); + } } - $block = fread($fp, 1160); - if ($doingbody) { + $size += strlen($block); + if ($download) { fwrite($download, $block); } else { - $response .= $block; - if (strpos($response, "\r\n\r\n")) { - list($this->headers, $block) = explode("\r\n\r\n", $response, 2); - $doingbody = true; - fwrite($download, $block); - } + $body .= $block; } } + } + $this->headers = $headers; + + if ($download) { fclose($download); } + else { + $this->headers .= "\r\n\r\n" . $body; + } fclose($fp); $options['hooks']->dispatch('fsockopen.after_request', array(&$this->headers)); diff --git a/tests/Proxy/HTTP.php b/tests/Proxy/HTTP.php new file mode 100644 index 000000000..351570dfe --- /dev/null +++ b/tests/Proxy/HTTP.php @@ -0,0 +1,131 @@ +markTestSkipped('Proxy not available'); + } + } + + public function transportProvider() { + return array( + array('Requests_Transport_cURL'), + array('Requests_Transport_fsockopen'), + ); + } + + /** + * @dataProvider transportProvider + */ + public function testConnectWithString($transport) { + $this->checkProxyAvailable(); + + $options = array( + 'proxy' => REQUESTS_HTTP_PROXY, + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + $this->assertEquals('http', $response->headers['x-requests-proxied']); + + $data = json_decode($response->body, true); + $this->assertEquals('http', $data['headers']['x-requests-proxy']); + } + + /** + * @dataProvider transportProvider + */ + public function testConnectWithArray($transport) { + $this->checkProxyAvailable(); + + $options = array( + 'proxy' => array(REQUESTS_HTTP_PROXY), + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + $this->assertEquals('http', $response->headers['x-requests-proxied']); + + $data = json_decode($response->body, true); + $this->assertEquals('http', $data['headers']['x-requests-proxy']); + } + + /** + * @dataProvider transportProvider + * @expectedException Requests_Exception + */ + public function testConnectInvalidParameters($transport) { + $this->checkProxyAvailable(); + + $options = array( + 'proxy' => array(REQUESTS_HTTP_PROXY, 'testuser', 'password', 'something'), + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + } + + /** + * @dataProvider transportProvider + */ + public function testConnectWithInstance($transport) { + $this->checkProxyAvailable(); + + $options = array( + 'proxy' => REQUESTS_HTTP_PROXY, + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + $this->assertEquals('http', $response->headers['x-requests-proxied']); + + $data = json_decode($response->body, true); + $this->assertEquals('http', $data['headers']['x-requests-proxy']); + } + + /** + * @dataProvider transportProvider + */ + public function testConnectWithAuth($transport) { + $this->checkProxyAvailable('auth'); + + $options = array( + 'proxy' => array( + REQUESTS_HTTP_PROXY_AUTH, + REQUESTS_HTTP_PROXY_AUTH_USER, + REQUESTS_HTTP_PROXY_AUTH_PASS + ), + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + $this->assertEquals(200, $response->status_code); + $this->assertEquals('http', $response->headers['x-requests-proxied']); + + $data = json_decode($response->body, true); + $this->assertEquals('http', $data['headers']['x-requests-proxy']); + } + + /** + * @dataProvider transportProvider + */ + public function testConnectWithInvalidAuth($transport) { + $this->checkProxyAvailable('auth'); + + $options = array( + 'proxy' => array( + REQUESTS_HTTP_PROXY_AUTH, + REQUESTS_HTTP_PROXY_AUTH_USER . '!', + REQUESTS_HTTP_PROXY_AUTH_PASS . '!' + ), + 'transport' => $transport, + ); + $response = Requests::get(httpbin('/get'), array(), $options); + $this->assertEquals(407, $response->status_code); + } +} \ No newline at end of file diff --git a/tests/Transport/Base.php b/tests/Transport/Base.php index 352728612..b591df07e 100755 --- a/tests/Transport/Base.php +++ b/tests/Transport/Base.php @@ -25,6 +25,27 @@ protected function getOptions($other = array()) { return $options; } + public function testResponseByteLimit() { + $limit = 104; + $options = array( + 'max_bytes' => $limit, + ); + $response = Requests::get(httpbin('/bytes/325'), array(), $this->getOptions($options)); + $this->assertEquals($limit, strlen($response->body)); + } + + public function testResponseByteLimitWithFile() { + $limit = 300; + $options = array( + 'max_bytes' => $limit, + 'filename' => tempnam(sys_get_temp_dir(), 'RLT') // RequestsLibraryTest + ); + $response = Requests::get(httpbin('/bytes/482'), array(), $this->getOptions($options)); + $this->assertEmpty($response->body); + $this->assertEquals($limit, filesize($options['filename'])); + unlink($options['filename']); + } + public function testSimpleGET() { $request = Requests::get(httpbin('/get'), array(), $this->getOptions()); $this->assertEquals(200, $request->status_code); @@ -706,4 +727,17 @@ public function testAlternatePort() { $this->assertEquals(1, $num, 'Response should contain the port number'); $this->assertEquals(8080, $matches[1]); } + + public function testProgressCallback() { + $mock = $this->getMockBuilder('stdClass')->setMethods(array('progress'))->getMock(); + $mock->expects($this->atLeastOnce())->method('progress'); + $hooks = new Requests_Hooks(); + $hooks->register('request.progress', array($mock, 'progress')); + $options = array( + 'hooks' => $hooks, + ); + $options = $this->getOptions($options); + + $response = Requests::get(httpbin('/get'), array(), $options); + } } diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 41b94e98d..8cc8c0e19 100755 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -2,13 +2,24 @@ date_default_timezone_set('UTC'); -$host = getenv('REQUESTS_TEST_HOST'); -if (empty($host)) { - $host = 'httpbin.org'; +function define_from_env($name, $default = false) { + $env = getenv($name); + if ($env) { + define($name, $env); + } + else { + define($name, $default); + } } -define('REQUESTS_TEST_HOST', getenv('REQUESTS_TEST_HOST') ? getenv('REQUESTS_TEST_HOST') : 'httpbin.org'); -define('REQUESTS_TEST_HOST_HTTP', getenv('REQUESTS_TEST_HOST_HTTP') ? getenv('REQUESTS_TEST_HOST_HTTP') : REQUESTS_TEST_HOST); -define('REQUESTS_TEST_HOST_HTTPS', getenv('REQUESTS_TEST_HOST_HTTPS') ? getenv('REQUESTS_TEST_HOST_HTTPS'): REQUESTS_TEST_HOST); + +define_from_env('REQUESTS_TEST_HOST', 'requests-php-tests.herokuapp.com'); +define_from_env('REQUESTS_TEST_HOST_HTTP', REQUESTS_TEST_HOST); +define_from_env('REQUESTS_TEST_HOST_HTTPS', REQUESTS_TEST_HOST); + +define_from_env('REQUESTS_HTTP_PROXY'); +define_from_env('REQUESTS_HTTP_PROXY_AUTH'); +define_from_env('REQUESTS_HTTP_PROXY_AUTH_USER'); +define_from_env('REQUESTS_HTTP_PROXY_AUTH_PASS'); include(dirname(dirname(__FILE__)) . '/library/Requests.php'); Requests::register_autoloader(); diff --git a/tests/phpunit.xml.dist b/tests/phpunit.xml.dist index 847b9b661..e46ea1719 100755 --- a/tests/phpunit.xml.dist +++ b/tests/phpunit.xml.dist @@ -7,6 +7,9 @@ Transport + + Proxy + ChunkedEncoding.php Cookies.php diff --git a/tests/utils/proxy/proxy.py b/tests/utils/proxy/proxy.py new file mode 100755 index 000000000..502c1d91f --- /dev/null +++ b/tests/utils/proxy/proxy.py @@ -0,0 +1,5 @@ +def request(context, flow): + flow.request.headers["x-requests-proxy"] = ["http"] + +def response(context, flow): + flow.response.headers["x-requests-proxied"] = ["http"] \ No newline at end of file diff --git a/tests/utils/proxy/start.sh b/tests/utils/proxy/start.sh new file mode 100755 index 000000000..4e7edeca6 --- /dev/null +++ b/tests/utils/proxy/start.sh @@ -0,0 +1,11 @@ +PROXYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PORT=${PORT:-9000} + +PROXYBIN=${PROXYBIN:-"$(which mitmdump)"} +ARGS="-s '$PROXYDIR/proxy.py' -p $PORT" +if [[ ! -z "$AUTH" ]]; then + ARGS="$ARGS --singleuser=$AUTH" +fi +PIDFILE="$PROXYDIR/proxy.pid" + +start-stop-daemon --start --background --pidfile $PIDFILE --make-pidfile --exec $PROXYBIN -- $ARGS diff --git a/tests/utils/proxy/stop.sh b/tests/utils/proxy/stop.sh new file mode 100755 index 000000000..9854d04f5 --- /dev/null +++ b/tests/utils/proxy/stop.sh @@ -0,0 +1,5 @@ +PROXYDIR="$PWD/$(dirname $0)" + +PIDFILE="$PROXYDIR/proxy.pid" + +start-stop-daemon --stop --pidfile $PIDFILE --make-pidfile && rm $PROXYDIR/proxy.pid