diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index bb4661c3d10fe..45e7e7d800408 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -1,5 +1,9 @@ Version history --------------- + 1.8.1 (Apr 5, 2019) + =================== +* http: fixed CVE-2019-9900 by rejecting HTTP/1.x headers with embedded NUL characters. +* http: fixed CVE-2019-9901 by normalizing HTTP paths prior to routing or L7 data plane processing. 1.8.0 (Pending) =============== diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index efbf38638eb25..2899d48bba0f2 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -19,6 +19,17 @@ namespace Envoy { namespace Http { +// Used by ASSERTs to validate internal consistency. E.g. valid HTTP header keys/values should +// never contain embedded NULLs. +static inline bool validHeaderString(absl::string_view s) { + for (const char c : {'\0', '\r', '\n'}) { + if (s.find(c) != absl::string_view::npos) { + return false; + } + } + return true; +} + /** * Wrapper for a lower case string used in header operations to generally avoid needless case * insensitive compares. @@ -35,6 +46,7 @@ class LowerCaseString { private: void lower() { std::transform(string_.begin(), string_.end(), string_.begin(), tolower); } + bool valid() const { return validHeaderString(string_); } std::string string_; }; @@ -166,6 +178,8 @@ class HeaderString { void freeDynamic(); + bool valid() const; + uint32_t string_length_; Type type_; }; diff --git a/source/common/chromium_url/BUILD b/source/common/chromium_url/BUILD new file mode 100644 index 0000000000000..9b07e76b00130 --- /dev/null +++ b/source/common/chromium_url/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "chromium_url", + srcs = [ + "url_canon.cc", + "url_canon_internal.cc", + "url_canon_path.cc", + "url_canon_stdstring.cc", + ], + hdrs = [ + "envoy_shim.h", + "url_canon.h", + "url_canon_internal.h", + "url_canon_stdstring.h", + "url_parse.h", + "url_parse_internal.h", + ], + deps = ["//source/common/common:assert_lib"], +) diff --git a/source/common/chromium_url/LICENSE b/source/common/chromium_url/LICENSE new file mode 100644 index 0000000000000..a32e00ce6be36 --- /dev/null +++ b/source/common/chromium_url/LICENSE @@ -0,0 +1,27 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/source/common/chromium_url/README.md b/source/common/chromium_url/README.md new file mode 100644 index 0000000000000..64d28b315dd20 --- /dev/null +++ b/source/common/chromium_url/README.md @@ -0,0 +1,15 @@ +This is a manually minified variant of +https://chromium.googlesource.com/chromium/src.git/+archive/74.0.3729.15/url.tar.gz, +providing just the parts needed for `url::CanonicalizePath()`. This is intended +to support a security release fix for CVE-2019-9901. Long term we need this to +be moved to absl or QUICHE for upgrades and long-term support. + +Some specific transforms of interest: +* `url_parse.h` is minified to just `Component` and flattened back into the URL + directory. It does not contain any non-Chromium authored code any longer and + so does not have a separate LICENSE. +* `envoy_shim.h` adapts various macros to the Envoy context. +* Anything not reachable from `url::CanonicalizePath()` has been dropped. +* Header include paths have changed as needed. +* BUILD was manually written. +* Various clang-tidy and format fixes. diff --git a/source/common/chromium_url/envoy_shim.h b/source/common/chromium_url/envoy_shim.h new file mode 100644 index 0000000000000..2b7443926c1f5 --- /dev/null +++ b/source/common/chromium_url/envoy_shim.h @@ -0,0 +1,17 @@ +#pragma once + +#include "common/common/assert.h" + +// This is a minimal Envoy adaptation layer for the Chromium URL library. +// NOLINT(namespace-envoy) + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete + +#define EXPORT_TEMPLATE_DECLARE(x) +#define EXPORT_TEMPLATE_DEFINE(x) +#define COMPONENT_EXPORT(x) + +#define DCHECK(x) ASSERT(x) +#define NOTREACHED() NOT_REACHED_GCOVR_EXCL_LINE diff --git a/source/common/chromium_url/url_canon.cc b/source/common/chromium_url/url_canon.cc new file mode 100644 index 0000000000000..91926b6f237b6 --- /dev/null +++ b/source/common/chromium_url/url_canon.cc @@ -0,0 +1,16 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "common/chromium_url/url_canon.h" + +#include "common/chromium_url/envoy_shim.h" + +namespace url { + +template class EXPORT_TEMPLATE_DEFINE(COMPONENT_EXPORT(URL)) CanonOutputT; + +} // namespace url diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h new file mode 100644 index 0000000000000..233f56a91c036 --- /dev/null +++ b/source/common/chromium_url/url_canon.h @@ -0,0 +1,187 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef URL_URL_CANON_H_ +#define URL_URL_CANON_H_ + +#include +#include + +#include "common/chromium_url/envoy_shim.h" +#include "common/chromium_url/url_parse.h" + +namespace url { + +// Canonicalizer output ------------------------------------------------------- + +// Base class for the canonicalizer output, this maintains a buffer and +// supports simple resizing and append operations on it. +// +// It is VERY IMPORTANT that no virtual function calls be made on the common +// code path. We only have two virtual function calls, the destructor and a +// resize function that is called when the existing buffer is not big enough. +// The derived class is then in charge of setting up our buffer which we will +// manage. +template class CanonOutputT { +public: + CanonOutputT() : buffer_(NULL), buffer_len_(0), cur_len_(0) {} + virtual ~CanonOutputT() {} + + // Implemented to resize the buffer. This function should update the buffer + // pointer to point to the new buffer, and any old data up to |cur_len_| in + // the buffer must be copied over. + // + // The new size |sz| must be larger than buffer_len_. + virtual void Resize(int sz) = 0; + + // Accessor for returning a character at a given position. The input offset + // must be in the valid range. + inline T at(int offset) const { return buffer_[offset]; } + + // Sets the character at the given position. The given position MUST be less + // than the length(). + inline void set(int offset, T ch) { buffer_[offset] = ch; } + + // Returns the number of characters currently in the buffer. + inline int length() const { return cur_len_; } + + // Returns the current capacity of the buffer. The length() is the number of + // characters that have been declared to be written, but the capacity() is + // the number that can be written without reallocation. If the caller must + // write many characters at once, it can make sure there is enough capacity, + // write the data, then use set_size() to declare the new length(). + int capacity() const { return buffer_len_; } + + // Called by the user of this class to get the output. The output will NOT + // be NULL-terminated. Call length() to get the + // length. + const T* data() const { return buffer_; } + T* data() { return buffer_; } + + // Shortens the URL to the new length. Used for "backing up" when processing + // relative paths. This can also be used if an external function writes a lot + // of data to the buffer (when using the "Raw" version below) beyond the end, + // to declare the new length. + // + // This MUST NOT be used to expand the size of the buffer beyond capacity(). + void set_length(int new_len) { cur_len_ = new_len; } + + // This is the most performance critical function, since it is called for + // every character. + void push_back(T ch) { + // In VC2005, putting this common case first speeds up execution + // dramatically because this branch is predicted as taken. + if (cur_len_ < buffer_len_) { + buffer_[cur_len_] = ch; + cur_len_++; + return; + } + + // Grow the buffer to hold at least one more item. Hopefully we won't have + // to do this very often. + if (!Grow(1)) + return; + + // Actually do the insertion. + buffer_[cur_len_] = ch; + cur_len_++; + } + + // Appends the given string to the output. + void Append(const T* str, int str_len) { + if (cur_len_ + str_len > buffer_len_) { + if (!Grow(cur_len_ + str_len - buffer_len_)) + return; + } + for (int i = 0; i < str_len; i++) + buffer_[cur_len_ + i] = str[i]; + cur_len_ += str_len; + } + + void ReserveSizeIfNeeded(int estimated_size) { + // Reserve a bit extra to account for escaped chars. + if (estimated_size > buffer_len_) + Resize(estimated_size + 8); + } + +protected: + // Grows the given buffer so that it can fit at least |min_additional| + // characters. Returns true if the buffer could be resized, false on OOM. + bool Grow(int min_additional) { + static const int kMinBufferLen = 16; + int new_len = (buffer_len_ == 0) ? kMinBufferLen : buffer_len_; + do { + if (new_len >= (1 << 30)) // Prevent overflow below. + return false; + new_len *= 2; + } while (new_len < buffer_len_ + min_additional); + Resize(new_len); + return true; + } + + T* buffer_; + int buffer_len_; + + // Used characters in the buffer. + int cur_len_; +}; + +// Simple implementation of the CanonOutput using new[]. This class +// also supports a static buffer so if it is allocated on the stack, most +// URLs can be canonicalized with no heap allocations. +template class RawCanonOutputT : public CanonOutputT { +public: + RawCanonOutputT() : CanonOutputT() { + this->buffer_ = fixed_buffer_; + this->buffer_len_ = fixed_capacity; + } + ~RawCanonOutputT() override { + if (this->buffer_ != fixed_buffer_) + delete[] this->buffer_; + } + + void Resize(int sz) override { + T* new_buf = new T[sz]; + memcpy(new_buf, this->buffer_, sizeof(T) * (this->cur_len_ < sz ? this->cur_len_ : sz)); + if (this->buffer_ != fixed_buffer_) + delete[] this->buffer_; + this->buffer_ = new_buf; + this->buffer_len_ = sz; + } + +protected: + T fixed_buffer_[fixed_capacity]; +}; + +// Explicitly instantiate commonly used instantiations. +extern template class EXPORT_TEMPLATE_DECLARE(COMPONENT_EXPORT(URL)) CanonOutputT; + +// Normally, all canonicalization output is in narrow characters. We support +// the templates so it can also be used internally if a wide buffer is +// required. +typedef CanonOutputT CanonOutput; + +template +class RawCanonOutput : public RawCanonOutputT {}; + +// Path. If the input does not begin in a slash (including if the input is +// empty), we'll prepend a slash to the path to make it canonical. +// +// The 8-bit version assumes UTF-8 encoding, but does not verify the validity +// of the UTF-8 (i.e., you can have invalid UTF-8 sequences, invalid +// characters, etc.). Normally, URLs will come in as UTF-16, so this isn't +// an issue. Somebody giving us an 8-bit path is responsible for generating +// the path that the server expects (we'll escape high-bit characters), so +// if something is invalid, it's their problem. +COMPONENT_EXPORT(URL) +bool CanonicalizePath(const char* spec, const Component& path, CanonOutput* output, + Component* out_path); + +} // namespace url + +#endif // URL_URL_CANON_H_ + diff --git a/source/common/chromium_url/url_canon_internal.cc b/source/common/chromium_url/url_canon_internal.cc new file mode 100644 index 0000000000000..7aeb4f3de1b88 --- /dev/null +++ b/source/common/chromium_url/url_canon_internal.cc @@ -0,0 +1,295 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "common/chromium_url/url_canon_internal.h" + +namespace url { + +// See the header file for this array's declaration. +const unsigned char kSharedCharTypeTable[0x100] = { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0x00 - 0x0f + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0x10 - 0x1f + 0, // 0x20 ' ' (escape spaces in queries) + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x21 ! + 0, // 0x22 " + 0, // 0x23 # (invalid in query since it marks the ref) + CHAR_QUERY | CHAR_USERINFO, // 0x24 $ + CHAR_QUERY | CHAR_USERINFO, // 0x25 % + CHAR_QUERY | CHAR_USERINFO, // 0x26 & + 0, // 0x27 ' (Try to prevent XSS.) + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x28 ( + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x29 ) + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x2a * + CHAR_QUERY | CHAR_USERINFO, // 0x2b + + CHAR_QUERY | CHAR_USERINFO, // 0x2c , + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x2d - + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_COMPONENT, // 0x2e . + CHAR_QUERY, // 0x2f / + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x30 0 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x31 1 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x32 2 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x33 3 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x34 4 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x35 5 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x36 6 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_OCT | + CHAR_COMPONENT, // 0x37 7 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_COMPONENT, // 0x38 8 + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_DEC | CHAR_COMPONENT, // 0x39 9 + CHAR_QUERY, // 0x3a : + CHAR_QUERY, // 0x3b ; + 0, // 0x3c < (Try to prevent certain types of XSS.) + CHAR_QUERY, // 0x3d = + 0, // 0x3e > (Try to prevent certain types of XSS.) + CHAR_QUERY, // 0x3f ? + CHAR_QUERY, // 0x40 @ + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x41 A + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x42 B + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x43 C + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x44 D + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x45 E + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x46 F + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x47 G + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x48 H + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x49 I + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4a J + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4b K + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4c L + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4d M + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4e N + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x4f O + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x50 P + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x51 Q + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x52 R + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x53 S + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x54 T + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x55 U + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x56 V + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x57 W + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_COMPONENT, // 0x58 X + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x59 Y + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x5a Z + CHAR_QUERY, // 0x5b [ + CHAR_QUERY, // 0x5c '\' + CHAR_QUERY, // 0x5d ] + CHAR_QUERY, // 0x5e ^ + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x5f _ + CHAR_QUERY, // 0x60 ` + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x61 a + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x62 b + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x63 c + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x64 d + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x65 e + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_HEX | CHAR_COMPONENT, // 0x66 f + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x67 g + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x68 h + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x69 i + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6a j + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6b k + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6c l + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6d m + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6e n + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x6f o + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x70 p + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x71 q + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x72 r + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x73 s + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x74 t + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x75 u + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x76 v + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x77 w + CHAR_QUERY | CHAR_USERINFO | CHAR_IPV4 | CHAR_COMPONENT, // 0x78 x + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x79 y + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x7a z + CHAR_QUERY, // 0x7b { + CHAR_QUERY, // 0x7c | + CHAR_QUERY, // 0x7d } + CHAR_QUERY | CHAR_USERINFO | CHAR_COMPONENT, // 0x7e ~ + 0, // 0x7f + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0x80 - 0x8f + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0x90 - 0x9f + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xa0 - 0xaf + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xb0 - 0xbf + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xc0 - 0xcf + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xd0 - 0xdf + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xe0 - 0xef + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, // 0xf0 - 0xff +}; + +const char kHexCharLookup[0x10] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', +}; + +const char kCharToHexLookup[8] = { + 0, // 0x00 - 0x1f + '0', // 0x20 - 0x3f: digits 0 - 9 are 0x30 - 0x39 + 'A' - 10, // 0x40 - 0x5f: letters A - F are 0x41 - 0x46 + 'a' - 10, // 0x60 - 0x7f: letters a - f are 0x61 - 0x66 + 0, // 0x80 - 0x9F + 0, // 0xA0 - 0xBF + 0, // 0xC0 - 0xDF + 0, // 0xE0 - 0xFF +}; + +} // namespace url diff --git a/source/common/chromium_url/url_canon_internal.h b/source/common/chromium_url/url_canon_internal.h new file mode 100644 index 0000000000000..7b2cae4f43157 --- /dev/null +++ b/source/common/chromium_url/url_canon_internal.h @@ -0,0 +1,247 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef URL_URL_CANON_INTERNAL_H_ +#define URL_URL_CANON_INTERNAL_H_ + +// This file is intended to be included in another C++ file where the character +// types are defined. This allows us to write mostly generic code, but not have +// template bloat because everything is inlined when anybody calls any of our +// functions. + +#include +#include + +#include "common/chromium_url/envoy_shim.h" +#include "common/chromium_url/url_canon.h" + +namespace url { + +// Character type handling ----------------------------------------------------- + +// Bits that identify different character types. These types identify different +// bits that are set for each 8-bit character in the kSharedCharTypeTable. +enum SharedCharTypes { + // Characters that do not require escaping in queries. Characters that do + // not have this flag will be escaped; see url_canon_query.cc + CHAR_QUERY = 1, + + // Valid in the username/password field. + CHAR_USERINFO = 2, + + // Valid in a IPv4 address (digits plus dot and 'x' for hex). + CHAR_IPV4 = 4, + + // Valid in an ASCII-representation of a hex digit (as in %-escaped). + CHAR_HEX = 8, + + // Valid in an ASCII-representation of a decimal digit. + CHAR_DEC = 16, + + // Valid in an ASCII-representation of an octal digit. + CHAR_OCT = 32, + + // Characters that do not require escaping in encodeURIComponent. Characters + // that do not have this flag will be escaped; see url_util.cc. + CHAR_COMPONENT = 64, +}; + +// This table contains the flags in SharedCharTypes for each 8-bit character. +// Some canonicalization functions have their own specialized lookup table. +// For those with simple requirements, we have collected the flags in one +// place so there are fewer lookup tables to load into the CPU cache. +// +// Using an unsigned char type has a small but measurable performance benefit +// over using a 32-bit number. +extern const unsigned char kSharedCharTypeTable[0x100]; + +// More readable wrappers around the character type lookup table. +inline bool IsCharOfType(unsigned char c, SharedCharTypes type) { + return !!(kSharedCharTypeTable[c] & type); +} +inline bool IsQueryChar(unsigned char c) { return IsCharOfType(c, CHAR_QUERY); } +inline bool IsIPv4Char(unsigned char c) { return IsCharOfType(c, CHAR_IPV4); } +inline bool IsHexChar(unsigned char c) { return IsCharOfType(c, CHAR_HEX); } +inline bool IsComponentChar(unsigned char c) { return IsCharOfType(c, CHAR_COMPONENT); } + +// Maps the hex numerical values 0x0 to 0xf to the corresponding ASCII digit +// that will be used to represent it. +COMPONENT_EXPORT(URL) extern const char kHexCharLookup[0x10]; + +// This lookup table allows fast conversion between ASCII hex letters and their +// corresponding numerical value. The 8-bit range is divided up into 8 +// regions of 0x20 characters each. Each of the three character types (numbers, +// uppercase, lowercase) falls into different regions of this range. The table +// contains the amount to subtract from characters in that range to get at +// the corresponding numerical value. +// +// See HexDigitToValue for the lookup. +extern const char kCharToHexLookup[8]; + +// Assumes the input is a valid hex digit! Call IsHexChar before using this. +inline unsigned char HexCharToValue(unsigned char c) { return c - kCharToHexLookup[c / 0x20]; } + +// Indicates if the given character is a dot or dot equivalent, returning the +// number of characters taken by it. This will be one for a literal dot, 3 for +// an escaped dot. If the character is not a dot, this will return 0. +template inline int IsDot(const CHAR* spec, int offset, int end) { + if (spec[offset] == '.') { + return 1; + } else if (spec[offset] == '%' && offset + 3 <= end && spec[offset + 1] == '2' && + (spec[offset + 2] == 'e' || spec[offset + 2] == 'E')) { + // Found "%2e" + return 3; + } + return 0; +} + +// Write a single character, escaped, to the output. This always escapes: it +// does no checking that thee character requires escaping. +// Escaping makes sense only 8 bit chars, so code works in all cases of +// input parameters (8/16bit). +template +inline void AppendEscapedChar(UINCHAR ch, CanonOutputT* output) { + output->push_back('%'); + output->push_back(kHexCharLookup[(ch >> 4) & 0xf]); + output->push_back(kHexCharLookup[ch & 0xf]); +} + +// UTF-8 functions ------------------------------------------------------------ + +// Reads one character in UTF-8 starting at |*begin| in |str| and places +// the decoded value into |*code_point|. If the character is valid, we will +// return true. If invalid, we'll return false and put the +// kUnicodeReplacementCharacter into |*code_point|. +// +// |*begin| will be updated to point to the last character consumed so it +// can be incremented in a loop and will be ready for the next character. +// (for a single-byte ASCII character, it will not be changed). +COMPONENT_EXPORT(URL) +bool ReadUTFChar(const char* str, int* begin, int length, unsigned* code_point_out); + +// Generic To-UTF-8 converter. This will call the given append method for each +// character that should be appended, with the given output method. Wrappers +// are provided below for escaped and non-escaped versions of this. +// +// The char_value must have already been checked that it's a valid Unicode +// character. +template +inline void DoAppendUTF8(unsigned char_value, Output* output) { + if (char_value <= 0x7f) { + Appender(static_cast(char_value), output); + } else if (char_value <= 0x7ff) { + // 110xxxxx 10xxxxxx + Appender(static_cast(0xC0 | (char_value >> 6)), output); + Appender(static_cast(0x80 | (char_value & 0x3f)), output); + } else if (char_value <= 0xffff) { + // 1110xxxx 10xxxxxx 10xxxxxx + Appender(static_cast(0xe0 | (char_value >> 12)), output); + Appender(static_cast(0x80 | ((char_value >> 6) & 0x3f)), output); + Appender(static_cast(0x80 | (char_value & 0x3f)), output); + } else if (char_value <= 0x10FFFF) { // Max Unicode code point. + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + Appender(static_cast(0xf0 | (char_value >> 18)), output); + Appender(static_cast(0x80 | ((char_value >> 12) & 0x3f)), output); + Appender(static_cast(0x80 | ((char_value >> 6) & 0x3f)), output); + Appender(static_cast(0x80 | (char_value & 0x3f)), output); + } else { + // Invalid UTF-8 character (>20 bits). + NOTREACHED(); + } +} + +// Helper used by AppendUTF8Value below. We use an unsigned parameter so there +// are no funny sign problems with the input, but then have to convert it to +// a regular char for appending. +inline void AppendCharToOutput(unsigned char ch, CanonOutput* output) { + output->push_back(static_cast(ch)); +} + +// Writes the given character to the output as UTF-8. This does NO checking +// of the validity of the Unicode characters; the caller should ensure that +// the value it is appending is valid to append. +inline void AppendUTF8Value(unsigned char_value, CanonOutput* output) { + DoAppendUTF8(char_value, output); +} + +// Writes the given character to the output as UTF-8, escaping ALL +// characters (even when they are ASCII). This does NO checking of the +// validity of the Unicode characters; the caller should ensure that the value +// it is appending is valid to append. +inline void AppendUTF8EscapedValue(unsigned char_value, CanonOutput* output) { + DoAppendUTF8(char_value, output); +} + +// Escaping functions --------------------------------------------------------- + +// Writes the given character to the output as UTF-8, escaped. Call this +// function only when the input is wide. Returns true on success. Failure +// means there was some problem with the encoding, we'll still try to +// update the |*begin| pointer and add a placeholder character to the +// output so processing can continue. +// +// We will append the character starting at ch[begin] with the buffer ch +// being |length|. |*begin| will be updated to point to the last character +// consumed (we may consume more than one for UTF-16) so that if called in +// a loop, incrementing the pointer will move to the next character. +// +// Every single output character will be escaped. This means that if you +// give it an ASCII character as input, it will be escaped. Some code uses +// this when it knows that a character is invalid according to its rules +// for validity. If you don't want escaping for ASCII characters, you will +// have to filter them out prior to calling this function. +// +// Assumes that ch[begin] is within range in the array, but does not assume +// that any following characters are. +inline bool AppendUTF8EscapedChar(const char* str, int* begin, int length, CanonOutput* output) { + // ReadUTF8Char will handle invalid characters for us and give us the + // kUnicodeReplacementCharacter, so we don't have to do special checking + // after failure, just pass through the failure to the caller. + unsigned ch; + bool success = ReadUTFChar(str, begin, length, &ch); + AppendUTF8EscapedValue(ch, output); + return success; +} + +// Given a '%' character at |*begin| in the string |spec|, this will decode +// the escaped value and put it into |*unescaped_value| on success (returns +// true). On failure, this will return false, and will not write into +// |*unescaped_value|. +// +// |*begin| will be updated to point to the last character of the escape +// sequence so that when called with the index of a for loop, the next time +// through it will point to the next character to be considered. On failure, +// |*begin| will be unchanged. +inline bool Is8BitChar(char /*c*/) { + return true; // this case is specialized to avoid a warning +} + +template +inline bool DecodeEscaped(const CHAR* spec, int* begin, int end, unsigned char* unescaped_value) { + if (*begin + 3 > end || !Is8BitChar(spec[*begin + 1]) || !Is8BitChar(spec[*begin + 2])) { + // Invalid escape sequence because there's not enough room, or the + // digits are not ASCII. + return false; + } + + unsigned char first = static_cast(spec[*begin + 1]); + unsigned char second = static_cast(spec[*begin + 2]); + if (!IsHexChar(first) || !IsHexChar(second)) { + // Invalid hex digits, fail. + return false; + } + + // Valid escape sequence. + *unescaped_value = (HexCharToValue(first) << 4) + HexCharToValue(second); + *begin += 2; + return true; +} + +} // namespace url + +#endif // URL_URL_CANON_INTERNAL_H_ + diff --git a/source/common/chromium_url/url_canon_path.cc b/source/common/chromium_url/url_canon_path.cc new file mode 100644 index 0000000000000..73c9eca0597d4 --- /dev/null +++ b/source/common/chromium_url/url_canon_path.cc @@ -0,0 +1,418 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "common/chromium_url/url_canon.h" +#include "common/chromium_url/url_canon_internal.h" +#include "common/chromium_url/url_parse_internal.h" + +namespace url { + +namespace { + +enum CharacterFlags { + // Pass through unchanged, whether escaped or unescaped. This doesn't + // actually set anything so you can't OR it to check, it's just to make the + // table below more clear when neither ESCAPE or UNESCAPE is set. + PASS = 0, + + // This character requires special handling in DoPartialPath. Doing this test + // first allows us to filter out the common cases of regular characters that + // can be directly copied. + SPECIAL = 1, + + // This character must be escaped in the canonical output. Note that all + // escaped chars also have the "special" bit set so that the code that looks + // for this is triggered. Not valid with PASS or ESCAPE + ESCAPE_BIT = 2, + ESCAPE = ESCAPE_BIT | SPECIAL, + + // This character must be unescaped in canonical output. Not valid with + // ESCAPE or PASS. We DON'T set the SPECIAL flag since if we encounter these + // characters unescaped, they should just be copied. + UNESCAPE = 4, + + // This character is disallowed in URLs. Note that the "special" bit is also + // set to trigger handling. + INVALID_BIT = 8, + INVALID = INVALID_BIT | SPECIAL, +}; + +// This table contains one of the above flag values. Note some flags are more +// than one bits because they also turn on the "special" flag. Special is the +// only flag that may be combined with others. +// +// This table is designed to match exactly what IE does with the characters. +// +// Dot is even more special, and the escaped version is handled specially by +// IsDot. Therefore, we don't need the "escape" flag, and even the "unescape" +// bit is never handled (we just need the "special") bit. +const unsigned char kPathCharLookup[0x100] = { + // NULL control chars... + INVALID, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, + // control chars... + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, + // ' ' ! " # $ % & ' ( ) * + // + , - . / + ESCAPE, PASS, ESCAPE, ESCAPE, PASS, ESCAPE, PASS, PASS, PASS, PASS, PASS, PASS, PASS, UNESCAPE, + SPECIAL, PASS, + // 0 1 2 3 4 5 6 7 8 9 : + // ; < = > ? + UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + UNESCAPE, PASS, PASS, ESCAPE, PASS, ESCAPE, ESCAPE, + // @ A B C D E F G H I J + // K L M N O + PASS, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + // P Q R S T U V W X Y Z + // [ \ ] ^ _ + UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + UNESCAPE, UNESCAPE, PASS, ESCAPE, PASS, ESCAPE, UNESCAPE, + // ` a b c d e f g h i j + // k l m n o + ESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + // p q r s t u v w x y z + // { | } ~ + UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, UNESCAPE, + UNESCAPE, UNESCAPE, ESCAPE, ESCAPE, ESCAPE, UNESCAPE, ESCAPE, + // ...all the high-bit characters are escaped + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, + ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE, ESCAPE}; + +enum DotDisposition { + // The given dot is just part of a filename and is not special. + NOT_A_DIRECTORY, + + // The given dot is the current directory. + DIRECTORY_CUR, + + // The given dot is the first of a double dot that should take us up one. + DIRECTORY_UP +}; + +// When the path resolver finds a dot, this function is called with the +// character following that dot to see what it is. The return value +// indicates what type this dot is (see above). This code handles the case +// where the dot is at the end of the input. +// +// |*consumed_len| will contain the number of characters in the input that +// express what we found. +// +// If the input is "../foo", |after_dot| = 1, |end| = 6, and +// at the end, |*consumed_len| = 2 for the "./" this function consumed. The +// original dot length should be handled by the caller. +template +DotDisposition ClassifyAfterDot(const CHAR* spec, int after_dot, int end, int* consumed_len) { + if (after_dot == end) { + // Single dot at the end. + *consumed_len = 0; + return DIRECTORY_CUR; + } + if (IsURLSlash(spec[after_dot])) { + // Single dot followed by a slash. + *consumed_len = 1; // Consume the slash + return DIRECTORY_CUR; + } + + int second_dot_len = IsDot(spec, after_dot, end); + if (second_dot_len) { + int after_second_dot = after_dot + second_dot_len; + if (after_second_dot == end) { + // Double dot at the end. + *consumed_len = second_dot_len; + return DIRECTORY_UP; + } + if (IsURLSlash(spec[after_second_dot])) { + // Double dot followed by a slash. + *consumed_len = second_dot_len + 1; + return DIRECTORY_UP; + } + } + + // The dots are followed by something else, not a directory. + *consumed_len = 0; + return NOT_A_DIRECTORY; +} + +// Rewinds the output to the previous slash. It is assumed that the output +// ends with a slash and this doesn't count (we call this when we are +// appending directory paths, so the previous path component has and ending +// slash). +// +// This will stop at the first slash (assumed to be at position +// |path_begin_in_output| and not go any higher than that. Some web pages +// do ".." too many times, so we need to handle that brokenness. +// +// It searches for a literal slash rather than including a backslash as well +// because it is run only on the canonical output. +// +// The output is guaranteed to end in a slash when this function completes. +void BackUpToPreviousSlash(int path_begin_in_output, CanonOutput* output) { + DCHECK(output->length() > 0); + + int i = output->length() - 1; + DCHECK(output->at(i) == '/'); + if (i == path_begin_in_output) + return; // We're at the first slash, nothing to do. + + // Now back up (skipping the trailing slash) until we find another slash. + i--; + while (output->at(i) != '/' && i > path_begin_in_output) + i--; + + // Now shrink the output to just include that last slash we found. + output->set_length(i + 1); +} + +// Looks for problematic nested escape sequences and escapes the output as +// needed to ensure they can't be misinterpreted. +// +// Our concern is that in input escape sequence that's invalid because it +// contains nested escape sequences might look valid once those are unescaped. +// For example, "%%300" is not a valid escape sequence, but after unescaping the +// inner "%30" this becomes "%00" which is valid. Leaving this in the output +// string can result in callers re-canonicalizing the string and unescaping this +// sequence, thus resulting in something fundamentally different than the +// original input here. This can cause a variety of problems. +// +// This function is called after we've just unescaped a sequence that's within +// two output characters of a previous '%' that we know didn't begin a valid +// escape sequence in the input string. We look for whether the output is going +// to turn into a valid escape sequence, and if so, convert the initial '%' into +// an escaped "%25" so the output can't be misinterpreted. +// +// |spec| is the input string we're canonicalizing. +// |next_input_index| is the index of the next unprocessed character in |spec|. +// |input_len| is the length of |spec|. +// |last_invalid_percent_index| is the index in |output| of a previously-seen +// '%' character. The caller knows this '%' character isn't followed by a valid +// escape sequence in the input string. +// |output| is the canonicalized output thus far. The caller guarantees this +// ends with a '%' followed by one or two characters, and the '%' is the one +// pointed to by |last_invalid_percent_index|. The last character in the string +// was just unescaped. +template +void CheckForNestedEscapes(const CHAR* spec, int next_input_index, int input_len, + int last_invalid_percent_index, CanonOutput* output) { + const int length = output->length(); + const char last_unescaped_char = output->at(length - 1); + + // If |output| currently looks like "%c", we need to try appending the next + // input character to see if this will result in a problematic escape + // sequence. Note that this won't trigger on the first nested escape of a + // two-escape sequence like "%%30%30" -- we'll allow the conversion to + // "%0%30" -- but the second nested escape will be caught by this function + // when it's called again in that case. + const bool append_next_char = last_invalid_percent_index == length - 2; + if (append_next_char) { + // If the input doesn't contain a 7-bit character next, this case won't be a + // problem. + if ((next_input_index == input_len) || (spec[next_input_index] >= 0x80)) + return; + output->push_back(static_cast(spec[next_input_index])); + } + + // Now output ends like "%cc". Try to unescape this. + int begin = last_invalid_percent_index; + unsigned char temp; + if (DecodeEscaped(output->data(), &begin, output->length(), &temp)) { + // New escape sequence found. Overwrite the characters following the '%' + // with "25", and push_back() the one or two characters that were following + // the '%' when we were called. + if (!append_next_char) + output->push_back(output->at(last_invalid_percent_index + 1)); + output->set(last_invalid_percent_index + 1, '2'); + output->set(last_invalid_percent_index + 2, '5'); + output->push_back(last_unescaped_char); + } else if (append_next_char) { + // Not a valid escape sequence, but we still need to undo appending the next + // source character so the caller can process it normally. + output->set_length(length); + } +} + +// Appends the given path to the output. It assumes that if the input path +// starts with a slash, it should be copied to the output. If no path has +// already been appended to the output (the case when not resolving +// relative URLs), the path should begin with a slash. +// +// If there are already path components (this mode is used when appending +// relative paths for resolving), it assumes that the output already has +// a trailing slash and that if the input begins with a slash, it should be +// copied to the output. +// +// We do not collapse multiple slashes in a row to a single slash. It seems +// no web browsers do this, and we don't want incompatibilities, even though +// it would be correct for most systems. +template +bool DoPartialPath(const CHAR* spec, const Component& path, int path_begin_in_output, + CanonOutput* output) { + int end = path.end(); + + // We use this variable to minimize the amount of work done when unescaping -- + // we'll only call CheckForNestedEscapes() when this points at one of the last + // couple of characters in |output|. + int last_invalid_percent_index = INT_MIN; + + bool success = true; + for (int i = path.begin; i < end; i++) { + UCHAR uch = static_cast(spec[i]); + if (sizeof(CHAR) > 1 && uch >= 0x80) { + // We only need to test wide input for having non-ASCII characters. For + // narrow input, we'll always just use the lookup table. We don't try to + // do anything tricky with decoding/validating UTF-8. This function will + // read one or two UTF-16 characters and append the output as UTF-8. This + // call will be removed in 8-bit mode. + success &= AppendUTF8EscapedChar(spec, &i, end, output); + } else { + // Normal ASCII character or 8-bit input, use the lookup table. + unsigned char out_ch = static_cast(uch); + unsigned char flags = kPathCharLookup[out_ch]; + if (flags & SPECIAL) { + // Needs special handling of some sort. + int dotlen; + if ((dotlen = IsDot(spec, i, end)) > 0) { + // See if this dot was preceded by a slash in the output. We + // assume that when canonicalizing paths, they will always + // start with a slash and not a dot, so we don't have to + // bounds check the output. + // + // Note that we check this in the case of dots so we don't have to + // special case slashes. Since slashes are much more common than + // dots, this actually increases performance measurably (though + // slightly). + DCHECK(output->length() > path_begin_in_output); + if (output->length() > path_begin_in_output && output->at(output->length() - 1) == '/') { + // Slash followed by a dot, check to see if this is means relative + int consumed_len; + switch (ClassifyAfterDot(spec, i + dotlen, end, &consumed_len)) { + case NOT_A_DIRECTORY: + // Copy the dot to the output, it means nothing special. + output->push_back('.'); + i += dotlen - 1; + break; + case DIRECTORY_CUR: // Current directory, just skip the input. + i += dotlen + consumed_len - 1; + break; + case DIRECTORY_UP: + BackUpToPreviousSlash(path_begin_in_output, output); + i += dotlen + consumed_len - 1; + break; + } + } else { + // This dot is not preceded by a slash, it is just part of some + // file name. + output->push_back('.'); + i += dotlen - 1; + } + + } else if (out_ch == '\\') { + // Convert backslashes to forward slashes + output->push_back('/'); + + } else if (out_ch == '%') { + // Handle escape sequences. + unsigned char unescaped_value; + if (DecodeEscaped(spec, &i, end, &unescaped_value)) { + // Valid escape sequence, see if we keep, reject, or unescape it. + // Note that at this point DecodeEscape() will have advanced |i| to + // the last character of the escape sequence. + char unescaped_flags = kPathCharLookup[unescaped_value]; + + if (unescaped_flags & UNESCAPE) { + // This escaped value shouldn't be escaped. Try to copy it. + output->push_back(unescaped_value); + // If we just unescaped a value within 2 output characters of the + // '%' from a previously-detected invalid escape sequence, we + // might have an input string with problematic nested escape + // sequences; detect and fix them. + if (last_invalid_percent_index >= (output->length() - 3)) { + CheckForNestedEscapes(spec, i + 1, end, last_invalid_percent_index, output); + } + } else { + // Either this is an invalid escaped character, or it's a valid + // escaped character we should keep escaped. In the first case we + // should just copy it exactly and remember the error. In the + // second we also copy exactly in case the server is sensitive to + // changing the case of any hex letters. + output->push_back('%'); + output->push_back(static_cast(spec[i - 1])); + output->push_back(static_cast(spec[i])); + if (unescaped_flags & INVALID_BIT) + success = false; + } + } else { + // Invalid escape sequence. IE7+ rejects any URLs with such + // sequences, while other browsers pass them through unchanged. We + // use the permissive behavior. + // TODO(brettw): Consider testing IE's strict behavior, which would + // allow removing the code to handle nested escapes above. + last_invalid_percent_index = output->length(); + output->push_back('%'); + } + + } else if (flags & INVALID_BIT) { + // For NULLs, etc. fail. + AppendEscapedChar(out_ch, output); + success = false; + + } else if (flags & ESCAPE_BIT) { + // This character should be escaped. + AppendEscapedChar(out_ch, output); + } + } else { + // Nothing special about this character, just append it. + output->push_back(out_ch); + } + } + } + return success; +} + +template +bool DoPath(const CHAR* spec, const Component& path, CanonOutput* output, Component* out_path) { + bool success = true; + out_path->begin = output->length(); + if (path.len > 0) { + // Write out an initial slash if the input has none. If we just parse a URL + // and then canonicalize it, it will of course have a slash already. This + // check is for the replacement and relative URL resolving cases of file + // URLs. + if (!IsURLSlash(spec[path.begin])) + output->push_back('/'); + + success = DoPartialPath(spec, path, out_path->begin, output); + } else { + // No input, canonical path is a slash. + output->push_back('/'); + } + out_path->len = output->length() - out_path->begin; + return success; +} + +} // namespace + +bool CanonicalizePath(const char* spec, const Component& path, CanonOutput* output, + Component* out_path) { + return DoPath(spec, path, output, out_path); +} + +} // namespace url + diff --git a/source/common/chromium_url/url_canon_stdstring.cc b/source/common/chromium_url/url_canon_stdstring.cc new file mode 100644 index 0000000000000..dc501d66ec26b --- /dev/null +++ b/source/common/chromium_url/url_canon_stdstring.cc @@ -0,0 +1,33 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "common/chromium_url/url_canon_stdstring.h" + +namespace url { + +StdStringCanonOutput::StdStringCanonOutput(std::string* str) : CanonOutput(), str_(str) { + cur_len_ = static_cast(str_->size()); // Append to existing data. + buffer_ = str_->empty() ? NULL : &(*str_)[0]; + buffer_len_ = static_cast(str_->size()); +} + +StdStringCanonOutput::~StdStringCanonOutput() { + // Nothing to do, we don't own the string. +} + +void StdStringCanonOutput::Complete() { + str_->resize(cur_len_); + buffer_len_ = cur_len_; +} + +void StdStringCanonOutput::Resize(int sz) { + str_->resize(sz); + buffer_ = str_->empty() ? NULL : &(*str_)[0]; + buffer_len_ = sz; +} + +} // namespace url diff --git a/source/common/chromium_url/url_canon_stdstring.h b/source/common/chromium_url/url_canon_stdstring.h new file mode 100644 index 0000000000000..d6a2a0d6fc707 --- /dev/null +++ b/source/common/chromium_url/url_canon_stdstring.h @@ -0,0 +1,59 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef URL_URL_CANON_STDSTRING_H_ +#define URL_URL_CANON_STDSTRING_H_ + +// This header file defines a canonicalizer output method class for STL +// strings. Because the canonicalizer tries not to be dependent on the STL, +// we have segregated it here. + +#include + +#include "common/chromium_url/envoy_shim.h" +#include "common/chromium_url/url_canon.h" + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete + +namespace url { + +// Write into a std::string given in the constructor. This object does not own +// the string itself, and the user must ensure that the string stays alive +// throughout the lifetime of this object. +// +// The given string will be appended to; any existing data in the string will +// be preserved. +// +// Note that when canonicalization is complete, the string will likely have +// unused space at the end because we make the string very big to start out +// with (by |initial_size|). This ends up being important because resize +// operations are slow, and because the base class needs to write directly +// into the buffer. +// +// Therefore, the user should call Complete() before using the string that +// this class wrote into. +class COMPONENT_EXPORT(URL) StdStringCanonOutput : public CanonOutput { +public: + StdStringCanonOutput(std::string* str); + ~StdStringCanonOutput() override; + + // Must be called after writing has completed but before the string is used. + void Complete(); + + void Resize(int sz) override; + +protected: + std::string* str_; + DISALLOW_COPY_AND_ASSIGN(StdStringCanonOutput); +}; + +} // namespace url + +#endif // URL_URL_CANON_STDSTRING_H_ + diff --git a/source/common/chromium_url/url_parse.h b/source/common/chromium_url/url_parse.h new file mode 100644 index 0000000000000..942eba48d12ac --- /dev/null +++ b/source/common/chromium_url/url_parse.h @@ -0,0 +1,50 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef URL_PARSE_H_ +#define URL_PARSE_H_ + +namespace url { + +// Component ------------------------------------------------------------------ + +// Represents a substring for URL parsing. +struct Component { + Component() : begin(0), len(-1) {} + + // Normal constructor: takes an offset and a length. + Component(int b, int l) : begin(b), len(l) {} + + int end() const { return begin + len; } + + // Returns true if this component is valid, meaning the length is given. Even + // valid components may be empty to record the fact that they exist. + bool is_valid() const { return (len != -1); } + + // Returns true if the given component is specified on false, the component + // is either empty or invalid. + bool is_nonempty() const { return (len > 0); } + + void reset() { + begin = 0; + len = -1; + } + + bool operator==(const Component& other) const { return begin == other.begin && len == other.len; } + + int begin; // Byte offset in the string of this component. + int len; // Will be -1 if the component is unspecified. +}; + +// Helper that returns a component created with the given begin and ending +// points. The ending point is non-inclusive. +inline Component MakeRange(int begin, int end) { return Component(begin, end - begin); } + +} // namespace url + +#endif // URL_PARSE_H_ + diff --git a/source/common/chromium_url/url_parse_internal.h b/source/common/chromium_url/url_parse_internal.h new file mode 100644 index 0000000000000..a8c15819048be --- /dev/null +++ b/source/common/chromium_url/url_parse_internal.h @@ -0,0 +1,18 @@ +// Envoy snapshot of Chromium URL path normalization, see README.md. +// NOLINT(namespace-envoy) + +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef URL_URL_PARSE_INTERNAL_H_ +#define URL_URL_PARSE_INTERNAL_H_ + +namespace url { + +// We treat slashes and backslashes the same for IE compatibility. +inline bool IsURLSlash(char ch) { return ch == '/' || ch == '\\'; } + +} // namespace url + +#endif // URL_URL_PARSE_INTERNAL_H_ diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 900442c49b979..75e06508fe29e 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -74,6 +74,7 @@ envoy_cc_library( hdrs = ["codes.h"], deps = [ ":headers_lib", + ":path_utility_lib", ":utility_lib", "//include/envoy/http:codes_interface", "//include/envoy/http:header_map_interface", @@ -284,3 +285,15 @@ envoy_cc_library( "@envoy_api//envoy/type:range_cc", ], ) + +envoy_cc_library( + name = "path_utility_lib", + srcs = ["path_utility.cc"], + hdrs = ["path_utility.h"], + external_deps = ["abseil_optional"], + deps = [ + "//include/envoy/http:header_map_interface", + "//source/common/chromium_url", + "//source/common/common:logger_lib", + ], +) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 3c5a9d56c8a38..857b32f8eca23 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -28,9 +28,12 @@ #include "common/http/headers.h" #include "common/http/http1/codec_impl.h" #include "common/http/http2/codec_impl.h" +#include "common/http/path_utility.h" #include "common/http/utility.h" #include "common/network/utility.h" +#include "absl/strings/escaping.h" + namespace Envoy { namespace Http { @@ -567,6 +570,13 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers, return; } + // Path sanitization should happen before any path access other than the above sanity check. + if (!ConnectionManagerUtility::maybeNormalizePath(*request_headers_)) { + sendLocalReply(Grpc::Common::hasGrpcContentType(*request_headers_), Code::BadRequest, "", + nullptr); + return; + } + if (protocol == Protocol::Http11 && request_headers_->Connection() && 0 == StringUtil::caseInsensitiveCompare(request_headers_->Connection()->value().c_str(), diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index ebe58b59d1dbe..8df31fb2e204d 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -8,6 +8,7 @@ #include "common/common/empty_string.h" #include "common/common/utility.h" #include "common/http/headers.h" +#include "common/http/path_utility.h" #include "common/http/utility.h" #include "common/network/utility.h" #include "common/runtime/uuid_util.h" @@ -328,5 +329,11 @@ void ConnectionManagerUtility::mutateResponseHeaders(Http::HeaderMap& response_h } } +/* static */ +bool ConnectionManagerUtility::maybeNormalizePath(HeaderMap& request_headers) { + ASSERT(request_headers.Path()); + return PathUtil::canonicalPath(*request_headers.Path()); +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/conn_manager_utility.h b/source/common/http/conn_manager_utility.h index c27886cda9936..89f7f52146578 100644 --- a/source/common/http/conn_manager_utility.h +++ b/source/common/http/conn_manager_utility.h @@ -36,6 +36,11 @@ class ConnectionManagerUtility { static void mutateResponseHeaders(Http::HeaderMap& response_headers, const Http::HeaderMap* request_headers, const std::string& via); + // Sanitize the path in the header map if forced by config. + // Side affect: the string view of Path header is invalidated. + // Return false if error happens during the sanitization. + static bool maybeNormalizePath(HeaderMap& request_headers); + private: /** * Mutate request headers if request needs to be traced. diff --git a/source/common/http/header_map_impl.cc b/source/common/http/header_map_impl.cc index 28728515ef12e..05df831e58d26 100644 --- a/source/common/http/header_map_impl.cc +++ b/source/common/http/header_map_impl.cc @@ -64,6 +64,8 @@ void HeaderString::freeDynamic() { } } +bool HeaderString::valid() const { return validHeaderString(getStringView()); } + void HeaderString::append(const char* data, uint32_t size) { switch (type_) { case Type::Reference: { @@ -318,6 +320,8 @@ bool HeaderMapImpl::operator==(const HeaderMapImpl& rhs) const { return true; } +bool HeaderMapImpl::operator!=(const HeaderMapImpl& rhs) const { return !operator==(rhs); } + void HeaderMapImpl::insertByKey(HeaderString&& key, HeaderString&& value) { StaticLookupEntry::EntryCb cb = ConstSingleton::get().find(key.c_str()); if (cb) { diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h index fd2ae9570e7e1..8a4dab481af36 100644 --- a/source/common/http/header_map_impl.h +++ b/source/common/http/header_map_impl.h @@ -61,6 +61,7 @@ class HeaderMapImpl : public HeaderMap, NonCopyable { * comparison (order matters). */ bool operator==(const HeaderMapImpl& rhs) const; + bool operator!=(const HeaderMapImpl& rhs) const; // Http::HeaderMap void addReference(const LowerCaseString& key, const std::string& value) override; diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index a08878e609d1c..2d9cd394ce8aa 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -396,7 +396,13 @@ void ConnectionImpl::onHeaderValue(const char* data, size_t length) { // Ignore trailers. return; } - + // http-parser should filter for this + // (https://tools.ietf.org/html/rfc7230#section-3.2.6), but it doesn't today. HeaderStrings + // have an invariant that they must not contain embedded zero characters + // (NUL, ASCII 0x0). + if (absl::string_view(data, length).find('\0') != absl::string_view::npos) { + throw CodecProtocolException("http/1.1 protocol error: header value contains NUL"); + } header_parsing_state_ = HeaderParsingState::Value; current_header_value_.append(data, length); } diff --git a/source/common/http/path_utility.cc b/source/common/http/path_utility.cc new file mode 100644 index 0000000000000..796c2c1cbd52b --- /dev/null +++ b/source/common/http/path_utility.cc @@ -0,0 +1,55 @@ +#include "common/http/path_utility.h" + +#include "common/chromium_url/url_canon.h" +#include "common/chromium_url/url_canon_stdstring.h" +#include "common/common/logger.h" + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace Envoy { +namespace Http { + +namespace { +absl::optional canonicalizePath(absl::string_view original_path) { + std::string canonical_path; + url::Component in_component(0, original_path.size()); + url::Component out_component; + url::StdStringCanonOutput output(&canonical_path); + if (!url::CanonicalizePath(original_path.data(), in_component, &output, &out_component)) { + return absl::nullopt; + } else { + output.Complete(); + return absl::make_optional(std::move(canonical_path)); + } +} +} // namespace + +/* static */ +bool PathUtil::canonicalPath(HeaderEntry& path_header) { + const auto original_path = path_header.value().getStringView(); + // canonicalPath is supposed to apply on path component in URL instead of :path header + const auto query_pos = original_path.find('?'); + auto normalized_path_opt = canonicalizePath( + query_pos == original_path.npos + ? original_path + : absl::string_view(original_path.data(), query_pos) // '?' is not included + ); + + if (!normalized_path_opt.has_value()) { + return false; + } + auto& normalized_path = normalized_path_opt.value(); + const absl::string_view query_suffix = + query_pos == original_path.npos + ? absl::string_view{} + : absl::string_view{original_path.data() + query_pos, original_path.size() - query_pos}; + if (query_suffix.size() > 0) { + normalized_path.insert(normalized_path.end(), query_suffix.begin(), query_suffix.end()); + } + path_header.value(std::move(normalized_path)); + return true; +} + +} // namespace Http +} // namespace Envoy diff --git a/source/common/http/path_utility.h b/source/common/http/path_utility.h new file mode 100644 index 0000000000000..ad0d32c3ff7d6 --- /dev/null +++ b/source/common/http/path_utility.h @@ -0,0 +1,19 @@ +#pragma once + +#include "envoy/http/header_map.h" + +namespace Envoy { +namespace Http { + +/** + * Path helper extracted from chromium project. + */ +class PathUtil { +public: + // Returns if the normalization succeeds. + // If it is successful, the param will be updated with the normalized path. + static bool canonicalPath(HeaderEntry& path_header); +}; + +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 13c84e925b469..de75fa7a835fd 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -270,3 +270,12 @@ envoy_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_cc_test( + name = "path_utility_test", + srcs = ["path_utility_test.cc"], + deps = [ + "//source/common/http:header_map_lib", + "//source/common/http:path_utility_lib", + ], +) diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 094d3b04fade8..934d498965ff4 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -127,7 +127,7 @@ class FuzzConfig : public ConnectionManagerConfig { Network::Address::Ipv4Instance local_address_{"127.0.0.1"}; absl::optional user_agent_; TracingConnectionManagerConfigPtr tracing_config_; - bool proxy_100_continue_ = true; + bool proxy_100_continue_{true}; Http::Http1Settings http1_settings_; }; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 9ff17fc5ad20f..0d3876ef6701b 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -546,6 +546,112 @@ TEST_F(HttpConnectionManagerImplTest, InvalidPathWithDualFilter) { conn_manager_->onData(fake_input, false); } +// Invalid paths are rejected with 400. +TEST_F(HttpConnectionManagerImplTest, PathFailedtoSanitize) { + InSequence s; + setup(false, ""); + // Enable path sanitizer + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, + {":path", "/ab%00c"}, // "%00" is not valid in path according to RFC + {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), true); + data.drain(4); + })); + + // This test also verifies that decoder/encoder filters have onDestroy() called only once. + MockStreamFilter* filter = new MockStreamFilter(); + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(StreamFilterSharedPtr{filter}); + })); + EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)); + EXPECT_CALL(*filter, setEncoderFilterCallbacks(_)); + + EXPECT_CALL(*filter, encodeHeaders(_, true)); + EXPECT_CALL(response_encoder_, encodeHeaders(_, true)) + .WillOnce(Invoke([](const HeaderMap& headers, bool) -> void { + EXPECT_STREQ("400", headers.Status()->value().c_str()); + })); + EXPECT_CALL(*filter, onDestroy()); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + +// Filters observe normalized paths, not the original path, when path +// normalization is configured. +TEST_F(HttpConnectionManagerImplTest, FilterShouldUseSantizedPath) { + setup(false, ""); + // Enable path sanitizer + const std::string original_path = "/x/%2E%2e/z"; + const std::string normalized_path = "/z"; + + MockStreamFilter* filter = new MockStreamFilter(); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter(StreamDecoderFilterSharedPtr{filter}); + })); + + EXPECT_CALL(*filter, decodeHeaders(_, true)) + .WillRepeatedly(Invoke([&](HeaderMap& header_map, bool) -> FilterHeadersStatus { + EXPECT_EQ(normalized_path, header_map.Path()->value().c_str()); + return FilterHeadersStatus::StopIteration; + })); + + EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{new TestHeaderMapImpl{ + {":authority", "host"}, {":path", original_path}, {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), true); + })); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + +// The router observes normalized paths, not the original path, when path +// normalization is configured. +TEST_F(HttpConnectionManagerImplTest, RouteShouldUseSantizedPath) { + setup(false, ""); + // Enable path sanitizer + const std::string original_path = "/x/%2E%2e/z"; + const std::string normalized_path = "/z"; + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{new TestHeaderMapImpl{ + {":authority", "host"}, {":path", original_path}, {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), true); + })); + + const std::string fake_cluster_name = "fake_cluster"; + + std::shared_ptr fake_cluster = + std::make_shared>(); + std::shared_ptr route = std::make_shared>(); + EXPECT_CALL(route->route_entry_, clusterName()).WillRepeatedly(ReturnRef(fake_cluster_name)); + + EXPECT_CALL(*route_config_provider_.route_config_, route(_, _)) + .WillOnce(Invoke([&](const Http::HeaderMap& header_map, uint64_t) { + EXPECT_EQ(normalized_path, header_map.Path()->value().c_str()); + return route; + })); + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillOnce(Invoke([&](FilterChainFactoryCallbacks&) -> void {})); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlow) { setup(false, ""); @@ -3243,6 +3349,5 @@ TEST(HttpConnectionManagerTracingStatsTest, verifyTracingStats) { ConnectionManagerImpl::chargeTracingStats(Tracing::Reason::NotTraceableRequestId, tracing_stats); EXPECT_EQ(1UL, tracing_stats.not_traceable_.value()); } - } // namespace Http } // namespace Envoy diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc index 7d8bb9413ece3..c58c743f1db30 100644 --- a/test/common/http/header_map_impl_test.cc +++ b/test/common/http/header_map_impl_test.cc @@ -864,7 +864,9 @@ TEST(HeaderMapImplTest, TestHeaderMapImplyCopy) { TestHeaderMapImpl baz{{"foo", "baz"}}; baz = *headers; EXPECT_STREQ("bar", baz.get(LowerCaseString("foo"))->value().c_str()); - baz = baz; + TestHeaderMapImpl tac{baz}; + EXPECT_STREQ("bar", tac.get(LowerCaseString("foo"))->value().c_str()); + baz = tac; EXPECT_STREQ("bar", baz.get(LowerCaseString("foo"))->value().c_str()); } diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 0755246b62352..69d361691f93f 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -13,6 +13,7 @@ #include "test/mocks/network/mocks.h" #include "test/test_common/printers.h" +#include "absl/strings/str_cat.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -282,6 +283,67 @@ TEST_F(Http1ServerConnectionImplTest, HostHeaderTranslation) { EXPECT_EQ(0U, buffer.length()); } +// Regression test for http-parser allowing embedded NULs in header values, +// verify we reject them. +TEST_F(Http1ServerConnectionImplTest, HeaderEmbeddedNulRejection) { + initialize(); + + InSequence sequence; + + Http::MockStreamDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer( + absl::StrCat("GET / HTTP/1.1\r\nHOST: h.com\r\nfoo: bar", std::string(1, '\0'), "baz\r\n")); + EXPECT_THROW_WITH_MESSAGE(codec_->dispatch(buffer), CodecProtocolException, + "http/1.1 protocol error: header value contains NUL"); +} + +// Mutate an HTTP GET with embedded NULs, this should always be rejected in some +// way (not necessarily with "head value contains NUL" though). +TEST_F(Http1ServerConnectionImplTest, HeaderMutateEmbeddedNul) { + const std::string example_input = "GET / HTTP/1.1\r\nHOST: h.com\r\nfoo: barbaz\r\n"; + + for (size_t n = 1; n < example_input.size(); ++n) { + initialize(); + + InSequence sequence; + + Http::MockStreamDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer( + absl::StrCat(example_input.substr(0, n), std::string(1, '\0'), example_input.substr(n))); + EXPECT_THROW_WITH_REGEX(codec_->dispatch(buffer), CodecProtocolException, + "http/1.1 protocol error:"); + } +} + +// Mutate an HTTP GET with CR or LF. These can cause an exception or maybe +// result in a valid decodeHeaders(). In any case, the validHeaderString() +// ASSERTs should validate we never have any embedded CR or LF. +TEST_F(Http1ServerConnectionImplTest, HeaderMutateEmbeddedCRLF) { + const std::string example_input = "GET / HTTP/1.1\r\nHOST: h.com\r\nfoo: barbaz\r\n"; + + for (const char c : {'\r', '\n'}) { + for (size_t n = 1; n < example_input.size(); ++n) { + initialize(); + + InSequence sequence; + + NiceMock decoder; + EXPECT_CALL(callbacks_, newStream(_)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer( + absl::StrCat(example_input.substr(0, n), std::string(1, c), example_input.substr(n))); + try { + codec_->dispatch(buffer); + } catch (CodecProtocolException) { + } + } + } +} + TEST_F(Http1ServerConnectionImplTest, CloseDuringHeadersComplete) { initialize(); diff --git a/test/common/http/path_utility_test.cc b/test/common/http/path_utility_test.cc new file mode 100644 index 0000000000000..2cc299465add0 --- /dev/null +++ b/test/common/http/path_utility_test.cc @@ -0,0 +1,89 @@ +#include +#include + +#include "common/http/header_map_impl.h" +#include "common/http/path_utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Http { + +class PathUtilityTest : public testing::Test { +public: + // This is an indirect way to build a header entry for + // PathUtil::canonicalPath(), since we don't have direct access to the + // HeaderMapImpl constructor. + HeaderEntry& pathHeaderEntry(const std::string& path_value) { + headers_.insertPath().value(path_value); + return *headers_.Path(); + } + HeaderMapImpl headers_; +}; + +// Already normalized path don't change. +TEST_F(PathUtilityTest, AlreadyNormalPaths) { + const std::vector normal_paths{"/xyz", "/x/y/z"}; + for (const auto& path : normal_paths) { + auto& path_header = pathHeaderEntry(path); + const auto result = PathUtil::canonicalPath(path_header); + EXPECT_TRUE(result) << "original path: " << path; + EXPECT_EQ(path_header.value().getStringView(), absl::string_view(path)); + } +} + +// Invalid paths are rejected. +TEST_F(PathUtilityTest, InvalidPaths) { + const std::vector invalid_paths{"/xyz/.%00../abc", "/xyz/%00.%00./abc", + "/xyz/AAAAA%%0000/abc"}; + for (const auto& path : invalid_paths) { + auto& path_header = pathHeaderEntry(path); + EXPECT_FALSE(PathUtil::canonicalPath(path_header)) << "original path: " << path; + } +} + +// Paths that are valid get normalized. +TEST_F(PathUtilityTest, NormalizeValidPaths) { + const std::vector> non_normal_pairs{ + {"/a/b/../c", "/a/c"}, // parent dir + {"/a/b/./c", "/a/b/c"}, // current dir + {"a/b/../c", "/a/c"}, // non / start + {"/a/b/../../../../c", "/c"}, // out number parent + {"/a/..\\c", "/c"}, // "..\\" canonicalization + {"/%c0%af", "/%c0%af"}, // 2 bytes unicode reserved characters + {"/%5c%25", "/%5c%25"}, // reserved characters + {"/a/b/%2E%2E/c", "/a/c"} // %2E escape + }; + + for (const auto& path_pair : non_normal_pairs) { + auto& path_header = pathHeaderEntry(path_pair.first); + const auto result = PathUtil::canonicalPath(path_header); + EXPECT_TRUE(result) << "original path: " << path_pair.first; + EXPECT_EQ(path_header.value().getStringView(), path_pair.second) + << "original path: " << path_pair.second; + } +} + +// Paths that are valid get normalized. +TEST_F(PathUtilityTest, NormalizeCasePath) { + const std::vector> non_normal_pairs{ + {"/A/B/C", "/A/B/C"}, // not normalize to lower case + {"/a/b/%2E%2E/c", "/a/c"}, // %2E can be normalized to . + {"/a/b/%2e%2e/c", "/a/c"}, // %2e can be normalized to . + {"/a/%2F%2f/c", "/a/%2F%2f/c"}, // %2F is not normalized to %2f + }; + + for (const auto& path_pair : non_normal_pairs) { + auto& path_header = pathHeaderEntry(path_pair.first); + const auto result = PathUtil::canonicalPath(path_header); + EXPECT_TRUE(result) << "original path: " << path_pair.first; + EXPECT_EQ(path_header.value().getStringView(), path_pair.second) + << "original path: " << path_pair.second; + } +} +// These test cases are explicitly not covered above: +// "/../c\r\n\" '\n' '\r' should be excluded by http parser +// "/a/\0c", '\0' should be excluded by http parser + +} // namespace Http +} // namespace Envoy diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc index 170aa143f790d..6551bb248789e 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_test.cc @@ -204,7 +204,7 @@ TEST_F(GzipFilterTest, isAcceptEncodingAllowed) { } { Http::TestHeaderMapImpl headers = { - {"accept-encoding", "\tdeflate\t, gzip\t ; q\t =\t 1.0,\t * ;q=0.5\n"}}; + {"accept-encoding", "\tdeflate\t, gzip\t ; q\t =\t 1.0,\t * ;q=0.5"}}; EXPECT_TRUE(isAcceptEncodingAllowed(headers)); EXPECT_EQ(3, stats_.counter("test.gzip.header_gzip").value()); } @@ -412,7 +412,7 @@ TEST_F(GzipFilterTest, isContentTypeAllowed) { EXPECT_TRUE(isContentTypeAllowed(headers)); } { - Http::TestHeaderMapImpl headers = {{"content-type", "\ttext/html\t\n"}}; + Http::TestHeaderMapImpl headers = {{"content-type", "\ttext/html\t"}}; EXPECT_TRUE(isContentTypeAllowed(headers)); } @@ -584,7 +584,7 @@ TEST_F(GzipFilterTest, isTransferEncodingAllowed) { EXPECT_FALSE(isTransferEncodingAllowed(headers)); } { - Http::TestHeaderMapImpl headers = {{"transfer-encoding", " gzip\t, chunked\t\n"}}; + Http::TestHeaderMapImpl headers = {{"transfer-encoding", " gzip\t, chunked\t"}}; EXPECT_FALSE(isTransferEncodingAllowed(headers)); } } diff --git a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc index 9e766f7bc4f53..2cca94f226b96 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc @@ -17,6 +17,18 @@ name: envoy.filters.http.rbac - any: true )EOF"; +const std::string RBAC_CONFIG_WITH_PREFIX_MATCH = R"EOF( +name: envoy.filters.http.rbac +config: + rules: + policies: + foo: + permissions: + - header: { name: ":path", prefix_match: "/foo" } + principals: + - any: true +)EOF"; + typedef HttpProtocolIntegrationTest RBACIntegrationTest; INSTANTIATE_TEST_CASE_P(Protocols, RBACIntegrationTest, @@ -66,6 +78,27 @@ TEST_P(RBACIntegrationTest, Denied) { EXPECT_STREQ("403", response->headers().Status()->value().c_str()); } +TEST_P(RBACIntegrationTest, RbacPrefixRuleUseNormalizePath) { + config_helper_.addFilter(RBAC_CONFIG_WITH_PREFIX_MATCH); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeRequestWithBody( + Http::TestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/foo/../bar"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "10.0.0.1"}, + }, + 1024); + + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_STREQ("403", response->headers().Status()->value().c_str()); +} + TEST_P(RBACIntegrationTest, RouteOverride) { config_helper_.addConfigModifier( [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& cfg) { diff --git a/test/integration/header_integration_test.cc b/test/integration/header_integration_test.cc index e25e7b79a4f62..dfa9e81a0cbc6 100644 --- a/test/integration/header_integration_test.cc +++ b/test/integration/header_integration_test.cc @@ -120,6 +120,24 @@ stat_prefix: header_test - header: key: "x-real-ip" value: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%" + - name: path-sanitization + domains: ["path-sanitization.com"] + routes: + - match: { prefix: "/private" } + route: + cluster: cluster_0 + request_headers_to_add: + - header: + key: "x-site" + value: "private" + - match: { prefix: "/public" } + route: + cluster: cluster_0 + request_headers_to_add: + - header: + key: "x-site" + value: "public" + )EOF"; } // namespace @@ -269,6 +287,8 @@ class HeaderIntegrationTest } } + // hcm.mutable_normalize_path()->set_value(normalize_path_); + if (append) { // The config specifies append by default: no modifications needed. return; @@ -386,6 +406,7 @@ class HeaderIntegrationTest } bool use_eds_{false}; + bool normalize_path_{false}; FakeHttpConnectionPtr eds_connection_; FakeStreamPtr eds_stream_; }; @@ -915,4 +936,33 @@ TEST_P(HeaderIntegrationTest, TestXFFParsing) { }); } +// Validates behavior when normalize path is on. +// Path to decide route and path to upstream are both +// the normalized. +TEST_P(HeaderIntegrationTest, TestPathAndRouteOnNormalizedPath) { + normalize_path_ = true; + initializeFilter(HeaderMode::Append, false); + performRequest( + Http::TestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/private/../public"}, + {":scheme", "http"}, + {":authority", "path-sanitization.com"}, + }, + Http::TestHeaderMapImpl{{":authority", "path-sanitization.com"}, + {":path", "/public"}, + {":method", "GET"}, + {"x-site", "public"}}, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"content-length", "0"}, + {":status", "200"}, + {"x-unmodified", "response"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"x-unmodified", "response"}, + {":status", "200"}, + }); +} } // namespace Envoy