diff --git a/test/config/integration/BUILD b/test/config/integration/BUILD index 2f4916e77be19..21faf9456494d 100644 --- a/test/config/integration/BUILD +++ b/test/config/integration/BUILD @@ -10,6 +10,7 @@ envoy_package() exports_files([ "echo_server.json", "server.json", + "server.yaml", "server_ads.yaml", "server_cors_filter.json", "server_grpc_json_transcoder.json", @@ -19,6 +20,7 @@ exports_files([ "server_ssl.json", "server_uds.json", "server_unix_listener.json", + "server_unix_listener.yaml", "server_xfcc.json", "tcp_proxy.json", ]) @@ -37,8 +39,8 @@ filegroup( filegroup( name = "server_config_files", srcs = [ - "server.json", - "server_unix_listener.json", + "server.yaml", + "server_unix_listener.yaml", ], ) diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml new file mode 100644 index 0000000000000..4e68a14193bb2 --- /dev/null +++ b/test/config/integration/server.yaml @@ -0,0 +1,460 @@ +static_resources: + listeners: + - address: + socket_address: + address: "{{ ip_loopback_address }}" + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - require_ssl: all + routes: + - cluster: cluster_1 + prefix: "/" + domains: + - www.redirect.com + name: redirect + - routes: + - prefix: "/" + cluster: cluster_1 + runtime: + key: some_key + default: 0 + - prefix: "/test/long/url" + rate_limits: + - actions: + - type: destination_cluster + cluster: cluster_1 + - prefix: "/test/" + cluster: cluster_2 + - prefix: "/websocket/test" + prefix_rewrite: "/websocket" + cluster: cluster_1 + use_websocket: true + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + filters: + - type: both + name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - type: decoder + name: rate_limit + config: + domain: foo + - type: decoder + name: router + config: {} + access_log: + - format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' + path: "/dev/null" + filter: + filters: + - type: status_code + op: ">=" + value: 500 + - type: duration + op: ">=" + value: 1000000 + type: logical_or + - path: "/dev/null" + deprecated_v1: true + deprecated_v1: + type: read + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + filters: + - type: both + name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - name: rate_limit + config: + domain: foo + type: decoder + - type: decoder + name: router + config: {} + access_log: + - filter: + type: logical_or + filters: + - value: 500 + type: status_code + op: ">=" + - type: duration + op: ">=" + value: 1555500 + format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' + path: "/dev/null" + - path: "/dev/null" + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - routes: + - prefix: "/" + cluster: cluster_1 + domains: + - www.redirect.com + name: redirect + require_ssl: all + - routes: + - prefix: "/" + cluster: cluster_1 + domains: + - www.namewithport.com:1234 + name: redirect + require_ssl: all + - routes: + - cluster: cluster_1 + runtime: + key: some_key + default: 0 + prefix: "/" + - rate_limits: + - actions: + - type: destination_cluster + cluster: cluster_1 + prefix: "/test/long/url" + - prefix: "/test/" + cluster: cluster_2 + - cluster: cluster_1 + use_websocket: true + prefix: "/websocket/test" + prefix_rewrite: "/websocket" + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + http1_settings: + allow_absolute_url: true + deprecated_v1: true + deprecated_v1: + type: read + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + route_config: + virtual_hosts: + - routes: + - cluster: cluster_3 + prefix: "/test/long/url" + domains: + - "*" + name: integration + filters: + - name: router + config: {} + type: decoder + codec_type: http1 + stat_prefix: router + deprecated_v1: true + deprecated_v1: + type: read + per_connection_buffer_limit_bytes: 1024 + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + filters: + - type: both + name: http_dynamo_filter + config: {} + - name: router + config: {} + type: decoder + codec_type: http1 + stat_prefix: router + route_config: + virtual_hosts: + - routes: + - cluster: cluster_3 + prefix: "/dynamo/url" + domains: + - "*" + name: integration + deprecated_v1: true + deprecated_v1: + type: read + per_connection_buffer_limit_bytes: 1024 + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + route_config: + virtual_hosts: + - domains: + - "*" + name: integration + routes: + - prefix: "/test/long/url" + cluster: cluster_3 + filters: + - type: both + name: grpc_http1_bridge + config: {} + - type: decoder + name: router + config: {} + codec_type: http1 + stat_prefix: router + deprecated_v1: true + deprecated_v1: + type: read + per_connection_buffer_limit_bytes: 1024 + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - routes: + - cluster: cluster_1 + prefix: "/" + domains: + - www.redirect.com + name: redirect + require_ssl: all + - routes: + - cluster: cluster_1 + runtime: + key: some_key + default: 0 + prefix: "/" + - prefix: "/test/long/url" + rate_limits: + - actions: + - type: destination_cluster + cluster: cluster_1 + - prefix: "/test/" + cluster: cluster_2 + - prefix: "/websocket/test" + prefix_rewrite: "/websocket" + cluster: cluster_1 + use_websocket: true + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + filters: + - type: both + name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - name: rate_limit + config: + domain: foo + type: decoder + - name: buffer + config: + max_request_time_s: 120 + max_request_bytes: 5242880 + type: decoder + - config: {} + type: decoder + name: router + access_log: + - filter: + filters: + - op: ">=" + value: 500 + type: status_code + - type: duration + op: ">=" + value: 1555500 + type: logical_or + format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' + path: "/dev/null" + - path: "/dev/null" + deprecated_v1: true + deprecated_v1: + type: read + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + filters: + - type: decoder + name: router + config: {} + codec_type: http1 + stat_prefix: rds_dummy + rds: + route_config_name: foo + cluster: rds + deprecated_v1: true + deprecated_v1: + type: read + - address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.redis_proxy + config: + value: + conn_pool: + op_timeout_ms: 400 + stat_prefix: redis + cluster_name: redis + deprecated_v1: true + deprecated_v1: + type: read + clusters: + - name: cds + connect_timeout: 5s + hosts: + - socket_address: + address: {{ ip_loopback_address }} + port_value: 4 + dns_lookup_family: "{{ dns_lookup_family }}" + - name: rds + connect_timeout: 5s + hosts: + - socket_address: + address: {{ ip_loopback_address }} + port_value: 4 + dns_lookup_family: "{{ dns_lookup_family }}" + - name: lds + connect_timeout: 5s + hosts: + - socket_address: + address: {{ ip_loopback_address }} + port_value: 4 + dns_lookup_family: "{{ dns_lookup_family }}" + - name: cluster_1 + connect_timeout: 5s + hosts: + - socket_address: + address: {{ ip_loopback_address }} + port_value: {{ upstream_0 }} + dns_lookup_family: "{{ dns_lookup_family }}" + - name: cluster_2 + type: STRICT_DNS + connect_timeout: 5s + hosts: + - socket_address: + address: localhost + port_value: {{ upstream_1 }} + dns_lookup_family: "{{ dns_lookup_family }}" + - name: cluster_3 + connect_timeout: 5s + per_connection_buffer_limit_bytes: 1024 + hosts: + - socket_address: + address: {{ ip_loopback_address }} + port_value: {{ upstream_0 }} + dns_lookup_family: "{{ dns_lookup_family }}" + - name: statsd + type: STRICT_DNS + connect_timeout: 5s + hosts: + - socket_address: + address: localhost + port_value: 4 + dns_lookup_family: "{{ dns_lookup_family }}" + - name: redis + type: STRICT_DNS + connect_timeout: 5s + lb_policy: RING_HASH + hosts: + - socket_address: + address: localhost + port_value: 4 + dns_lookup_family: "{{ dns_lookup_family }}" + outlier_detection: {} +dynamic_resources: + lds_config: + api_config_source: + cluster_names: + - lds + refresh_delay: 30s + cds_config: + api_config_source: + cluster_names: + - cds + refresh_delay: 30s +cluster_manager: {} +flags_path: "/invalid_flags" +stats_sinks: +- name: envoy.statsd + config: + address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 8125 +- name: envoy.statsd + config: + tcp_cluster_name: statsd +watchdog: {} +runtime: + symlink_root: "{{ test_rundir }}/test/common/runtime/test_data/current" + subdirectory: envoy + override_subdirectory: envoy_override +admin: + access_log_path: "/dev/null" + profile_path: "{{ test_tmpdir }}/envoy.prof" + address: + socket_address: + address: {{ ip_loopback_address }} + port_value: 0 diff --git a/test/config/integration/server_unix_listener.yaml b/test/config/integration/server_unix_listener.yaml new file mode 100644 index 0000000000000..eb1b02f2de3d2 --- /dev/null +++ b/test/config/integration/server_unix_listener.yaml @@ -0,0 +1,44 @@ +static_resources: + listeners: + - address: + pipe: + path: "{{ socket_dir }}/unix-sockets.listener_0" + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + value: + filters: + - type: decoder + name: router + config: {} + codec_type: auto + stat_prefix: router + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - domains: + - "*" + name: vhost_0 + routes: + - prefix: "/" + cluster: cluster_0 + deprecated_v1: true + deprecated_v1: + type: read + clusters: + - name: cluster_0 + connect_timeout: 5s + hosts: + - socket_address: + address: "{{ ip_loopback_address }}" + port_value: 0 + dns_lookup_family: V4_ONLY +cluster_manager: {} +watchdog: {} +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: "{{ ip_loopback_address }}" + port_value: 0 diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index 09cbe85caec4d..e75825b2ca1c4 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -90,24 +90,26 @@ JSON_TEST_ARRAY=() # Parameterize IPv4 and IPv6 testing. if [[ -z "${ENVOY_IP_TEST_VERSIONS}" ]] || [[ "${ENVOY_IP_TEST_VERSIONS}" == "all" ]] \ || [[ "${ENVOY_IP_TEST_VERSIONS}" == "v4only" ]]; then - HOT_RESTART_JSON_V4="${TEST_TMPDIR}"/hot_restart_v4.json + HOT_RESTART_JSON_V4="${TEST_TMPDIR}"/hot_restart_v4.yaml echo building ${HOT_RESTART_JSON_V4} ... - cat "${TEST_RUNDIR}"/test/config/integration/server.json | + cat "${TEST_RUNDIR}"/test/config/integration/server.yaml | sed -e "s#{{ upstream_. }}#0#g" | \ sed -e "s#{{ test_rundir }}#$TEST_RUNDIR#" | \ + sed -e "s#{{ test_tmpdir }}#$TEST_TMPDIR#" | \ sed -e "s#{{ ip_loopback_address }}#127.0.0.1#" | \ - sed -e "s#{{ dns_lookup_family }}#v4_only#" | \ + sed -e "s#{{ dns_lookup_family }}#V4_ONLY#" | \ cat > "${HOT_RESTART_JSON_V4}" JSON_TEST_ARRAY+=("${HOT_RESTART_JSON_V4}") fi if [[ -z "${ENVOY_IP_TEST_VERSIONS}" ]] || [[ "${ENVOY_IP_TEST_VERSIONS}" == "all" ]] \ || [[ "${ENVOY_IP_TEST_VERSIONS}" == "v6only" ]]; then - HOT_RESTART_JSON_V6="${TEST_TMPDIR}"/hot_restart_v6.json - cat "${TEST_RUNDIR}"/test/config/integration/server.json | + HOT_RESTART_JSON_V6="${TEST_TMPDIR}"/hot_restart_v6.yaml + cat "${TEST_RUNDIR}"/test/config/integration/server.yaml | sed -e "s#{{ upstream_. }}#0#g" | \ sed -e "s#{{ test_rundir }}#$TEST_RUNDIR#" | \ - sed -e "s#{{ ip_loopback_address }}#[::1]#" | \ + sed -e "s#{{ test_tmpdir }}#$TEST_TMPDIR#" | \ + sed -e "s#{{ ip_loopback_address }}#::1#" | \ sed -e "s#{{ dns_lookup_family }}#v6_only#" | \ cat > "${HOT_RESTART_JSON_V6}" JSON_TEST_ARRAY+=("${HOT_RESTART_JSON_V6}") @@ -115,9 +117,9 @@ fi # Also test for listening on UNIX domain sockets. We use IPv4 for the # upstreams to avoid too much wild sedding. -HOT_RESTART_JSON_UDS="${TEST_TMPDIR}"/hot_restart_uds.json +HOT_RESTART_JSON_UDS="${TEST_TMPDIR}"/hot_restart_uds.yaml SOCKET_DIR="$(mktemp -d /tmp/envoy_test_hotrestart.XXXXXX)" -cat "${TEST_RUNDIR}"/test/config/integration/server_unix_listener.json | +cat "${TEST_RUNDIR}"/test/config/integration/server_unix_listener.yaml | sed -e "s#{{ socket_dir }}#${SOCKET_DIR}#" | \ sed -e "s#{{ ip_loopback_address }}#127.0.0.1#" | \ cat > "${HOT_RESTART_JSON_UDS}" @@ -150,9 +152,9 @@ do FIRST_SERVER_PID=$BACKGROUND_PID - start_test Updating original config json listener addresses + start_test Updating original config listener addresses sleep 3 - UPDATED_HOT_RESTART_JSON="${TEST_TMPDIR}"/hot_restart_updated."${TEST_INDEX}".json + UPDATED_HOT_RESTART_JSON="${TEST_TMPDIR}"/hot_restart_updated."${TEST_INDEX}".yaml "${TEST_RUNDIR}"/tools/socket_passing "-o" "${HOT_RESTART_JSON}" "-a" "${ADMIN_ADDRESS_PATH_0}" \ "-u" "${UPDATED_HOT_RESTART_JSON}" @@ -197,7 +199,7 @@ do sleep 7 start_test Checking that listener addresses have not changed - HOT_RESTART_JSON_1="${TEST_TMPDIR}"/hot_restart.1."${TEST_INDEX}".json + HOT_RESTART_JSON_1="${TEST_TMPDIR}"/hot_restart.1."${TEST_INDEX}".yaml "${TEST_RUNDIR}"/tools/socket_passing "-o" "${UPDATED_HOT_RESTART_JSON}" "-a" "${ADMIN_ADDRESS_PATH_1}" \ "-u" "${HOT_RESTART_JSON_1}" CONFIG_DIFF=$(diff "${UPDATED_HOT_RESTART_JSON}" "${HOT_RESTART_JSON_1}") @@ -213,7 +215,7 @@ do sleep 3 start_test Checking that listener addresses have not changed - HOT_RESTART_JSON_2="${TEST_TMPDIR}"/hot_restart.2."${TEST_INDEX}".json + HOT_RESTART_JSON_2="${TEST_TMPDIR}"/hot_restart.2."${TEST_INDEX}".yaml "${TEST_RUNDIR}"/tools/socket_passing "-o" "${UPDATED_HOT_RESTART_JSON}" "-a" "${ADMIN_ADDRESS_PATH_2}" \ "-u" "${HOT_RESTART_JSON_2}" CONFIG_DIFF=$(diff "${UPDATED_HOT_RESTART_JSON}" "${HOT_RESTART_JSON_2}") diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index e76fc76e22a7c..fd2afaa722aba 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -91,7 +91,7 @@ TEST(OptionsImplTest, All) { TEST(OptionsImplTest, SetAll) { std::unique_ptr options = createOptionsImpl("envoy -c hello"); bool v2_config_only = options->v2ConfigOnly(); - bool hot_restart_disabled = options->v2ConfigOnly(); + bool hot_restart_disabled = options->hotRestartDisabled(); options->setBaseId(109876); options->setConcurrency(42); options->setConfigPath("foo"); diff --git a/tools/socket_passing.py b/tools/socket_passing.py index d8372e46e2fde..cdbbcf0a9d99b 100755 --- a/tools/socket_passing.py +++ b/tools/socket_passing.py @@ -13,6 +13,7 @@ import httplib import json import os.path +import re import sys import time @@ -20,38 +21,56 @@ # with failure if the file is not found. ADMIN_FILE_TIMEOUT_SECS = 20 -def GenerateNewConfig(original_json, admin_address, updated_json): +# Because the hot restart files are yaml but yaml support is not included in +# python by default, we parse this fairly manually. +def GenerateNewConfig(original_yaml, admin_address, updated_json): # Get original listener addresses - with open(original_json, 'r') as original_json_file: - # Import original config file in order to get a deterministic output. This - # allows us to diff the original config file and the updated config file - # output from this script to check for any changes. - parsed_json = json.load(original_json_file, object_pairs_hook=OrderedDict) - original_listeners = parsed_json['listeners'] - - sys.stdout.write('Admin address is ' + admin_address + '\n') - try: - admin_conn = httplib.HTTPConnection(admin_address) - admin_conn.request('GET', '/listeners') - admin_response = admin_conn.getresponse() - if not admin_response.status == 200: - return False - discovered_listeners = json.loads(admin_response.read()) - except Exception as e: - sys.stderr.write('Cannot connect to admin: %s\n' % e) - return False - else: - if len(discovered_listeners) != len(original_listeners): + with open(original_yaml, 'r') as original_file: + sys.stdout.write('Admin address is ' + admin_address + '\n') + try: + admin_conn = httplib.HTTPConnection(admin_address) + admin_conn.request('GET', '/listeners') + admin_response = admin_conn.getresponse() + if not admin_response.status == 200: + return False + discovered_listeners = json.loads(admin_response.read()) + except Exception as e: + sys.stderr.write('Cannot connect to admin: %s\n' % e) return False - for discovered, original in zip(discovered_listeners, original_listeners): - if discovered.startswith('/'): - original['address'] = 'unix://' + discovered - else: - original['address'] = 'tcp://' + discovered - with open(updated_json, 'w') as outfile: - json.dump(OrderedDict(parsed_json), outfile, indent=2, separators=(',',':')) - finally: - admin_conn.close() + else: + raw_yaml = original_file.readlines() + index = 0; + for discovered in discovered_listeners: + replaced = False; + if discovered.startswith('/'): + for index in range(index + 1, len(raw_yaml) - 1): + if 'pipe:' in raw_yaml[index] and 'path:' in raw_yaml[index + 1]: + raw_yaml[index + 1] = re.sub( + 'path:.*', 'path: "' + discovered + '"', raw_yaml[index + 1]) + replaced = True + break + else: + addr, _, port = discovered.rpartition(':') + if addr[0] == '[': + addr = addr[1:-1] # strip [] from ipv6 address. + for index in range(index + 1, len(raw_yaml) - 2): + if ('socket_address:' in raw_yaml[index] and 'address:' in raw_yaml[index + 1] + and'port_value:' in raw_yaml[index + 2]): + raw_yaml[index + 1] = re.sub( + 'address:.*', 'address: "' + addr + '"', raw_yaml[index + 1]) + raw_yaml[index + 2] = re.sub( + 'port_value:.*', 'port_value: ' + port, raw_yaml[index + 2]) + replaced = True + break + if replaced: + sys.stderr.write('replaced listener at line ' + str(index) + ' with ' + discovered + '\n') + else: + sys.stderr.write('Failed to replace a discovered listener ' + discovered + '\n') + return False; + with open(updated_json, 'w') as outfile: + outfile.writelines(raw_yaml) + finally: + admin_conn.close() return True