diff --git a/.vscode/README.md b/.vscode/README.md new file mode 100644 index 000000000..d413d1c4a --- /dev/null +++ b/.vscode/README.md @@ -0,0 +1,16 @@ +# Developing Nighthawk using VSCode + +Install VSCode and click `file->open workspace` in the menu. +Next, open [nighthawk.code-workspace](../nighthawk.code-workspace). + +You can now use `shift+ctrl+p` or `shitf-command+p` (osx) to run +various [tasks](tasks.json) associated to Nighthawk development, +by selecting `Tasks: run task` in the dropdown: + +![VSCode tasks dropdown](./tasks.png) + +It's recommended to check out the plug-in gallery for facilitating +work with bazel, python and c++. + +We are happy to answer any questions on getting started on +[Slack](https://envoyproxy.slack.com/archives/CDX3CGTT9). diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..b4f920825 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,45 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) test-server tests", + "type": "cppdbg", + "request": "launch", + "preLaunchTask": "Direct/Dbg: build tests", + "program": "${workspaceRoot}/bazel-bin/test/server/http_test_server_filter_integration_test", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/", + "environment": [ + { + "name": "TEST_TMPDIR", + "value": "/tmp/foo" + }, + { + "name": "TEST_WORKSPACE", + "value": "." + }, + { + "name": "TEST_SRCDIR", + "value": "${workspaceRoot}/" + }, + { + "name": "ENVOY_IP_TEST_VERSIONS", + "value": "" + } + ], + "externalConsole": false, + "MIMode": "gdb", + "sourceFileMap": { + "/proc/self/cwd/external": "${workspaceFolder}/bazel-nighthawk/external/" + }, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..768a1a824 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "python.pythonPath": "/usr/bin/python3", + "python.analysis.logLevel": "Warning", + "python.formatting.provider": "yapf", + "python.formatting.yapfArgs": ["--style", "nighthawk/tools/.style.yapf"], + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.enabled": true +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..21b6ed8a7 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,253 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Refresh compilation database.", + "type": "shell", + "command": "tools/gen_compilation_database.py --vscode --include_external --include_genfiles --include_headers", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Direct Update CLI README.md docs", + "type": "shell", + "command": "BAZEL_BUILD_OPTIONS=\"-c fastbuild\" tools/update_cli_readme_documentation.sh --mode fix", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Build Nighthawk-dev dockage", + "type": "shell", + "command": "ci/do_ci.sh docker", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Docker Fix Format", + "type": "shell", + "command": "ci/run_envoy_docker.sh 'ci/do_ci.sh check_format'", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Direct Fix Format", + "type": "shell", + "command": "ci/do_ci.sh fix_format", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Direct/Dbg: binaries", + "type": "shell", + "command": "bazel build -c dbg //:nighthawk", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Direct/Dbg: build tests", + "type": "shell", + "command": "bazel build -c dbg //test/...", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Direct/Opt: binaries", + "type": "shell", + "command": "bazel build -c opt //:nighthawk", + "problemMatcher": ["$gcc"], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Docker/Coverage tests", + "type": "shell", + "command": "ci/run_envoy_docker.sh 'ci/do_ci.sh coverage'", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Docker/TSAN tests", + "type": "shell", + "command": "ci/run_envoy_docker.sh 'ci/do_ci.sh tsan'", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Docker/ASAN tests", + "type": "shell", + "command": "ci/run_envoy_docker.sh 'ci/do_ci.sh asan'", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Docker/clang-tidy", + "type": "shell", + "command": "ci/run_envoy_docker.sh 'ci/do_ci.sh clang_tidy'", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Direct/ASAN tests", + "type": "shell", + "command": "ci/do_ci.sh asan", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Direct/TSAN tests", + "type": "shell", + "command": "ci/do_ci.sh tsan", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Direct/Opt tests", + "type": "shell", + "command": "bazel test -c opt //test:nighthawk_test", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Build benchmarks", + "type": "shell", + "command": "bazel build -c fastbuild \"//benchmarks:*\"", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Benchmark execution with binaries from this repo", + "type": "shell", + "command": "bazel test --test_summary=detailed --test_output=all --test_arg=--log-cli-level=info --test_env=ENVOY_IP_TEST_VERSIONS=v4only --test_env=HEAPPROFILE= --test_env=HEAPCHECK= --cache_test_results=no --compilation_mode=opt --cxxopt=-g --cxxopt=-ggdb3 //benchmarks:*", + "options": { + "cwd": "${workspaceRoot}/nighthawk", + "env": { + "TMPDIR": "${workspaceRoot}/nighthawk/benchmarks/tmp", + "ENVOY_IP_TEST_VERSIONS": "v4only" + } + }, + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Benchmark execution with NH & Envoy docker images", + "type": "shell", + "command": "benchmarks/run_benchmark.sh", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Direct repeat tests 1000x", + "type": "shell", + "command": "bazel test --cache_test_results=no --test_env=ENVOY_IP_TEST_VERSIONS=all --runs_per_test=1000 --jobs 50 -c dbg --local_resources 20000,20,0.25 //test:*", + "problemMatcher": ["$gcc"], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Direct/Fastbuild: run test(s)", + "type": "shell", + "command": "bazel", + "args": [ + "test", + "-c", + "fastbuild", + "--test_env=ENVOY_IP_TEST_VERSIONS=v4only", + "--cache_test_results=no", + "${input:testTarget}" + ], + "group": { + "kind": "test", + "isDefault": true + } + } + ], + "inputs": [ + { + "type": "pickString", + "id": "testTarget", + "description": "type", + "options": [ + "//test:benchmark_http_client_test", + "//test:client_test", + "//test:client_worker_test", + "//test:factories_test", + "//test:frequency_test", + "//test:options_test", + "//test:output_formatter_test", + "//test:output_transform_main_test", + "//test:platform_util_test", + "//test:process_test", + "//test:python_test", + "//test:rate_limiter_test", + "//test:request_generator_test", + "//test:sequencer_test", + "//test:service_main_test", + "//test:service_test", + "//test:statistic_test", + "//test:stream_decoder_test", + "//test:termination_predicate_test", + "//test:utility_test", + "//test:worker_test", + "//test:sni_utility_test", + "//test/server:http_test_server_filter_integration_test", + "//test/server:http_dynamic_delay_filter_integration_test", + "//test/..." + ], + "default": "//test/..." + } + ] +} diff --git a/.vscode/tasks.png b/.vscode/tasks.png new file mode 100644 index 000000000..beb1827f9 Binary files /dev/null and b/.vscode/tasks.png differ diff --git a/include/nighthawk/adaptive_load/BUILD b/include/nighthawk/adaptive_load/BUILD index caf45f17c..efbef7c3b 100644 --- a/include/nighthawk/adaptive_load/BUILD +++ b/include/nighthawk/adaptive_load/BUILD @@ -16,6 +16,8 @@ envoy_basic_cc_library( include_prefix = "nighthawk/adaptive_load", deps = [ "//api/adaptive_load:adaptive_load_proto_cc_proto", + "//api/client:grpc_service_lib", + "@envoy//include/envoy/common:time_interface", ], ) @@ -27,6 +29,8 @@ envoy_basic_cc_library( include_prefix = "nighthawk/adaptive_load", deps = [ "//api/adaptive_load:adaptive_load_proto_cc_proto", + "@envoy//include/envoy/common:base_includes", + "@envoy//include/envoy/config:typed_config_interface", ], ) @@ -38,6 +42,8 @@ envoy_basic_cc_library( include_prefix = "nighthawk/adaptive_load", deps = [ "//api/adaptive_load:adaptive_load_proto_cc_proto", + "@envoy//include/envoy/common:base_includes", + "@envoy//include/envoy/config:typed_config_interface", ], ) @@ -49,6 +55,8 @@ envoy_basic_cc_library( include_prefix = "nighthawk/adaptive_load", deps = [ "//api/adaptive_load:adaptive_load_proto_cc_proto", + "@envoy//include/envoy/common:base_includes", + "@envoy//include/envoy/config:typed_config_interface", ], ) @@ -60,5 +68,7 @@ envoy_basic_cc_library( include_prefix = "nighthawk/adaptive_load", deps = [ "//api/adaptive_load:adaptive_load_proto_cc_proto", + "@envoy//include/envoy/common:base_includes", + "@envoy//include/envoy/config:typed_config_interface", ], ) diff --git a/nighthawk.code-workspace b/nighthawk.code-workspace new file mode 100644 index 000000000..876a1499c --- /dev/null +++ b/nighthawk.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index 73ec0ba57..12724897c 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -5,6 +5,7 @@ import argparse import glob import json +import logging import os import shlex import subprocess @@ -12,22 +13,36 @@ def runBazelBuildForCompilationDatabase(bazel_options, bazel_targets): - query = 'attr(include_prefix, ".+", kind(cc_library, deps({})))'.format( - ' union '.join(bazel_targets)) - build_targets = subprocess.check_output(["bazel", "query", query]).decode().splitlines() + query_targets = ' union '.join(bazel_targets) + query = ' union '.join( + q.format(query_targets) for q in [ + 'attr(include_prefix, ".+", kind(cc_library, deps({})))', + 'attr(strip_include_prefix, ".+", kind(cc_library, deps({})))', + 'attr(generator_function, ".*proto_library", kind(cc_.*, deps({})))', + ]) + build_targets = subprocess.check_output(["bazel", "query", "--notool_deps", + query]).decode().splitlines() subprocess.check_call(["bazel", "build"] + bazel_options + build_targets) # This method is equivalent to https://github.com/grailbio/bazel-compilation-database/blob/master/generate.sh def generateCompilationDatabase(args): - # We need to download all remote outputs for generated source code, we don't care about built - # binaries so just always strip and use dynamic link to minimize download size. + # We need to download all remote outputs for generated source code. This option lives here to override those + # specified in bazelrc. bazel_options = shlex.split(os.environ.get("BAZEL_BUILD_OPTIONS", "")) + [ - "-c", "fastbuild", "--build_tag_filters=-manual", - "--experimental_remote_download_outputs=all", "--strip=always" + "--config=compdb", + "--remote_download_outputs=all", ] + if args.keep_going: + bazel_options.append("-k") if args.run_bazel_build: - runBazelBuildForCompilationDatabase(bazel_options, args.bazel_targets) + try: + runBazelBuildForCompilationDatabase(bazel_options, args.bazel_targets) + except subprocess.CalledProcessError as e: + if not args.keep_going: + raise + else: + logging.warning("bazel build failed {}: {}".format(e.returncode, e.cmd)) subprocess.check_call(["bazel", "build"] + bazel_options + [ "--aspects=@bazel_compdb//:aspects.bzl%compilation_database_aspect", @@ -98,16 +113,12 @@ def fixCompilationDatabase(args, db): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Generate JSON compilation database') parser.add_argument('--run_bazel_build', action='store_true') + parser.add_argument('-k', '--keep_going', action='store_true') parser.add_argument('--include_external', action='store_true') parser.add_argument('--include_genfiles', action='store_true') parser.add_argument('--include_headers', action='store_true') parser.add_argument('--vscode', action='store_true') # @@@ - parser.add_argument('bazel_targets', - nargs='*', - default=[ - "//source/...", "//test:*", "//test/integration/...", "//test/client/...", - "//test/server/...", "//tools/..." - ]) + parser.add_argument('bazel_targets', nargs='*', default=["//..."]) args = parser.parse_args() fixCompilationDatabase(args, generateCompilationDatabase(args))