diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 1320669f485..bc9396678af 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -43,6 +43,7 @@ RUN yum install -y \ telnet-0.17-66.el7 \ traceroute-2.0.22-2.el7 \ unzip-6.0-22.el7_9 \ + openssl-1.0.2k-24.el7_9.x86_64 \ vim-enhanced-7.4.629-8.el7_9 && \ yum clean all && \ rm -rf /var/cache/yum @@ -177,13 +178,13 @@ RUN yum -y install \ rm -rf /var/cache/yum WORKDIR /tmp -RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ - echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ +RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \ + echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07 kubectl" > kubectl.txt && \ sha256sum --quiet -c kubectl.txt && \ mv kubectl /usr/local/bin/kubectl && \ chmod 755 /usr/local/bin/kubectl && \ - curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ - echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.7.34.zip -o "awscliv2.zip" && \ + echo "daf9253f0071b5cfee9532bc5220bedd7a5d29d4e0f92b42b9e3e4c496341e88 awscliv2.zip" > awscliv2.txt && \ sha256sum --quiet -c awscliv2.txt && \ unzip -qq awscliv2.zip && \ ./aws/install && \ diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks index 6fa4d6ee97d..cde5f858598 100644 --- a/packaging/docker/Dockerfile.eks +++ b/packaging/docker/Dockerfile.eks @@ -53,6 +53,18 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6 mv tini /usr/bin/ && \ rm -rf /tmp/* +RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \ + echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07 kubectl" > kubectl.txt && \ + sha256sum --quiet -c kubectl.txt && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod 755 /usr/local/bin/kubectl && \ + curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.7.34.zip -o "awscliv2.zip" && \ + echo "daf9253f0071b5cfee9532bc5220bedd7a5d29d4e0f92b42b9e3e4c496341e88 awscliv2.zip" > awscliv2.txt && \ + sha256sum --quiet -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* + WORKDIR / FROM golang:1.16.7-bullseye AS go-build @@ -173,19 +185,6 @@ RUN yum -y install \ yum clean all && \ rm -rf /var/cache/yum -WORKDIR /tmp -RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ - echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ - sha256sum --quiet -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ - echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ - sha256sum --quiet -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - # TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation # time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh index 5b5ef4b0857..5cf55127cd8 100755 --- a/packaging/docker/build-images.sh +++ b/packaging/docker/build-images.sh @@ -2,12 +2,17 @@ set -Eeuo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) reset=$(tput sgr0) +red=$(tput setaf 1) blue=$(tput setaf 4) -function logg() { +function logg () { printf "${blue}##### $(date +"%H:%M:%S") # %-56.55s #####${reset}\n" "${1}" } + function loge () { + printf "${red}##### $(date +"%H:%M:%S") # %-56.55s #####${reset}\n" "${1}" + } + function pushd () { command pushd "$@" > /dev/null } @@ -16,7 +21,19 @@ function popd () { command popd > /dev/null } -function create_fake_website_directory() { +function error_exit () { + echo "${red}################################################################################${reset}" + loge "${0} FAILED" + echo "${red}################################################################################${reset}" + } + + trap error_exit ERR + + function create_fake_website_directory () { + if [ ${#} -ne 1 ]; then + loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}" + fi + local stripped_binaries_and_from_where="${1}" fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' ) logg "PREPARING WEBSITE" website_directory="${script_dir}/website" @@ -112,7 +129,7 @@ function create_fake_website_directory() { fdb_website="file:///tmp/website" } -function compile_ycsb() { +function compile_ycsb () { logg "COMPILING YCSB" if [ "${use_development_java_bindings}" == "true" ]; then logg "INSTALL JAVA BINDINGS" @@ -150,7 +167,13 @@ function compile_ycsb() { popd || exit 128 } -function build_and_push_images(){ +function build_and_push_images () { + if [ ${#} -ne 3 ]; then + loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}" + fi + local dockerfile_name="${1}" + local use_development_java_bindings="${2}" + local push_docker_images="${3}" declare -a tags_to_push=() for image in "${image_list[@]}"; do logg "BUILDING ${image}" @@ -237,16 +260,12 @@ image_list=( ) registry="" tag_base="foundationdb/" -# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW -stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) -dockerfile_name="Dockerfile" -use_development_java_bindings="false" -push_docker_images="false" if [ -n "${OKTETO_NAMESPACE+x}" ]; then logg "RUNNING IN OKTETO/AWS" # these are defaults for the Apple development environment - aws_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region") + imdsv2_token=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + aws_region=$(curl -H "X-aws-ec2-metadata-token: ${imdsv2_token}" "http://169.254.169.254/latest/meta-data/placement/region") aws_account_id=$(aws --output text sts get-caller-identity --query 'Account') build_output_directory="${HOME}/build_output" fdb_library_versions=( "${fdb_version}" ) @@ -257,20 +276,24 @@ if [ -n "${OKTETO_NAMESPACE+x}" ]; then else tag_postfix="${OKTETO_NAME:-dev}" fi - stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) - dockerfile_name="Dockerfile.eks" - use_development_java_bindings="true" - push_docker_images="true" + + # build regular images + create_fake_website_directory stripped_local + build_and_push_images Dockerfile true true + + # build debug images + create_fake_website_directory unstripped_local + build_and_push_images Dockerfile.eks true true else echo "Dear ${USER}, you probably need to edit this file before running it. " echo "${0} has a very narrow set of situations where it will be successful," echo "or even useful, when executed unedited" exit 1 + # this set of options will creat standard images from a local build + # create_fake_website_directory stripped_local + # build_and_push_images Dockerfile false false fi -create_fake_website_directory -build_and_push_images - echo "${blue}################################################################################${reset}" logg "COMPLETED ${0}" echo "${blue}################################################################################${reset}" diff --git a/packaging/docker/fdb.bash b/packaging/docker/fdb.bash index 5d23fc41339..75f2ee03cd7 100755 --- a/packaging/docker/fdb.bash +++ b/packaging/docker/fdb.bash @@ -5,7 +5,7 @@ # # This source file is part of the FoundationDB open source project # -# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors +# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packaging/docker/run_ycsb.sh b/packaging/docker/run_ycsb.sh index deb065a7288..bfe3e8df6e1 100755 --- a/packaging/docker/run_ycsb.sh +++ b/packaging/docker/run_ycsb.sh @@ -1,22 +1,44 @@ #!/usr/bin/env bash -set -Eeuxo pipefail +set -Eeuo pipefail + +function logg () { + printf "##### $(date +'%Y-%m-%dT%H:%M:%SZ') # %-56.55s #####\n" "${1}" +} + +function error_exit () { + echo "################################################################################" + logg "${0} FAILED" + logg "RUN_ID: ${RUN_ID}" + logg "WORKLOAD: ${WORKLOAD}" + logg "ENVIRONMENT IS:" + env + echo "################################################################################" +} + +trap error_exit ERR namespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) -POD_NUM=$(echo $POD_NAME | cut -d - -f3) -KEY="ycsb_load_${POD_NUM}_of_${NUM_PODS}_complete" -CLI=$(ls /var/dynamic-conf/bin/*/fdbcli | head -n1) -echo "WAITING FOR ALL PODS TO COME UP" -while [[ $(kubectl get pods -n ${namespace} -l name=ycsb,run=${RUN_ID} --field-selector=status.phase=Running | grep -cv NAME) -lt ${NUM_PODS} ]]; do +logg "WAITING FOR ${NUM_PODS} PODS TO COME UP IN ${namespace}" +while [[ $(kubectl get pods -n "${namespace}" -l name=ycsb,run="${RUN_ID}" --field-selector=status.phase=Running | grep -cv NAME) -lt ${NUM_PODS} ]]; do sleep 1 done -echo "ALL PODS ARE UP" +logg "${NUM_PODS} PODS ARE UP IN ${namespace}" -echo "RUNNING YCSB" -./bin/ycsb.sh ${MODE} foundationdb -s -P workloads/${WORKLOAD} ${YCSB_ARGS} -echo "YCSB FINISHED" +logg "RUNNING YCSB ${WORKLOAD}" +set -x +./bin/ycsb.sh "${MODE}" foundationdb -s -P "workloads/${WORKLOAD}" "${YCSB_ARGS}" +set +x +logg "YCSB ${WORKLOAD} FINISHED" -echo "COPYING HISTOGRAMS TO S3" -aws s3 sync --sse aws:kms --exclude "*" --include "histogram.*" /tmp s3://${BUCKET}/ycsb_histograms/${namespace}/${POD_NAME} -echo "COPYING HISTOGRAMS TO S3 FINISHED" +logg "COPYING HISTOGRAMS TO S3" +set -x +aws s3 sync --sse aws:kms --exclude "*" --include "histogram.*" /tmp "s3://${BUCKET}/ycsb_histograms/${namespace}/${POD_NAME}" +set +x +logg "COPYING HISTOGRAMS TO S3 FINISHED" +echo "################################################################################" +logg "COMPLETED ${0}" +logg "RUN_ID: ${RUN_ID}" +logg "WORKLOAD: ${WORKLOAD}" +echo "################################################################################" diff --git a/packaging/docker/sidecar.py b/packaging/docker/sidecar.py index 666cb82816f..002b19524f7 100755 --- a/packaging/docker/sidecar.py +++ b/packaging/docker/sidecar.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -# entrypoint.py +# sidecar.py # # This source file is part of the FoundationDB open source project # -# Copyright 2018-2021 Apple Inc. and the FoundationDB project authors +# Copyright 2018-2022 Apple Inc. and the FoundationDB project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,24 +22,21 @@ import argparse import hashlib import ipaddress -import logging import json +import logging import os -import re import shutil import socket import ssl -import stat -import time -import traceback import sys import tempfile +import time +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from http.server import HTTPServer, ThreadingHTTPServer, BaseHTTPRequestHandler - -from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer log = logging.getLogger(__name__) log.setLevel(logging.INFO) @@ -166,9 +163,7 @@ def __init__(self): ) parser.add_argument( "--require-not-empty", - help=( - "A file that must be present and non-empty " "in the input directory" - ), + help=("A file that must be present and non-empty in the input directory"), action="append", ) args = parser.parse_args() @@ -243,7 +238,7 @@ def __init__(self): if self.main_container_version == self.primary_version: self.substitutions["BINARY_DIR"] = "/usr/bin" else: - self.substitutions["BINARY_DIR"] = target_path = str( + self.substitutions["BINARY_DIR"] = str( Path("%s/bin/%s" % (args.main_container_conf_dir, self.primary_version)) ) @@ -346,42 +341,16 @@ class ThreadingHTTPServerV6(ThreadingHTTPServer): address_family = socket.AF_INET6 -class Server(BaseHTTPRequestHandler): +class SidecarHandler(BaseHTTPRequestHandler): + # We don't want to load the ssl context for each request so we hold it as a static variable. ssl_context = None - @classmethod - def start(cls): - """ - This method starts the server. - """ - config = Config.shared() - colon_index = config.bind_address.rindex(":") - port_index = colon_index + 1 - address = config.bind_address[:colon_index] - port = config.bind_address[port_index:] - log.info(f"Listening on {address}:{port}") - - if address.startswith("[") and address.endswith("]"): - server = ThreadingHTTPServerV6((address[1:-1], int(port)), cls) - else: - server = ThreadingHTTPServer((address, int(port)), cls) - - if config.enable_tls: - context = Server.load_ssl_context() - server.socket = context.wrap_socket(server.socket, server_side=True) - observer = Observer() - event_handler = CertificateEventHandler() - for path in set( - [ - Path(config.certificate_file).parent.as_posix(), - Path(config.key_file).parent.as_posix(), - ] - ): - observer.schedule(event_handler, path) - observer.start() - - server.serve_forever() + def __init__(self, config, *args, **kwargs): + self.config = config + self.ssl_context = self.__class__.ssl_context + super().__init__(*args, **kwargs) + # This method allows to trigger a reload of the ssl context and updates the static variable. @classmethod def load_ssl_context(cls): config = Config.shared() @@ -390,6 +359,7 @@ def load_ssl_context(cls): cls.ssl_context.check_hostname = False cls.ssl_context.verify_mode = ssl.CERT_OPTIONAL cls.ssl_context.load_cert_chain(config.certificate_file, config.key_file) + return cls.ssl_context def send_text(self, text, code=200, content_type="text/plain", add_newline=True): @@ -407,16 +377,14 @@ def send_text(self, text, code=200, content_type="text/plain", add_newline=True) self.wfile.write(response) def check_request_cert(self, path): - config = Config.shared() - if path == "/ready": return True - if not config.enable_tls: + if not self.config.enable_tls: return True approved = self.check_cert( - self.connection.getpeercert(), config.peer_verification_rules + self.connection.getpeercert(), self.config.peer_verification_rules ) if not approved: self.send_error(401, "Client certificate was not approved") @@ -517,24 +485,33 @@ def do_GET(self): if not self.check_request_cert(self.path): return if self.path.startswith("/check_hash/"): + file_path = os.path.relpath(self.path, "/check_hash") try: - self.send_text(check_hash(self.path[12:]), add_newline=False) + self.send_text( + self.check_hash(file_path), + add_newline=False, + ) except FileNotFoundError: - self.send_error(404, "Path not found") - self.end_headers() + self.send_error(404, f"{file_path} not found") + if self.path.startswith("/is_present/"): + file_path = os.path.relpath(self.path, "/is_present") + if self.is_present(file_path): + self.send_text("OK") + else: + self.send_error(404, f"{file_path} not found") elif self.path == "/ready": - self.send_text(ready()) + self.send_text("OK") elif self.path == "/substitutions": - self.send_text(get_substitutions()) + self.send_text(self.get_substitutions()) else: self.send_error(404, "Path not found") - self.end_headers() except RequestException as e: self.send_error(400, e.message) + except (ConnectionResetError, BrokenPipeError) as ex: + log.error(f"connection was reset {ex}") except Exception as ex: log.error(f"Error processing request {ex}", exc_info=True) self.send_error(500) - self.end_headers() def do_POST(self): """ @@ -544,15 +521,15 @@ def do_POST(self): if not self.check_request_cert(self.path): return if self.path == "/copy_files": - self.send_text(copy_files()) + self.send_text(copy_files(self.config)) elif self.path == "/copy_binaries": - self.send_text(copy_binaries()) + self.send_text(copy_binaries(self.config)) elif self.path == "/copy_libraries": - self.send_text(copy_libraries()) + self.send_text(copy_libraries(self.config)) elif self.path == "/copy_monitor_conf": - self.send_text(copy_monitor_conf()) + self.send_text(copy_monitor_conf(self.config)) elif self.path == "/refresh_certs": - self.send_text(refresh_certs()) + self.send_text(self.refresh_certs()) elif self.path == "/restart": self.send_text("OK") exit(1) @@ -563,16 +540,38 @@ def do_POST(self): raise e except RequestException as e: self.send_error(400, e.message) - except e: - log.error("Error processing request", exc_info=True) + except (ConnectionResetError, BrokenPipeError) as ex: + log.error(f"connection was reset {ex}") + except Exception as ex: + log.error(f"Error processing request {ex}", exc_info=True) self.send_error(500) - self.end_headers() def log_message(self, format, *args): log.info(format % args) + def refresh_certs(self): + if not self.config.enable_tls: + raise RequestException("Server is not using TLS") + SidecarHandler.load_ssl_context() + return "OK" + + def get_substitutions(self): + return json.dumps(self.config.substitutions) + + def check_hash(self, filename): + with open(os.path.join(self.config.output_dir, filename), "rb") as contents: + m = hashlib.sha256() + m.update(contents.read()) + return m.hexdigest() + + def is_present(self, filename): + return os.path.exists(os.path.join(self.config.output_dir, filename)) + class CertificateEventHandler(FileSystemEventHandler): + def __init__(self): + FileSystemEventHandler.__init__(self) + def on_any_event(self, event): if event.is_directory: return None @@ -589,23 +588,15 @@ def on_any_event(self, event): ) time.sleep(10) log.info("Reloading certificates") - Server.load_ssl_context() - + SidecarHandler.load_ssl_context() -def check_hash(filename): - with open(os.path.join(Config.shared().output_dir, filename), "rb") as contents: - m = hashlib.sha256() - m.update(contents.read()) - return m.hexdigest() - -def copy_files(): - config = Config.shared() +def copy_files(config): if config.require_not_empty: for filename in config.require_not_empty: path = os.path.join(config.input_dir, filename) if not os.path.isfile(path) or os.path.getsize(path) == 0: - raise Exception("No contents for file %s" % path) + raise Exception(f"No contents for file {path}") for filename in config.copy_files: tmp_file = tempfile.NamedTemporaryFile( @@ -617,8 +608,7 @@ def copy_files(): return "OK" -def copy_binaries(): - config = Config.shared() +def copy_binaries(config): if config.main_container_version != config.primary_version: for binary in config.copy_binaries: path = Path(f"/usr/bin/{binary}") @@ -638,8 +628,7 @@ def copy_binaries(): return "OK" -def copy_libraries(): - config = Config.shared() +def copy_libraries(config): for version in config.copy_libraries: path = Path(f"/var/fdb/lib/libfdb_c_{version}.so") if version == config.copy_libraries[0]: @@ -658,8 +647,7 @@ def copy_libraries(): return "OK" -def copy_monitor_conf(): - config = Config.shared() +def copy_monitor_conf(config): if config.input_monitor_conf: with open( os.path.join(config.input_dir, config.input_monitor_conf) @@ -683,35 +671,58 @@ def copy_monitor_conf(): return "OK" -def get_substitutions(): - return json.dumps(Config.shared().substitutions) - - -def ready(): - return "OK" - - -def refresh_certs(): - if not Config.shared().enable_tls: - raise RequestException("Server is not using TLS") - Server.load_ssl_context() - return "OK" - - class RequestException(Exception): def __init__(self, message): super().__init__(message) self.message = message +def start_sidecar_server(config): + """ + This method starts the HTTP server with the sidecar handler. + """ + colon_index = config.bind_address.rindex(":") + port_index = colon_index + 1 + address = config.bind_address[:colon_index] + port = config.bind_address[port_index:] + log.info(f"Listening on {address}:{port}") + + handler = partial( + SidecarHandler, + config, + ) + + if address.startswith("[") and address.endswith("]"): + server = ThreadingHTTPServerV6((address[1:-1], int(port)), handler) + else: + server = ThreadingHTTPServer((address, int(port)), handler) + + if config.enable_tls: + context = SidecarHandler.load_ssl_context() + server.socket = context.wrap_socket(server.socket, server_side=True) + observer = Observer() + event_handler = CertificateEventHandler() + for path in set( + [ + Path(config.certificate_file).parent.as_posix(), + Path(config.key_file).parent.as_posix(), + ] + ): + observer.schedule(event_handler, path) + observer.start() + + server.serve_forever() + + if __name__ == "__main__": logging.basicConfig(format="%(asctime)-15s %(levelname)s %(message)s") - copy_files() - copy_binaries() - copy_libraries() - copy_monitor_conf() + config = Config.shared() + copy_files(config) + copy_binaries(config) + copy_libraries(config) + copy_monitor_conf(config) - if Config.shared().init_mode: + if config.init_mode: sys.exit(0) - Server.start() + start_sidecar_server(config) diff --git a/packaging/docker/sidecar_test.py b/packaging/docker/sidecar_test.py new file mode 100644 index 00000000000..1825c158872 --- /dev/null +++ b/packaging/docker/sidecar_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +# sidecar_test.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2018-2022 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +import socket +import tempfile +import unittest +from functools import partial +from http.server import HTTPServer +from threading import Thread +from unittest.mock import MagicMock + +import requests + +from sidecar import SidecarHandler + + +# This test suite starts a real server with a mocked configuration and will do some requests against it. +class TestSidecar(unittest.TestCase): + def setUp(self): + super(TestSidecar, self).setUp() + self.get_free_port() + self.server_url = f"http://localhost:{self.test_server_port}" + self.mock_config = MagicMock() + # We don't want to use TLS for the local tests for now. + self.mock_config.enable_tls = False + self.mock_config.output_dir = tempfile.mkdtemp() + + handler = partial( + SidecarHandler, + self.mock_config, + ) + self.mock_server = HTTPServer(("localhost", self.test_server_port), handler) + + # Start running mock server in a separate thread. + # Daemon threads automatically shut down when the main process exits. + self.mock_server_thread = Thread(target=self.mock_server.serve_forever) + self.mock_server_thread.setDaemon(True) + self.mock_server_thread.start() + + def tearDown(self): + shutil.rmtree(self.mock_config.output_dir) + super(TestSidecar, self).tearDown() + + # Helper method to get a free port + def get_free_port(self): + s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM) + s.bind(("localhost", 0)) + __, port = s.getsockname() + s.close() + self.test_server_port = port + + def test_get_ready(self): + r = requests.get(f"{self.server_url }/ready") + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, "OK\n") + + def test_get_substitutions(self): + expected = {"key": "value"} + self.mock_config.substitutions = expected + r = requests.get(f"{self.server_url }/substitutions") + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json(), expected) + + def test_get_check_hash_no_found(self): + r = requests.get(f"{self.server_url }/check_hash/foobar") + self.assertEqual(r.status_code, 404) + self.assertRegex(r.text, "foobar not found") + + def test_get_check_hash(self): + with open(os.path.join(self.mock_config.output_dir, "foobar"), "w") as f: + f.write("hello world") + r = requests.get(f"{self.server_url }/check_hash/foobar") + self.assertEqual(r.status_code, 200) + self.assertEqual( + r.text, "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + + def test_get_check_hash_nested(self): + test_path = os.path.join(self.mock_config.output_dir, "nested/foobar") + os.makedirs(os.path.dirname(test_path), exist_ok=True) + with open(test_path, "w") as f: + f.write("hello world") + r = requests.get(f"{self.server_url }/check_hash/nested/foobar") + self.assertEqual(r.status_code, 200) + self.assertEqual( + r.text, "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + + def test_get_is_present_no_found(self): + r = requests.get(f"{self.server_url }/is_present/foobar") + self.assertEqual(r.status_code, 404) + self.assertRegex(r.text, "foobar not found") + + def test_get_is_present(self): + with open(os.path.join(self.mock_config.output_dir, "foobar"), "w") as f: + f.write("hello world") + r = requests.get(f"{self.server_url }/is_present/foobar") + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, "OK\n") + + def test_get_is_present_nested(self): + test_path = os.path.join(self.mock_config.output_dir, "nested/foobar") + os.makedirs(os.path.dirname(test_path), exist_ok=True) + with open(test_path, "w") as f: + f.write("hello world") + r = requests.get(f"{self.server_url }/is_present/nested/foobar") + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, "OK\n") + + def test_get_not_found(self): + r = requests.get(f"{self.server_url }/foobar") + self.assertEqual(r.status_code, 404) + self.assertRegex(r.text, "Path not found") + + +# TODO(johscheuer): Add test cases for post requests. +# TODO(johscheuer): Add test cases for TLS. +if __name__ == "__main__": + unittest.main()