Skip to content

Commit

Permalink
Support TLS listeners
Browse files Browse the repository at this point in the history
Added
=====

- TLS configuration in broker config
- Helper script for generating test certs and CAs
- TLS options for NtcChannel
- Loading certificates and authority data specified from bmqbrkrcfg.json
- SessionOptions to bmq package for configuring client sessions
- --tls-authority and --tls-version options to bmqtool to configure
  session options
- Client sessions will now require broker TLS sessions when TLS protocol
  versions are specified
- Create CertificateStore component for bmqio
- Integration tests for TLS

Changed
=======

- Update ntf-core and bde dependencies

Signed-off-by: Taylor Foxhall <[email protected]>

Signed-off-by: Evgeny Malygin <[email protected]>
  • Loading branch information
hallfox committed Dec 17, 2024
1 parent 5a6670d commit a1e957d
Show file tree
Hide file tree
Showing 45 changed files with 4,223 additions and 1,350 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,5 @@ src/applications/bmqbrkr/etc/etc

# 'sim_cpp11_features.pl' backups
*.bak
/venv

38 changes: 32 additions & 6 deletions CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,42 @@
},
{
"name": "macos-arm64-vcpkg",
"description": "VCPKG based configuration for building on arm-based MacOS",
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"description":
"VCPKG based configuration for building on arm-based MacOS",
"toolchainFile":
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"inherits": "base",
"cacheVariables": {
"VCPKG_INSTALL_OPTIONS": "--allow-unsupported",
"FLEX_INCLUDE_DIR": "/opt/homebrew/opt/flex/include"
}
},
{
"name": "macos-arm64-darwin",
"description":
"build-darwin.sh based configuration for building on arm-based MacOS",
"toolchainFile":
"${sourceDir}/thirdparty/bde-tools/BdeBuildSystem/toolchains/darwin/clang-default.cmake",
"inherits": "base",
"environment": {
"BREW_PKG_CONFIG_PATH":
"/opt/homebrew/lib/pkgconfig:/opt/homebrew/opt/zlib/lib/pkgconfig:/opt/homebrew/opt/googletest/lib/pkgconfig",
"PKG_CONFIG_PATH":
"${sourceDir}/install/lib/pkgconfig:$env{BREW_PKG_CONFIG_PATH}"
},
"cacheVariables": {
"FLEX_ROOT": "/opt/homebrew/opt/flex",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install",
"CMAKE_PREFIX_PATH":
"${sourceDir}/install;${sourceDir}/thirdparty/bde-tools/BdeBuildSystem"
}
},
{
"name": "macos-x64-vcpkg",
"description": "VCPKG based configuration for building on x86_64-based MacOS",
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"description":
"VCPKG based configuration for building on x86_64-based MacOS",
"toolchainFile":
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"inherits": "base",
"cacheVariables": {
"VCPKG_INSTALL_OPTIONS": "--allow-unsupported",
Expand All @@ -37,8 +61,10 @@
},
{
"name": "linux-x64-vcpkg",
"description": "VCPKG based configuration for building on x86_64-based Linux",
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"description":
"VCPKG based configuration for building on x86_64-based Linux",
"toolchainFile":
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"inherits": "base"
}
]
Expand Down
17 changes: 13 additions & 4 deletions bin/build-darwin.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,28 @@ mkdir -p "${DIR_THIRDPARTY}"
DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
mkdir -p "${DIR_BUILD}"

DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
mkdir -p "${DIR_INSTALL}"


# :: Clone dependencies :::::::::::::::::::::::::::::::::::::::::::::::::::::::
if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
git clone --depth 1 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
pushd "${DIR_THIRDPARTY}/bde-tools"
git reset --hard 964f78d36577ef1643b9074412608df4850e5b33 # 4.17.0.0
popd
fi
if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
git clone --depth 1 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
pushd "${DIR_THIRDPARTY}/bde"
git reset --hard ec094b4454738c311482fc54c3bde1d21c9f6893 # 4.18.0.0
popd
fi
if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
git clone --depth 1 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
pushd "${DIR_THIRDPARTY}/ntf-core"
git reset --hard 2.5.4
popd
fi


Expand Down
17 changes: 13 additions & 4 deletions bin/build-ubuntu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,19 +75,28 @@ mkdir -p "${DIR_THIRDPARTY}"
DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
mkdir -p "${DIR_BUILD}"

DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
mkdir -p "${DIR_INSTALL}"

# :: Clone dependencies :::::::::::::::::::::::::::::::::::::::::::::::::::::::

if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
git clone --depth 1 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
pushd "${DIR_THIRDPARTY}/bde-tools"
git reset --hard 964f78d36577ef1643b9074412608df4850e5b33 # 4.17.0.0
popd
fi
if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
git clone --depth 1 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
pushd "${DIR_THIRDPARTY}/bde"
git reset --hard ec094b4454738c311482fc54c3bde1d21c9f6893 # 4.18.0.0
popd
fi
if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
git clone --depth 1 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
pushd "${DIR_THIRDPARTY}/ntf-core"
git reset --hard 2.5.4
popd
fi
# prometheus-cpp and its dependency for the plugin
if [ "${BUILD_PROMETHEUS}" == true ]; then
Expand Down
127 changes: 127 additions & 0 deletions bin/gen-tls-certs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#!/bin/bash
#
#
# This scripts generates:
# - root CA certificate
# - server certificate and keystore
# - client keys
#
# Based off of
# https://github.com/confluentinc/librdkafka/blob/master/tests/gen-ssl-certs.sh

OP="$1"
CA_CERT="$2"
PFX="$3"
HOST="$4"

C=NN
ST=NN
L=NN
O=NN
OU=NN
CN="$HOST"


# Password
PASS="secret"

# Cert validity, in days
VALIDITY=10000

set -e

export LC_ALL=C

if [[ $OP == "ca" && -n "$CA_CERT" && -n "$3" ]]; then
CN="$3"
openssl req -new -x509 -keyout "${CA_CERT}.key" -out "${CA_CERT}" -days $VALIDITY -passin "pass:$PASS" -passout "pass:$PASS" <<EOF
${C}
${ST}
${L}
${O}
${OU}
${CN}
$USER@${CN}
.
.
EOF



elif [[ $OP == "server" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then
HOST_CERT_CONFIG_PATH="${PFX}host_cert.cnf"
HOST_PRIVATE_RSA_KEY_PATH="${PFX}host_private_key_rsa.pem"
HOST_PRIVATE_KEY_PATH="${PFX}private_key.pem"
HOST_CSR_PATH="${PFX}host_csr.pem"
HOST_CERT_PATH="${PFX}host_cert.pem"
HOST_CERT_CHAIN_PATH="${PFX}client_${CN}.pem"

# Create the CA cert config file
echo "Setting up host certs..."

cat <<EOF > "${HOST_CERT_CONFIG_PATH}"
[req]
default_bits = 2048
distinguished_name = req_distinguished_name
req_extensions = v3_req
prompt = no
[req_distinguished_name]
C = ${C}
ST = ${ST}
L = ${L}
O = ${O}
CN = ${CN}
[v3_req]
subjectAltName = @alt_names
[alt_names]
DNS.1 = ${CN}
DNS.2 = localhost.
EOF

#Step 1
echo "############ Generating key"
openssl genrsa -out "${HOST_PRIVATE_RSA_KEY_PATH}" 2048
openssl pkcs8 -nocrypt -topk8 -v1 PBE-SHA1-RC4-128 -inform pem -outform pem -in "${HOST_PRIVATE_RSA_KEY_PATH}" -out "${HOST_PRIVATE_KEY_PATH}"

#Step 2
echo "############ Generate the CSR"
openssl req -nodes -new -extensions v3_req -sha256 -config "${HOST_CERT_CONFIG_PATH}" -key "${HOST_PRIVATE_KEY_PATH}" -out "${HOST_CSR_PATH}"

#Step 3
echo "############ Generate the cert"
openssl x509 -req -in "${HOST_CSR_PATH}" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${HOST_CERT_PATH}" -days ${VALIDITY} -sha256 -extensions v3_req -extfile "${HOST_CERT_CONFIG_PATH}" -passin "pass:${PASS}"

cat "${HOST_CERT_PATH}" > "${HOST_CERT_CHAIN_PATH}"


elif [[ $OP == "client" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then

# Standard OpenSSL keys
echo "############ Generating key"
openssl genrsa -nodes -passout "pass:${PASS}" -out "${PFX}client.key" 2048

echo "############ Generating request"
openssl req -passin "pass:${PASS}" -passout "pass:${PASS}" -key "${PFX}client.key" -new -out "${PFX}client.req" \
<<EOF
$C
$ST
$L
$O
$OU
$CN
.
$PASS
.
EOF

echo "########### Signing key"
openssl x509 -req -passin "pass:${PASS}" -in "${PFX}client.req" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${PFX}client.pem" -days ${VALIDITY}


else
echo "Usage: $0 ca <ca-cert-file> <CN>"
echo " $0 server|client <ca-cert-file> <file_prefix> <hostname>"
echo ""
exit 1
fi

6 changes: 3 additions & 3 deletions docker/build_deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ fetch_git() {
}

fetch_deps() {
fetch_git bloomberg bde-tools 4.8.0.0
fetch_git bloomberg bde 4.8.0.0
fetch_git bloomberg ntf-core 2.4.2
fetch_git bloomberg bde-tools 4.13.0.0
fetch_git bloomberg bde 4.18.0.0
fetch_git bloomberg ntf-core 2.5.4
}

configure() {
Expand Down
12 changes: 12 additions & 0 deletions docs/docs/features/tls.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# TLS in BlazingMQ

BlazingMQ supports authenticating brokers using TLS.


## Generating Test Certs

```sh
mkdir -p certs && cd certs
../bin/gen-tls-certs.sh ca ca-cert blazingmq
../bin/gen-tls-certs.sh server ca-cert broker_bmqc00_ bmqc00
```
24 changes: 21 additions & 3 deletions src/applications/bmqbrkr/etc/bmqbrkrcfg.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
"rotationBytes": 268435456,
"logfileFormat": "%d (%t) %s %F:%l %m\n\n",
"consoleFormat": "%d (%t) %s %F:%l %m\n",
"loggingVerbosity": "INFO",
"consoleSeverityThreshold": "INFO",
"loggingVerbosity": "TRACE",
"consoleSeverityThreshold": "TRACE",
"categories": [
"BMQBRKR:INFO:green",
"BMQ*:INFO:green",
Expand Down Expand Up @@ -88,11 +88,29 @@
"highWatermark": 1073741824,
"nodeLowWatermark": 5242880,
"nodeHighWatermark": 10485760,
"heartbeatIntervalMs": 3000
"heartbeatIntervalMs": 3000,
"listeners": [
{
"name": "TCPListener",
"port": 30114,
"tls": false
},
{
"name": "TLSListener",
"port": 30115,
"tls": true
}
]
}
},
"bmqconfConfig": {
"cacheTTLSeconds": 30
},
"tlsConfig": {
"certificateAuthority": "/blazingmq/certs/ca-cert",
"certificate": "/blazingmq/certs/broker_host_cert.pem",
"key": "/blazingmq/certs/broker_private_key.pem",
"version": ""
}
}
}
12 changes: 12 additions & 0 deletions src/applications/bmqtool/bmqtool.m.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,18 @@ static bool parseArgs(Parameters* parameters, int argc, const char* argv[])
"address and port of the broker",
balcl::TypeInfo(&params.broker()),
balcl::OccurrenceInfo(params.broker())},
{"tls-authority",
"tlsAuthority",
"Path to the certificate authority for TLS mode."
"The empty string value means that TLS is disabled, "
"non-empty string value means that TLS is enabled",
balcl::TypeInfo(&params.tlsAuthority()),
balcl::OccurrenceInfo(params.tlsAuthority())},
{"tls-versions",
"tlsVersions",
"TLS protocol versions, has effect only in TLS mode",
balcl::TypeInfo(&params.tlsVersions()),
balcl::OccurrenceInfo(params.tlsVersions())},
{"q|queueuri",
"uri",
"URI of the queue (for auto/syschk modes)",
Expand Down
2 changes: 2 additions & 0 deletions src/applications/bmqtool/bmqtoolcmd.xsd
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@
<sequence>
<element name='mode' type='string' default="cli"/>
<element name='broker' type='string' default="tcp://localhost:30114"/>
<element name='tlsAuthority' type='string' default=""/>
<element name='tlsVersions' type='string' default="TLSv1.3"/>
<element name='queueUri' type='string' default=""/>
<element name='queueFlags' type='string' default=""/>
<element name='latency' type='string' default="none"/>
Expand Down
5 changes: 5 additions & 0 deletions src/applications/bmqtool/m_bmqtool_application.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,11 @@ int Application::initialize()
.setNumProcessingThreads(d_parameters_p->numProcessingThreads())
.configureEventQueue(1000, 10 * 1000);

if (!d_parameters_p->certificateAuthority().empty()) {
options.setTlsDetails(d_parameters_p->certificateAuthority(),
"TLSv1.3");
}

// Create the session
if (d_parameters_p->noSessionEventHandler()) {
d_session_mp.load(new (*d_allocator_p)
Expand Down
Loading

0 comments on commit a1e957d

Please sign in to comment.