diff --git a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD new file mode 100644 index 0000000000000..adc77e5b5e0d3 --- /dev/null +++ b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD @@ -0,0 +1,8 @@ +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "fixed_heap", + srcs = ["fixed_heap.proto"], +) diff --git a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto new file mode 100644 index 0000000000000..08e3c6536f5d3 --- /dev/null +++ b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package envoy.config.resource_monitor.fixed_heap.v2alpha; +option go_package = "v2alpha"; + +message FixedHeapConfig { + // Limit of the Envoy process heap size. This is used to calculate heap memory pressure which + // is defined as (current heap size)/max_heap_size_bytes. + uint64 max_heap_size_bytes = 1; +} diff --git a/include/envoy/server/BUILD b/include/envoy/server/BUILD index fae78b50ab2ad..b2cf79f0a606c 100644 --- a/include/envoy/server/BUILD +++ b/include/envoy/server/BUILD @@ -180,3 +180,20 @@ envoy_cc_library( "//source/common/protobuf", ], ) + +envoy_cc_library( + name = "resource_monitor_interface", + hdrs = ["resource_monitor.h"], + deps = [ + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "resource_monitor_config_interface", + hdrs = ["resource_monitor_config.h"], + deps = [ + ":resource_monitor_interface", + "//include/envoy/event:dispatcher_interface", + ], +) diff --git a/include/envoy/server/resource_monitor.h b/include/envoy/server/resource_monitor.h new file mode 100644 index 0000000000000..3fd01b52ac3b7 --- /dev/null +++ b/include/envoy/server/resource_monitor.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "envoy/common/exception.h" +#include "envoy/common/pure.h" + +namespace Envoy { +namespace Server { + +// Struct for reporting usage for a particular resource. +struct ResourceUsage { + // Fraction of (resource usage)/(resource limit). + double resource_pressure_; +}; + +class ResourceMonitor { +public: + virtual ~ResourceMonitor() {} + + /** + * Notifies caller of updated resource usage. + */ + class Callbacks { + public: + virtual ~Callbacks() {} + + /** + * Called when the request for updated resource usage succeeds. + * @param usage the updated resource usage + */ + virtual void onSuccess(const ResourceUsage& usage) PURE; + + /** + * Called when the request for updated resource usage fails. + * @param error the exception caught when trying to get updated resource usage + */ + virtual void onFailure(const EnvoyException& error) PURE; + }; + + /** + * Recalculate resource usage. + * This must be non-blocking so if RPCs need to be made they should be + * done asynchronously and invoke the callback when finished. + */ + virtual void updateResourceUsage(Callbacks& callbacks) PURE; +}; + +typedef std::unique_ptr ResourceMonitorPtr; + +} // namespace Server +} // namespace Envoy diff --git a/include/envoy/server/resource_monitor_config.h b/include/envoy/server/resource_monitor_config.h new file mode 100644 index 0000000000000..3ab8328f3de7f --- /dev/null +++ b/include/envoy/server/resource_monitor_config.h @@ -0,0 +1,53 @@ +#pragma once + +#include "envoy/common/pure.h" +#include "envoy/event/dispatcher.h" +#include "envoy/server/resource_monitor.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +class ResourceMonitorFactoryContext { +public: + virtual ~ResourceMonitorFactoryContext() {} + + /** + * @return Event::Dispatcher& the main thread's dispatcher. This dispatcher should be used + * for all singleton processing. + */ + virtual Event::Dispatcher& dispatcher() PURE; +}; + +/** + * Implemented by each resource monitor and registered via Registry::registerFactory() + * or the convenience class RegistryFactory. + */ +class ResourceMonitorFactory { +public: + virtual ~ResourceMonitorFactory() {} + + /** + * Create a particular resource monitor implementation. + * @param config const ProtoBuf::Message& supplies the config for the resource monitor + * implementation. + * @param context ResourceMonitorFactoryContext& supplies the resource monitor's context. + * @return ResourceMonitorPtr the resource monitor instance. Should not be nullptr. + * @throw EnvoyException if the implementation is unable to produce an instance with + * the provided parameters. + */ + virtual ResourceMonitorPtr createResourceMonitor(const Protobuf::Message& config, + ResourceMonitorFactoryContext& context) PURE; + + /** + * @return std::string the identifying name for a particular implementation of a resource + * monitor produced by the factory. + */ + virtual std::string name() PURE; +}; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/source/common/memory/stats.cc b/source/common/memory/stats.cc index 38fc68fe6dee6..f73930bc81cef 100644 --- a/source/common/memory/stats.cc +++ b/source/common/memory/stats.cc @@ -21,6 +21,12 @@ uint64_t Stats::totalCurrentlyReserved() { return value; } +uint64_t Stats::totalPageHeapUnmapped() { + size_t value = 0; + MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", &value); + return value; +} + } // namespace Memory } // namespace Envoy @@ -31,6 +37,7 @@ namespace Memory { uint64_t Stats::totalCurrentlyAllocated() { return 0; } uint64_t Stats::totalCurrentlyReserved() { return 0; } +uint64_t Stats::totalPageHeapUnmapped() { return 0; } } // namespace Memory } // namespace Envoy diff --git a/source/common/memory/stats.h b/source/common/memory/stats.h index 7dba0850e337a..ccfe9785bed4f 100644 --- a/source/common/memory/stats.h +++ b/source/common/memory/stats.h @@ -20,6 +20,11 @@ class Stats { * allocated. */ static uint64_t totalCurrentlyReserved(); + + /** + * @return uint64_t the number of bytes in free, unmapped pages in the page heap. + */ + static uint64_t totalPageHeapUnmapped(); }; } // namespace Memory diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 04b1d40a6adb0..dad0243f89dc5 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -70,6 +70,12 @@ EXTENSIONS = { "envoy.filters.network.tcp_proxy": "//source/extensions/filters/network/tcp_proxy:config", "envoy.filters.network.thrift_proxy": "//source/extensions/filters/network/thrift_proxy:config", + # + # Resource monitors + # + + "envoy.resource_monitors.fixed_heap": "//source/extensions/resource_monitors/fixed_heap:config", + # # Stat sinks # diff --git a/source/extensions/resource_monitors/BUILD b/source/extensions/resource_monitors/BUILD new file mode 100644 index 0000000000000..6156949edef64 --- /dev/null +++ b/source/extensions/resource_monitors/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "well_known_names", + hdrs = ["well_known_names.h"], + deps = [ + "//source/common/singleton:const_singleton", + ], +) diff --git a/source/extensions/resource_monitors/common/BUILD b/source/extensions/resource_monitors/common/BUILD new file mode 100644 index 0000000000000..ff6773aaa8d13 --- /dev/null +++ b/source/extensions/resource_monitors/common/BUILD @@ -0,0 +1,18 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "factory_base_lib", + hdrs = ["factory_base.h"], + deps = [ + "//include/envoy/server:resource_monitor_config_interface", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/extensions/resource_monitors/common/factory_base.h b/source/extensions/resource_monitors/common/factory_base.h new file mode 100644 index 0000000000000..899f35237e137 --- /dev/null +++ b/source/extensions/resource_monitors/common/factory_base.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/server/resource_monitor_config.h" + +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace Common { + +template +class FactoryBase : public Server::Configuration::ResourceMonitorFactory { +public: + Server::ResourceMonitorPtr + createResourceMonitor(const Protobuf::Message& config, + Server::Configuration::ResourceMonitorFactoryContext& context) override { + return createResourceMonitorFromProtoTyped( + MessageUtil::downcastAndValidate(config), context); + } + + std::string name() override { return name_; } + +protected: + FactoryBase(const std::string& name) : name_(name) {} + +private: + virtual Server::ResourceMonitorPtr createResourceMonitorFromProtoTyped( + const ConfigProto& config, + Server::Configuration::ResourceMonitorFactoryContext& context) PURE; + + const std::string name_; +}; + +} // namespace Common +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/fixed_heap/BUILD b/source/extensions/resource_monitors/fixed_heap/BUILD new file mode 100644 index 0000000000000..f9042c54305ee --- /dev/null +++ b/source/extensions/resource_monitors/fixed_heap/BUILD @@ -0,0 +1,34 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "fixed_heap_monitor", + srcs = ["fixed_heap_monitor.cc"], + hdrs = ["fixed_heap_monitor.h"], + deps = [ + "//include/envoy/server:resource_monitor_config_interface", + "//source/common/common:assert_lib", + "//source/common/memory:stats_lib", + "@envoy_api//envoy/config/resource_monitor/fixed_heap/v2alpha:fixed_heap_cc", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":fixed_heap_monitor", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/resource_monitors:well_known_names", + "//source/extensions/resource_monitors/common:factory_base_lib", + ], +) diff --git a/source/extensions/resource_monitors/fixed_heap/config.cc b/source/extensions/resource_monitors/fixed_heap/config.cc new file mode 100644 index 0000000000000..d0313789ff014 --- /dev/null +++ b/source/extensions/resource_monitors/fixed_heap/config.cc @@ -0,0 +1,30 @@ +#include "extensions/resource_monitors/fixed_heap/config.h" + +#include "envoy/registry/registry.h" + +#include "common/protobuf/utility.h" + +#include "extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +Server::ResourceMonitorPtr FixedHeapMonitorFactory::createResourceMonitorFromProtoTyped( + const envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig& config, + Server::Configuration::ResourceMonitorFactoryContext& /*unused_context*/) { + return std::make_unique(config); +} + +/** + * Static registration for the fixed heap resource monitor factory. @see RegistryFactory. + */ +static Registry::RegisterFactory + registered_; + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/fixed_heap/config.h b/source/extensions/resource_monitors/fixed_heap/config.h new file mode 100644 index 0000000000000..2a619c6813d9e --- /dev/null +++ b/source/extensions/resource_monitors/fixed_heap/config.h @@ -0,0 +1,30 @@ +#pragma once + +#include "envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.pb.validate.h" +#include "envoy/server/resource_monitor_config.h" + +#include "extensions/resource_monitors/common/factory_base.h" +#include "extensions/resource_monitors/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +class FixedHeapMonitorFactory + : public Common::FactoryBase< + envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig> { +public: + FixedHeapMonitorFactory() + : FactoryBase(ResourceMonitorNames::get().FIXED_HEAP_RESOURCE_MONITOR) {} + +private: + Server::ResourceMonitorPtr createResourceMonitorFromProtoTyped( + const envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig& config, + Server::Configuration::ResourceMonitorFactoryContext& context) override; +}; + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc new file mode 100644 index 0000000000000..a968856aa04db --- /dev/null +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc @@ -0,0 +1,37 @@ +#include "extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h" + +#include "common/common/assert.h" +#include "common/memory/stats.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +uint64_t MemoryStatsReader::reservedHeapBytes() { return Memory::Stats::totalCurrentlyReserved(); } + +uint64_t MemoryStatsReader::unmappedHeapBytes() { return Memory::Stats::totalPageHeapUnmapped(); } + +FixedHeapMonitor::FixedHeapMonitor( + const envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig& config, + std::unique_ptr stats) + : max_heap_(config.max_heap_size_bytes()), stats_(std::move(stats)) { + ASSERT(max_heap_ > 0); +} + +void FixedHeapMonitor::updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) { + const size_t physical = stats_->reservedHeapBytes(); + const size_t unmapped = stats_->unmappedHeapBytes(); + ASSERT(physical >= unmapped); + const size_t used = physical - unmapped; + + Server::ResourceUsage usage; + usage.resource_pressure_ = used / static_cast(max_heap_); + + callbacks.onSuccess(usage); +} + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h new file mode 100644 index 0000000000000..9bc4bb2697c0a --- /dev/null +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h @@ -0,0 +1,44 @@ +#pragma once + +#include "envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.pb.validate.h" +#include "envoy/server/resource_monitor.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +/** + * Helper class for getting memory heap stats. + */ +class MemoryStatsReader { +public: + MemoryStatsReader() {} + virtual ~MemoryStatsReader() {} + + // Memory reserved for the process by the heap. + virtual uint64_t reservedHeapBytes(); + // Memory in free, unmapped pages in the page heap. + virtual uint64_t unmappedHeapBytes(); +}; + +/** + * Heap memory monitor with a statically configured maximum. + */ +class FixedHeapMonitor : public Server::ResourceMonitor { +public: + FixedHeapMonitor( + const envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig& config, + std::unique_ptr stats = std::make_unique()); + + void updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) override; + +private: + const uint64_t max_heap_; + std::unique_ptr stats_; +}; + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/well_known_names.h b/source/extensions/resource_monitors/well_known_names.h new file mode 100644 index 0000000000000..e573b74753fbc --- /dev/null +++ b/source/extensions/resource_monitors/well_known_names.h @@ -0,0 +1,23 @@ +#pragma once + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { + +/** + * Well-known resource monior names. + * NOTE: New resource monitors should use the well known name: envoy.resource_monitors.name. + */ +class ResourceMonitorNameValues { +public: + // Heap monitor with statically configured max. + const std::string FIXED_HEAP_RESOURCE_MONITOR = "envoy.resource_monitors.fixed_heap"; +}; + +typedef ConstSingleton ResourceMonitorNames; + +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index 6ff07ca41dd86..f80b6528f3220 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -241,6 +241,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "resource_monitor_config_lib", + hdrs = ["resource_monitor_config_impl.h"], + deps = [ + "//include/envoy/server:resource_monitor_config_interface", + ], +) + envoy_cc_library( name = "server_lib", srcs = ["server.cc"], diff --git a/source/server/resource_monitor_config_impl.h b/source/server/resource_monitor_config_impl.h new file mode 100644 index 0000000000000..2fcfcc443907b --- /dev/null +++ b/source/server/resource_monitor_config_impl.h @@ -0,0 +1,21 @@ +#pragma once + +#include "envoy/server/resource_monitor_config.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +class ResourceMonitorFactoryContextImpl : public ResourceMonitorFactoryContext { +public: + ResourceMonitorFactoryContextImpl(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} + + Event::Dispatcher& dispatcher() override { return dispatcher_; } + +private: + Event::Dispatcher& dispatcher_; +}; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/test/extensions/resource_monitors/fixed_heap/BUILD b/test/extensions/resource_monitors/fixed_heap/BUILD new file mode 100644 index 0000000000000..3d1c8eff0ab3b --- /dev/null +++ b/test/extensions/resource_monitors/fixed_heap/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "fixed_heap_monitor_test", + srcs = ["fixed_heap_monitor_test.cc"], + extension_name = "envoy.resource_monitors.fixed_heap", + external_deps = ["abseil_optional"], + deps = [ + "//source/extensions/resource_monitors/fixed_heap:fixed_heap_monitor", + ], +) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.resource_monitors.fixed_heap", + deps = [ + "//include/envoy/registry", + "//source/extensions/resource_monitors/fixed_heap:config", + "//source/server:resource_monitor_config_lib", + "//test/mocks/event:event_mocks", + "@envoy_api//envoy/config/resource_monitor/fixed_heap/v2alpha:fixed_heap_cc", + ], +) diff --git a/test/extensions/resource_monitors/fixed_heap/config_test.cc b/test/extensions/resource_monitors/fixed_heap/config_test.cc new file mode 100644 index 0000000000000..5bdd672804d2c --- /dev/null +++ b/test/extensions/resource_monitors/fixed_heap/config_test.cc @@ -0,0 +1,34 @@ +#include "envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "server/resource_monitor_config_impl.h" + +#include "extensions/resource_monitors/fixed_heap/config.h" + +#include "test/mocks/event/mocks.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +TEST(FixedHeapMonitorFactoryTest, CreateMonitor) { + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.resource_monitors.fixed_heap"); + EXPECT_NE(factory, nullptr); + + envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig config; + config.set_max_heap_size_bytes(std::numeric_limits::max()); + Event::MockDispatcher dispatcher; + Server::Configuration::ResourceMonitorFactoryContextImpl context(dispatcher); + auto monitor = factory->createResourceMonitor(config, context); + EXPECT_NE(monitor, nullptr); +} + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc new file mode 100644 index 0000000000000..637d5fe2a9433 --- /dev/null +++ b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc @@ -0,0 +1,56 @@ +#include "extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h" + +#include "absl/types/optional.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace FixedHeapMonitor { + +class MockMemoryStatsReader : public MemoryStatsReader { +public: + MockMemoryStatsReader() {} + + MOCK_METHOD0(reservedHeapBytes, uint64_t()); + MOCK_METHOD0(unmappedHeapBytes, uint64_t()); +}; + +class ResourcePressure : public Server::ResourceMonitor::Callbacks { +public: + void onSuccess(const Server::ResourceUsage& usage) override { + pressure_ = usage.resource_pressure_; + } + + void onFailure(const EnvoyException& error) override { error_ = error; } + + bool hasPressure() const { return pressure_.has_value(); } + bool hasError() const { return error_.has_value(); } + + double pressure() const { return *pressure_; } + +private: + absl::optional pressure_; + absl::optional error_; +}; + +TEST(FixedHeapMonitorTest, ComputesCorrectUsage) { + envoy::config::resource_monitor::fixed_heap::v2alpha::FixedHeapConfig config; + config.set_max_heap_size_bytes(1000); + auto stats_reader = std::make_unique(); + EXPECT_CALL(*stats_reader, reservedHeapBytes()).WillOnce(testing::Return(800)); + EXPECT_CALL(*stats_reader, unmappedHeapBytes()).WillOnce(testing::Return(100)); + std::unique_ptr monitor(new FixedHeapMonitor(config, std::move(stats_reader))); + + ResourcePressure resource; + monitor->updateResourceUsage(resource); + EXPECT_TRUE(resource.hasPressure()); + EXPECT_FALSE(resource.hasError()); + EXPECT_EQ(resource.pressure(), 0.7); +} + +} // namespace FixedHeapMonitor +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy