From addd498fb948752705838bf0b8df7c75f11481f1 Mon Sep 17 00:00:00 2001 From: kares Date: Tue, 23 Mar 2021 07:35:06 +0100 Subject: [PATCH 1/2] Build: reduce plugin version script memory usage --- rakelib/plugins_docs_dependencies.rake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rakelib/plugins_docs_dependencies.rake b/rakelib/plugins_docs_dependencies.rake index c01d1d27d82..f2d053fc16f 100644 --- a/rakelib/plugins_docs_dependencies.rake +++ b/rakelib/plugins_docs_dependencies.rake @@ -126,6 +126,7 @@ class PluginVersionWorking end def try_plugin(plugin, successful_dependencies) + Bundler::DepProxy.__clear! builder = Bundler::Dsl.new gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load gemfile.update(plugin) @@ -203,6 +204,14 @@ task :generate_plugins_version do end end end + DepProxy.class_eval do + # Bundler caches it's dep-proxy objects (which contain Gem::Dependency objects) from all resolutions. + # The Hash itself continues to grow between dependency resolutions and hold up a lot of memory, to avoid + # the issue we expose a way of clear-ing the cached objects before each plugin resolution. + def self.__clear! + @proxies.clear + end + end end PluginVersionWorking.new.generate From e49b589f500a60c3dcd82e6e63fccd8e163e543f Mon Sep 17 00:00:00 2001 From: kares Date: Tue, 23 Mar 2021 07:35:56 +0100 Subject: [PATCH 2/2] Build: further reduce pressure from RGs index building --- rakelib/plugins_docs_dependencies.rake | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rakelib/plugins_docs_dependencies.rake b/rakelib/plugins_docs_dependencies.rake index f2d053fc16f..b9ef6224141 100644 --- a/rakelib/plugins_docs_dependencies.rake +++ b/rakelib/plugins_docs_dependencies.rake @@ -212,6 +212,21 @@ task :generate_plugins_version do @proxies.clear end end + + Fetcher::CompactIndex.class_eval do + alias_method :__bundle_worker, :bundle_worker + # The compact index is built using multiple threads and this is hard-coded atm to 25 threads: + # `Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func)` + # each thread might built up a Bundler::Source::Rubygems object which retains more than 100MB. + # By limiting the worker thread count we make sure not to produce too many of these objects. + def bundle_worker(func = nil) + __bundle_worker(func).tap do |worker| + size = worker.instance_variable_get(:@size) + fail("@size = #{size.inspect} is no longer an integer") unless size.is_a?(Integer) + worker.instance_variable_set(:@size, 2) + end + end + end end PluginVersionWorking.new.generate