Caching optimized IR

Changelog.md

@@ -19,6 +19,7 @@ Compiler Features:
  * Standard JSON Interface: Do not perform IR optimization when only unoptimized IR is requested.
  * Standard JSON Interface: Add ``transientStorageLayout`` output.
  * Yul: Drop the deprecated typed Yul dialect that was only accessible via ``--yul`` in the CLI.
+ * Yul Optimizer: Caching of optimized IR to speed up optimization of contracts with bytecode dependencies.
  * Yul Optimizer: The optimizer now treats some previously unrecognized identical literals as identical.
  * Commandline Interface: Allow the use of ``--asm-json`` output option in assembler mode to export EVM assembly of the contracts in JSON format.
 

libsolidity/interface/CompilerStack.cpp

@@ -111,6 +111,7 @@ static int g_compilerStackCounts = 0;
 
 CompilerStack::CompilerStack(ReadCallback::Callback _readFile):
 	m_readFile{std::move(_readFile)},
+	m_objectOptimizer(std::make_shared<yul::ObjectOptimizer>()),
 	m_errorReporter{m_errorList}
 {
 	// Because TypeProvider is currently a singleton API, we must ensure that
@@ -1493,7 +1494,8 @@ void CompilerStack::generateIR(ContractDefinition const& _contract, bool _unopti
 		m_eofVersion,
 		YulStack::Language::StrictAssembly,
 		m_optimiserSettings,
-		m_debugInfoSelection
+		m_debugInfoSelection,
+		m_objectOptimizer
 	);
 	bool yulAnalysisSuccessful = stack.parseAndAnalyze("", compiledContract.yulIR);
 	solAssert(
@@ -1530,7 +1532,8 @@ void CompilerStack::generateEVMFromIR(ContractDefinition const& _contract)
 		m_eofVersion,
 		yul::YulStack::Language::StrictAssembly,
 		m_optimiserSettings,
-		m_debugInfoSelection
+		m_debugInfoSelection,
+		m_objectOptimizer
 	);
 	bool analysisSuccessful = stack.parseAndAnalyze("", compiledContract.yulIROptimized);
 	solAssert(analysisSuccessful);

libsolidity/interface/CompilerStack.h

@@ -49,6 +49,8 @@
 #include <libsolutil/LazyInit.h>
 #include <libsolutil/JSON.h>
 
+#include <libyul/ObjectOptimizer.h>
+
 #include <functional>
 #include <memory>
 #include <ostream>
@@ -379,6 +381,8 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 		return VersionIsRelease ? MetadataFormat::WithReleaseVersionTag : MetadataFormat::WithPrereleaseVersionTag;
 	}
 
+	yul::ObjectOptimizer const& objectOptimizer() const { return *m_objectOptimizer; }
+
 private:
 	/// The state per source unit. Filled gradually during parsing.
 	struct Source
@@ -552,6 +556,7 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	std::shared_ptr<GlobalContext> m_globalContext;
 	std::vector<Source const*> m_sourceOrder;
 	std::map<std::string const, Contract> m_contracts;
+	std::shared_ptr<yul::ObjectOptimizer> m_objectOptimizer;
 
 	langutil::ErrorList m_errorList;
 	langutil::ErrorReporter m_errorReporter;

libyul/CMakeLists.txt

@@ -29,6 +29,8 @@ add_library(yul
 	FunctionReferenceResolver.h
 	Object.cpp
 	Object.h
+	ObjectOptimizer.cpp
+	ObjectOptimizer.h
 	ObjectParser.cpp
 	ObjectParser.h
 	Scope.cpp

libyul/Object.cpp

@@ -53,16 +53,6 @@ std::string Object::toString(
 	yulAssert(hasCode(), "No code");
 	yulAssert(debugData, "No debug data");
 
-	std::string useSrcComment;
-
-	if (debugData->sourceNames)
-		useSrcComment =
-			"/// @use-src " +
-			joinHumanReadable(ranges::views::transform(*debugData->sourceNames, [](auto&& _pair) {
-				return std::to_string(_pair.first) + ":" + util::escapeAndQuoteString(*_pair.second);
-			})) +
-			"\n";
-
 	std::string inner = "code " + AsmPrinter(
 		_printingMode,
 		_dialect,
@@ -74,7 +64,11 @@ std::string Object::toString(
 	for (auto const& obj: subObjects)
 		inner += "\n" + obj->toString(_dialect, _printingMode, _debugInfoSelection, _soliditySourceProvider);
 
-	return useSrcComment + "object \"" + name + "\" {\n" + indent(inner) + "\n}";
+	return
+		debugData->formatUseSrcComment() +
+		"object \"" + name + "\" {\n" +
+		indent(inner) + "\n" +
+		"}";
 }
 
 Json Data::toJson() const
@@ -85,6 +79,21 @@ Json Data::toJson() const
 	return ret;
 }
 
+std::string ObjectDebugData::formatUseSrcComment() const
+{
+	if (!sourceNames)
+		return "";
+
+	auto formatIdNamePair = [](auto&& _pair) {
+		return std::to_string(_pair.first) + ":" + util::escapeAndQuoteString(*_pair.second);
+	};
+
+	std::string serializedSourceNames = joinHumanReadable(
+		ranges::views::transform(*sourceNames, formatIdNamePair)
+	);
+	return "/// @use-src " + serializedSourceNames + "\n";
+}
+
 Json Object::toJson() const
 {
 	yulAssert(hasCode(), "No code");

libyul/Object.h

@@ -84,6 +84,8 @@ struct Data: public ObjectNode
 struct ObjectDebugData
 {
 	std::optional<SourceNameMap> sourceNames = {};
+
+	std::string formatUseSrcComment() const;
 };
 
 
@@ -99,9 +101,9 @@ struct Object: public ObjectNode
 		AsmPrinter::TypePrinting printingMode = AsmPrinter::TypePrinting::Full,
 		langutil::DebugInfoSelection const& _debugInfoSelection = langutil::DebugInfoSelection::Default(),
 		langutil::CharStreamProvider const* _soliditySourceProvider = nullptr
-	) const;
+	) const override;
 	/// @returns a compact JSON representation of the AST.
-	Json toJson() const;
+	Json toJson() const override;
 	/// @returns the set of names of data objects accessible from within the code of
 	/// this object, including the name of object itself
 	/// Handles all names containing dots as reserved identifiers, not accessible as data.

libyul/ObjectOptimizer.cpp

@@ -0,0 +1,167 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+
+#include <libyul/ObjectOptimizer.h>
+
+#include <libyul/AsmAnalysisInfo.h>
+#include <libyul/AsmAnalysis.h>
+#include <libyul/AsmPrinter.h>
+#include <libyul/AST.h>
+#include <libyul/Exceptions.h>
+#include <libyul/backends/evm/EVMDialect.h>
+#include <libyul/backends/evm/EVMMetrics.h>
+#include <libyul/optimiser/ASTCopier.h>
+#include <libyul/optimiser/Suite.h>
+
+#include <liblangutil/DebugInfoSelection.h>
+
+#include <libsolutil/Keccak256.h>
+
+#include <boost/algorithm/string.hpp>
+
+#include <limits>
+#include <numeric>
+
+using namespace solidity;
+using namespace solidity::langutil;
+using namespace solidity::util;
+using namespace solidity::yul;
+
+
+Dialect const& yul::languageToDialect(Language _language, EVMVersion _version)
+{
+	switch (_language)
+	{
+	case Language::Assembly:
+	case Language::StrictAssembly:
+		return EVMDialect::strictAssemblyForEVMObjects(_version);
+	}
+	util::unreachable();
+}
+
+void ObjectOptimizer::optimize(Object& _object, Settings const& _settings)
+{
+	yulAssert(_object.subId == std::numeric_limits<size_t>::max(), "Not a top-level object.");
+
+	optimize(_object, _settings, true /* _isCreation */);
+}
+
+void ObjectOptimizer::optimize(Object& _object, Settings const& _settings, bool _isCreation)
+{
+	yulAssert(_object.code());
+	yulAssert(_object.debugData);
+
+	for (auto& subNode: _object.subObjects)
+		if (auto subObject = dynamic_cast<Object*>(subNode.get()))
+		{
+			bool isCreation = !boost::ends_with(subObject->name, "_deployed");
+			optimize(
+				*subObject,
+				_settings,
+				isCreation
+			);
+		}
+
+	Dialect const& dialect = languageToDialect(_settings.language, _settings.evmVersion);
+	std::unique_ptr<GasMeter> meter;
+	if (EVMDialect const* evmDialect = dynamic_cast<EVMDialect const*>(&dialect))
+		meter = std::make_unique<GasMeter>(*evmDialect, _isCreation, _settings.expectedExecutionsPerDeployment);
+
+	std::optional<h256> cacheKey = calculateCacheKey(_object.code()->root(), *_object.debugData, _settings, _isCreation);
+	if (cacheKey.has_value() && m_cachedObjects.count(*cacheKey) != 0)
+	{
+		overwriteWithOptimizedObject(*cacheKey, _object);
+		return;
+	}
+
+	OptimiserSuite::run(
+		dialect,
+		meter.get(),
+		_object,
+		_settings.optimizeStackAllocation,
+		_settings.yulOptimiserSteps,
+		_settings.yulOptimiserCleanupSteps,
+		_isCreation ? std::nullopt : std::make_optional(_settings.expectedExecutionsPerDeployment),
+		{}
+	);
+
+	if (cacheKey.has_value())
+		storeOptimizedObject(*cacheKey, _object, dialect);
+}
+
+void ObjectOptimizer::storeOptimizedObject(util::h256 _cacheKey, Object const& _optimizedObject, Dialect const& _dialect)
+{
+	m_cachedObjects[_cacheKey] = CachedObject{
+		std::make_shared<Block>(ASTCopier{}.translate(_optimizedObject.code()->root())),
+		&_dialect,
+	};
+}
+
+void ObjectOptimizer::overwriteWithOptimizedObject(util::h256 _cacheKey, Object& _object) const
+{
+	yulAssert(m_cachedObjects.count(_cacheKey) != 0);
+	CachedObject const& cachedObject = m_cachedObjects.at(_cacheKey);
+
+	yulAssert(cachedObject.optimizedAST);
+	_object.setCode(std::make_shared<AST>(ASTCopier{}.translate(*cachedObject.optimizedAST)));
+	yulAssert(_object.code());
+
+	// There's no point in caching AnalysisInfo because it references AST nodes. It can't be shared
+	// by multiple ASTs and it's easier to recalculate it than properly clone it.
+	yulAssert(cachedObject.dialect);
+	_object.analysisInfo = std::make_shared<AsmAnalysisInfo>(
+		AsmAnalyzer::analyzeStrictAssertCorrect(
+			*cachedObject.dialect,
+			_object
+		)
+	);
+
+	// NOTE: Source name index is included in the key so it must be identical. No need to store and restore it.
+}
+
+std::optional<h256> ObjectOptimizer::calculateCacheKey(
+	Block const& _ast,
+	ObjectDebugData const& _debugData,
+	Settings const& _settings,
+	bool _isCreation
+)
+{
+	AsmPrinter asmPrinter(
+		AsmPrinter::TypePrinting::OmitDefault,
+		languageToDialect(_settings.language, _settings.evmVersion),
+		_debugData.sourceNames,
+		DebugInfoSelection::All()
+	);
+
+	bytes rawKey;
+	// NOTE: AsmPrinter never prints nativeLocations included in debug data, so ASTs differing only
+	// in that regard are considered equal here.  This is fine because the optimizer does not keep
+	// them up to date across AST transformations anyway so in any use where they need to be reliable,
+	// we just regenerate them by reparsing the object.
+	rawKey += keccak256(asmPrinter(_ast)).asBytes();
+	rawKey += keccak256(_debugData.formatUseSrcComment()).asBytes();
+	rawKey += h256(u256(_settings.language)).asBytes();
+	rawKey += FixedHash<1>(uint8_t(_settings.optimizeStackAllocation ? 0 : 1)).asBytes();
+	rawKey += h256(u256(_settings.expectedExecutionsPerDeployment)).asBytes();
+	rawKey += FixedHash<1>(uint8_t(_isCreation ? 0 : 1)).asBytes();
+	rawKey += keccak256(_settings.evmVersion.name()).asBytes();
+	rawKey += keccak256(_settings.yulOptimiserSteps).asBytes();
+	rawKey += keccak256(_settings.yulOptimiserCleanupSteps).asBytes();
+
+	return h256(keccak256(rawKey));
+}