feat(avm): sequential lookup resolution

barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.test.cpp

@@ -78,7 +78,7 @@ void complete_proving_key_for_test(bb::RelationParameters<FF>& relation_paramete
     relation_parameters.eccvm_set_permutation_delta = relation_parameters.eccvm_set_permutation_delta.invert();
 
     // Compute z_perm and inverse polynomial for our logarithmic-derivative lookup method
-    compute_logderivative_inverse<ECCVMFlavor, ECCVMFlavor::LookupRelation>(
+    compute_logderivative_inverse<FF, ECCVMFlavor::LookupRelation>(
         pk->polynomials, relation_parameters, pk->circuit_size);
     compute_grand_products<ECCVMFlavor>(pk->polynomials, relation_parameters);
 

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp

@@ -83,7 +83,7 @@ void ECCVMProver::execute_log_derivative_commitments_round()
         gamma * (gamma + beta_sqr) * (gamma + beta_sqr + beta_sqr) * (gamma + beta_sqr + beta_sqr + beta_sqr);
     relation_parameters.eccvm_set_permutation_delta = relation_parameters.eccvm_set_permutation_delta.invert();
     // Compute inverse polynomial for our logarithmic-derivative lookup method
-    compute_logderivative_inverse<Flavor, typename Flavor::LookupRelation>(
+    compute_logderivative_inverse<typename Flavor::FF, typename Flavor::LookupRelation>(
         key->polynomials, relation_parameters, key->circuit_size);
     transcript->send_to_verifier(commitment_labels.lookup_inverses,
                                  key->commitment_key->commit(key->polynomials.lookup_inverses));

barretenberg/cpp/src/barretenberg/eccvm/eccvm_trace_checker.cpp

@@ -31,7 +31,7 @@ bool ECCVMTraceChecker::check(Builder& builder, numeric::RNG* engine_ptr)
 
     ProverPolynomials polynomials(builder);
     const size_t num_rows = polynomials.get_polynomial_size();
-    compute_logderivative_inverse<Flavor, ECCVMLookupRelation<FF>>(polynomials, params, num_rows);
+    compute_logderivative_inverse<FF, ECCVMLookupRelation<FF>>(polynomials, params, num_rows);
     compute_grand_product<Flavor, ECCVMSetRelation<FF>>(polynomials, params);
 
     polynomials.z_perm_shift = Polynomial(polynomials.z_perm.shifted());

barretenberg/cpp/src/barretenberg/honk/proof_system/logderivative_library.hpp

@@ -1,4 +1,7 @@
 #pragma once
+
+#include "barretenberg/common/constexpr_utils.hpp"
+
 #include <typeinfo>
 
 namespace bb {
@@ -22,10 +25,9 @@ namespace bb {
  * The specific algebraic relations that define read terms and write terms are defined in Flavor::LookupRelation
  *
  */
-template <typename Flavor, typename Relation, typename Polynomials>
+template <typename FF, typename Relation, typename Polynomials>
 void compute_logderivative_inverse(Polynomials& polynomials, auto& relation_parameters, const size_t circuit_size)
 {
-    using FF = typename Flavor::FF;
     using Accumulator = typename Relation::ValueAccumulator0;
     constexpr size_t READ_TERMS = Relation::READ_TERMS;
     constexpr size_t WRITE_TERMS = Relation::WRITE_TERMS;

barretenberg/cpp/src/barretenberg/vm/avm/generated/circuit_builder.cpp

@@ -883,7 +883,7 @@ bool AvmCircuitBuilder::check_circuit() const
         using Relation = std::tuple_element_t<i, AvmFlavor::LookupRelations>;
         checks.push_back([&, num_rows](SignalErrorFn signal_error) {
             // Check the logderivative relation
-            bb::compute_logderivative_inverse<Flavor, Relation>(polys, params, num_rows);
+            bb::compute_logderivative_inverse<typename Flavor::FF, Relation>(polys, params, num_rows);
 
             typename Relation::SumcheckArrayOfValuesOverSubrelations lookup_result;
 

barretenberg/cpp/src/barretenberg/vm/avm/generated/prover.cpp

@@ -67,7 +67,7 @@ void AvmProver::execute_log_derivative_inverse_round()
         using Relation = std::tuple_element_t<relation_idx, Flavor::LookupRelations>;
         tasks.push_back([&]() {
             AVM_TRACK_TIME(std::string("prove/execute_log_derivative_inverse_round/") + std::string(Relation::NAME),
-                           (compute_logderivative_inverse<Flavor, Relation>(
+                           (compute_logderivative_inverse<FF, Relation>(
                                prover_polynomials, relation_parameters, key->circuit_size)));
         });
     });

barretenberg/cpp/src/barretenberg/vm/avm/tests/full_poseidon2.test.cpp

@@ -88,7 +88,7 @@ TEST(AvmFullPoseidon2, shouldHashCorrectly)
     using PermRelations = perm_pos2_fixed_pos2_perm_relation<FF>;
 
     // Check the logderivative relation
-    bb::compute_logderivative_inverse<AvmFlavor, PermRelations>(polys, params, num_rows);
+    bb::compute_logderivative_inverse<FF, PermRelations>(polys, params, num_rows);
 
     typename PermRelations::SumcheckArrayOfValuesOverSubrelations lookup_result;
 

barretenberg/cpp/src/barretenberg/vm/avm/tests/merkle_tree.test.cpp

@@ -109,7 +109,7 @@ TEST(AvmMerkleTree, shouldCheckMembership)
     using PermRelations = perm_merkle_poseidon2_relation<FF>;
 
     // Check the logderivative relation
-    bb::compute_logderivative_inverse<AvmFlavor, PermRelations>(polys, params, num_rows);
+    bb::compute_logderivative_inverse<FF, PermRelations>(polys, params, num_rows);
 
     typename PermRelations::SumcheckArrayOfValuesOverSubrelations lookup_result;
 

barretenberg/cpp/src/barretenberg/vm/avm/tests/range_check.test.cpp

@@ -118,7 +118,7 @@ TEST(AvmRangeCheck, shouldRangeCheck)
         using LookupRelations = std::tuple_element_t<i, AllLookupRelations>;
 
         // Check the logderivative relation
-        bb::compute_logderivative_inverse<AvmFlavor, LookupRelations>(polys, params, num_rows);
+        bb::compute_logderivative_inverse<FF, LookupRelations>(polys, params, num_rows);
 
         typename LookupRelations::SumcheckArrayOfValuesOverSubrelations lookup_result;
 

barretenberg/cpp/src/barretenberg/vm2/constraining/check_circuit.cpp

@@ -57,7 +57,7 @@ void run_check_circuit(AvmFlavor::ProverPolynomials& polys, size_t num_rows)
         using Relation = std::tuple_element_t<i, typename AvmFlavor::LookupRelations>;
         checks.push_back([&, num_rows]() {
             // Compute logderivs.
-            bb::compute_logderivative_inverse<AvmFlavor, Relation>(polys, params, num_rows);
+            bb::compute_logderivative_inverse<typename AvmFlavor::FF, Relation>(polys, params, num_rows);
 
             // Check the logderivative relation
             typename Relation::SumcheckArrayOfValuesOverSubrelations lookup_result{};

barretenberg/cpp/src/barretenberg/vm2/generated/prover.cpp

@@ -67,7 +67,7 @@ void AvmProver::execute_log_derivative_inverse_round()
         using Relation = std::tuple_element_t<relation_idx, Flavor::LookupRelations>;
         tasks.push_back([&]() {
             AVM_TRACK_TIME(std::string("prove/execute_log_derivative_inverse_round/") + std::string(Relation::NAME),
-                           (compute_logderivative_inverse<Flavor, Relation>(
+                           (compute_logderivative_inverse<FF, Relation>(
                                prover_polynomials, relation_parameters, key->circuit_size)));
         });
     });

barretenberg/cpp/src/barretenberg/vm2/tracegen/lib/lookup_builder.hpp

@@ -3,6 +3,7 @@
 #include <array>
 #include <bit>
 #include <cstddef>
+#include <cstdint>
 #include <stdexcept>
 
 #include "barretenberg/vm2/common/field.hpp"
@@ -54,9 +55,12 @@ template <typename LookupSettings_> class BaseLookupTraceBuilder : public Intera
 // This class is used when the lookup is into a non-precomputed table.
 // It calculates the counts by trying to find the tuple in the destination columns.
 // It creates an index of the destination columns on init, and uses it to find the tuple efficiently.
-template <typename LookupSettings_> class LookupIntoDynamicTable : public BaseLookupTraceBuilder<LookupSettings_> {
+// This class should work for any lookup that is not precomputed.
+// However, consider using a more specialized and faster class.
+template <typename LookupSettings_>
+class LookupIntoDynamicTableGeneric : public BaseLookupTraceBuilder<LookupSettings_> {
   public:
-    virtual ~LookupIntoDynamicTable() = default;
+    virtual ~LookupIntoDynamicTableGeneric() = default;
 
   protected:
     using LookupSettings = LookupSettings_;
@@ -89,6 +93,57 @@ template <typename LookupSettings_> class LookupIntoDynamicTable : public BaseLo
     unordered_flat_map<ArrayTuple, uint32_t> row_idx;
 };
 
+// This class is used when the lookup is into a non-precomputed table.
+// It is optimized for the case when the source and destination tuples
+// are expected to be in the same order (possibly with other tuples in the middle
+// in the destination table).
+// The approach is that for a given source row, you start sequentially looking at the
+// destination rows until you find a match. Then you move to the next source row.
+// Then you keep looking from the last destination row you found a match.
+// WARNING: Do not use this class if you expect to "reuse" destination rows.
+// In this case the two tables will likely not be in order.
+template <typename LookupSettings> class LookupIntoDynamicTableSequential : public InteractionBuilderInterface {
+  public:
+    ~LookupIntoDynamicTableSequential() override = default;
+
+    void process(TraceContainer& trace) override
+    {
+        uint32_t dst_row = 0;
+        uint32_t max_dst_row = trace.get_column_rows(LookupSettings::DST_SELECTOR);
+
+        trace.visit_column(LookupSettings::SRC_SELECTOR, [&](uint32_t row, const FF& src_sel_value) {
+            assert(src_sel_value == 1);
+            (void)src_sel_value; // Avoid GCC complaining of unused parameter when asserts are disabled.
+
+            auto src_values = trace.get_multiple(LookupSettings::SRC_COLUMNS, row);
+
+            // We set a dummy value in the inverse column so that the size of the column is right.
+            // The correct value will be set by the prover.
+            trace.set(LookupSettings::INVERSES, row, 0xdeadbeef);
+
+            // We find the first row in the destination columns where the values match.
+            while (dst_row < max_dst_row) {
+                // TODO: As an optimization, we could try to only walk the rows where the selector is active.
+                // We can't just do a visit because we cannot skip rows with that.
+                auto dst_selector = trace.get(LookupSettings::DST_SELECTOR, dst_row);
+                if (dst_selector == 1 && src_values == trace.get_multiple(LookupSettings::DST_COLUMNS, dst_row)) {
+                    trace.set(LookupSettings::COUNTS, dst_row, trace.get(LookupSettings::COUNTS, dst_row) + 1);
+                    return; // Done with this source row.
+                }
+                ++dst_row;
+            }
+
+            throw std::runtime_error("Failed computing counts for " + std::string(LookupSettings::NAME) +
+                                     ". Could not find tuple in destination.");
+        });
+
+        // We set a dummy value in the inverse column so that the size of the column is right.
+        // The correct value will be set by the prover.
+        trace.visit_column(LookupSettings::DST_SELECTOR,
+                           [&](uint32_t row, const FF&) { trace.set(LookupSettings::INVERSES, row, 0xdeadbeef); });
+    }
+};
+
 } // namespace bb::avm2::tracegen
 
 // Define a hash function for std::array so that it can be used as a key in a std::unordered_map.

barretenberg/cpp/src/barretenberg/vm2/tracegen_helper.cpp

@@ -183,7 +183,7 @@ TraceContainer AvmTraceGenHelper::generate_trace(EventsContainer&& events)
     {
         auto jobs_interactions = make_jobs<std::unique_ptr<InteractionBuilderInterface>>(
             std::make_unique<LookupIntoBitwise<lookup_dummy_precomputed_lookup_settings>>(),
-            std::make_unique<LookupIntoDynamicTable<lookup_dummy_dynamic_lookup_settings>>(),
+            std::make_unique<LookupIntoDynamicTableGeneric<lookup_dummy_dynamic_lookup_settings>>(),
             std::make_unique<PermutationBuilder<perm_dummy_dynamic_permutation_settings>>(),
             std::make_unique<LookupIntoIndexedByClk<lookup_rng_chk_diff_lookup_settings>>(),
             std::make_unique<LookupIntoIndexedByClk<lookup_rng_chk_pow_2_lookup_settings>>(),

bb-pilcom/bb-pil-backend/templates/circuit_builder.cpp.hbs

@@ -165,7 +165,7 @@ bool AvmCircuitBuilder::check_circuit() const {
         using Relation = std::tuple_element_t<i, AvmFlavor::LookupRelations>;
         checks.push_back([&, num_rows](SignalErrorFn signal_error) {
             // Check the logderivative relation
-            bb::compute_logderivative_inverse<Flavor, Relation>(polys, params, num_rows);
+            bb::compute_logderivative_inverse<typename Flavor::FF, Relation>(polys, params, num_rows);
 
             typename Relation::SumcheckArrayOfValuesOverSubrelations lookup_result;
 

bb-pilcom/bb-pil-backend/templates/prover.cpp.hbs

@@ -67,7 +67,7 @@ void AvmProver::execute_log_derivative_inverse_round()
         using Relation = std::tuple_element_t<relation_idx, Flavor::LookupRelations>;
         tasks.push_back([&]() {
             AVM_TRACK_TIME(std::string("prove/execute_log_derivative_inverse_round/") + std::string(Relation::NAME),
-                           (compute_logderivative_inverse<Flavor, Relation>(
+                           (compute_logderivative_inverse<FF, Relation>(
                                prover_polynomials, relation_parameters, key->circuit_size)));
         });
     });