feat: Make macros operate on token streams instead of AST nodes

compiler/noirc_frontend/src/ast/expression.rs

@@ -7,7 +7,7 @@ use crate::ast::{
 };
 use crate::macros_api::StructId;
 use crate::node_interner::ExprId;
-use crate::token::{Attributes, Token};
+use crate::token::{Attributes, Token, Tokens};
 use acvm::{acir::AcirField, FieldElement};
 use iter_extended::vecmap;
 use noirc_errors::{Span, Spanned};
@@ -33,18 +33,10 @@ pub enum ExpressionKind {
     Tuple(Vec<Expression>),
     Lambda(Box<Lambda>),
     Parenthesized(Box<Expression>),
-    Quote(BlockExpression, Span),
+    Quote(Tokens),
     Unquote(Box<Expression>),
     Comptime(BlockExpression, Span),
 
-    /// Unquote expressions are replaced with UnquoteMarkers when Quoted
-    /// expressions are resolved. Since the expression being quoted remains an
-    /// ExpressionKind (rather than a resolved ExprId), the UnquoteMarker must be
-    /// here in the AST even though it is technically HIR-only.
-    /// Each usize in an UnquoteMarker is an index which corresponds to the index of the
-    /// expression in the Hir::Quote expression list to replace it with.
-    UnquoteMarker(usize),
-
     // This variant is only emitted when inlining the result of comptime
     // code. It is used to translate function values back into the AST while
     // guaranteeing they have the same instantiated type and definition id without resolving again.
@@ -557,12 +549,14 @@ impl Display for ExpressionKind {
             }
             Lambda(lambda) => lambda.fmt(f),
             Parenthesized(sub_expr) => write!(f, "({sub_expr})"),
-            Quote(block, _) => write!(f, "quote {block}"),
             Comptime(block, _) => write!(f, "comptime {block}"),
             Error => write!(f, "Error"),
             Resolved(_) => write!(f, "?Resolved"),
             Unquote(expr) => write!(f, "$({expr})"),
-            UnquoteMarker(index) => write!(f, "${index}"),
+            Quote(tokens) => {
+                let tokens = vecmap(&tokens.0, ToString::to_string);
+                write!(f, "quote {{ {} }}", tokens.join(" "))
+            }
         }
     }
 }

compiler/noirc_frontend/src/elaborator/expressions.rs

@@ -18,7 +18,7 @@ use crate::{
             HirArrayLiteral, HirBinaryOp, HirBlockExpression, HirCallExpression, HirCastExpression,
             HirConstructorExpression, HirIfExpression, HirIndexExpression, HirInfixExpression,
             HirLambda, HirMemberAccess, HirMethodCallExpression, HirMethodReference,
-            HirPrefixExpression, HirQuoted,
+            HirPrefixExpression,
         },
         traits::TraitConstraint,
     },
@@ -28,6 +28,7 @@ use crate::{
         MethodCallExpression, PrefixExpression,
     },
     node_interner::{DefinitionKind, ExprId, FuncId},
+    token::Tokens,
     QuotedType, Shared, StructType, Type,
 };
 
@@ -58,7 +59,7 @@ impl<'context> Elaborator<'context> {
             ExpressionKind::Tuple(tuple) => self.elaborate_tuple(tuple),
             ExpressionKind::Lambda(lambda) => self.elaborate_lambda(*lambda),
             ExpressionKind::Parenthesized(expr) => return self.elaborate_expression(*expr),
-            ExpressionKind::Quote(quote, _) => self.elaborate_quote(quote),
+            ExpressionKind::Quote(quote) => self.elaborate_quote(quote),
             ExpressionKind::Comptime(comptime, _) => {
                 return self.elaborate_comptime_block(comptime, expr.span)
             }
@@ -68,9 +69,6 @@ impl<'context> Elaborator<'context> {
                 self.push_err(ResolverError::UnquoteUsedOutsideQuote { span: expr.span });
                 (HirExpression::Error, Type::Error)
             }
-            ExpressionKind::UnquoteMarker(index) => {
-                unreachable!("UnquoteMarker({index}) remaining in runtime code")
-            }
         };
         let id = self.interner.push_expr(hir_expr);
         self.interner.push_expr_location(id, expr.span, self.file);
@@ -646,11 +644,9 @@ impl<'context> Elaborator<'context> {
         (expr, Type::Function(arg_types, Box::new(body_type), Box::new(env_type)))
     }
 
-    fn elaborate_quote(&mut self, mut block: BlockExpression) -> (HirExpression, Type) {
-        let mut unquoted_exprs = Vec::new();
-        self.find_unquoted_exprs_in_block(&mut block, &mut unquoted_exprs);
-        let quoted = HirQuoted { quoted_block: block, unquoted_exprs };
-        (HirExpression::Quote(quoted), Type::Quoted(QuotedType::Expr))
+    fn elaborate_quote(&mut self, mut tokens: Tokens) -> (HirExpression, Type) {
+        tokens = self.find_unquoted_exprs_tokens(tokens);
+        (HirExpression::Quote(tokens), Type::Quoted(QuotedType::Quoted))
     }
 
     fn elaborate_comptime_block(&mut self, block: BlockExpression, span: Span) -> (ExprId, Type) {
@@ -716,7 +712,7 @@ impl<'context> Elaborator<'context> {
         location: Location,
         return_type: Type,
     ) -> Option<(HirExpression, Type)> {
-        self.unify(&return_type, &Type::Quoted(QuotedType::Expr), || {
+        self.unify(&return_type, &Type::Quoted(QuotedType::Quoted), || {
             TypeCheckError::MacroReturningNonExpr { typ: return_type.clone(), span: location.span }
         });