store strings instead of tokens in nodes

Gemfile.lock

@@ -15,8 +15,7 @@ GEM
       power_assert
 
 PLATFORMS
-  arm64-darwin-21
-  x86_64-linux
+  ruby
 
 DEPENDENCIES
   bundler (~> 2)

bin/template

@@ -47,12 +47,12 @@ class NodeListParam < Struct.new(:name)
 end
 
 # This represents a parameter to a node that is a token. We pass them as
-# references and store them by copying.
-class TokenParam < Struct.new(:name)
-  def decl = "yp_token_t #{name}"
+# references and store by converting to a string.
+class StringParam < Struct.new(:name)
+  def decl = "yp_string_t #{name}"
   def param = "yp_token_t *#{name}"
-  def assign = "*#{name}"
-  def rbs_class = "Token"
+  def assign = "token_to_string(*#{name})"
+  def rbs_class = "String"
   def child_nodes = nil
 
   def start_location(params) = "#{name}->start - parser->start"
@@ -85,10 +85,10 @@ class NodeType
           OptionalNodeParam.new(name, $1)
         when "node[]"
           NodeListParam.new(name)
-        when "token"
-          TokenParam.new(name)
+        when "string"
+          StringParam.new(name)
         else
-          raise
+          raise "Unknown field type #{type.inspect}"
         end
       end
 

bin/templates/extension.c.erb

@@ -21,10 +21,10 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       for (size_t index = 0; index < node->as.<%= node.human %>.<%= param.name %>->size; index++) {
         rb_ary_push(argv[<%= index %>], node_new(parser, node->as.<%= node.human %>.<%= param.name %>->nodes[index]));
       }
-      <%- when TokenParam -%>
-      argv[<%= index %>] = token_new(parser, &node->as.<%= node.human %>.<%= param.name %>);
+      <%- when StringParam -%>
+      argv[<%= index %>] = string_new(parser, &node->as.<%= node.human %>.<%= param.name %>);
       <%- else -%>
-      <%- raise -%>
+      <%- raise "Unsupported param type #{param.class}" -%>
       <%- end -%>
       <%- end -%>
 

bin/templates/yarp.c.erb

@@ -31,7 +31,7 @@ yp_node_dealloc(yp_parser_t *parser, yp_node_t *node) {
     case <%= node.type %>:
       <%- node.params.each do |param| -%>
       <%- case param -%>
-      <%- when TokenParam -%>
+      <%- when StringParam -%>
       <%- when NodeParam -%>
       yp_node_dealloc(parser, node->as.<%= node.human %>.<%= param.name %>);
       <%- when OptionalNodeParam -%>

config.yml

@@ -278,45 +278,45 @@ nodes:
   - name: Assignment
     child_nodes:
       - node target
-      - token operator
+      - string operator
       - node value
     location: target->value
   - name: Binary
     child_nodes:
       - node left
-      - token operator
+      - string operator
       - node right
     location: left->right
   - name: CharacterLiteral
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: FloatLiteral
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: Identifier
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: IfModifier
     child_nodes:
       - node statement
-      - token keyword
+      - string keyword
       - node predicate
     location: statement->predicate
   - name: ImaginaryLiteral
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: IntegerLiteral
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: OperatorAssignment
     child_nodes:
       - node target
-      - token operator
+      - string operator
       - node value
     location: target->value
   - name: Program
@@ -325,21 +325,21 @@ nodes:
     location: statements
   - name: RationalLiteral
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: Range
     child_nodes:
       - node?operator left
-      - token operator
+      - string operator
       - node?operator right
     location: left->right
   - name: Redo
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: Retry
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: Statements
     child_nodes:
@@ -348,30 +348,30 @@ nodes:
   - name: Ternary
     child_nodes:
       - node predicate
-      - token question_mark
+      - string question_mark
       - node true_expression
-      - token colon
+      - string colon
       - node false_expression
     location: predicate->false_expression
   - name: UnlessModifier
     child_nodes:
       - node statement
-      - token keyword
+      - string keyword
       - node predicate
     location: statement->predicate
   - name: UntilModifier
     child_nodes:
       - node statement
-      - token keyword
+      - string keyword
       - node predicate
     location: statement->predicate
   - name: VariableReference
     child_nodes:
-      - token value
+      - string value
     location: value
   - name: WhileModifier
     child_nodes:
       - node statement
-      - token keyword
+      - string keyword
       - node predicate
     location: statement->predicate

ext/yarp/extconf.rb

@@ -2,4 +2,8 @@
 
 require "mkmf"
 $INCFLAGS << " -I$(top_srcdir)" if $extmk
+
+# There are complains about missing memcpy and strlen
+$CFLAGS << ' -Wno-implicit-function-declaration'
+
 create_makefile "yarp/yarp"

ext/yarp/extension.c

@@ -1,3 +1,5 @@
+#include <string.h>
+
 #include "gen_token_type.h"
 #include "yarp.h"
 #include <ruby.h>
@@ -39,6 +41,11 @@ token_new(yp_parser_t *parser, yp_token_t *token) {
   return rb_class_new_instance(3, argv, rb_cYARPToken);
 }
 
+static VALUE
+string_new(yp_parser_t *parser, yp_string_t *string) {
+  return rb_str_new(string_ptr(string), string_length(string));
+}
+
 /******************************************************************************/
 /* BEGIN TEMPLATE                                                             */
 /******************************************************************************/
@@ -53,7 +60,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.assignment.target);
 
       // operator
-      argv[1] = token_new(parser, &node->as.assignment.operator);
+      argv[1] = string_new(parser, &node->as.assignment.operator);
 
       // value
       argv[2] = node_new(parser, node->as.assignment.value);
@@ -70,7 +77,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.binary.left);
 
       // operator
-      argv[1] = token_new(parser, &node->as.binary.operator);
+      argv[1] = string_new(parser, &node->as.binary.operator);
 
       // right
       argv[2] = node_new(parser, node->as.binary.right);
@@ -84,7 +91,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.character_literal.value);
+      argv[0] = string_new(parser, &node->as.character_literal.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -95,7 +102,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.float_literal.value);
+      argv[0] = string_new(parser, &node->as.float_literal.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -106,7 +113,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.identifier.value);
+      argv[0] = string_new(parser, &node->as.identifier.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -120,7 +127,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.if_modifier.statement);
 
       // keyword
-      argv[1] = token_new(parser, &node->as.if_modifier.keyword);
+      argv[1] = string_new(parser, &node->as.if_modifier.keyword);
 
       // predicate
       argv[2] = node_new(parser, node->as.if_modifier.predicate);
@@ -134,7 +141,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.imaginary_literal.value);
+      argv[0] = string_new(parser, &node->as.imaginary_literal.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -145,7 +152,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.integer_literal.value);
+      argv[0] = string_new(parser, &node->as.integer_literal.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -159,7 +166,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.operator_assignment.target);
 
       // operator
-      argv[1] = token_new(parser, &node->as.operator_assignment.operator);
+      argv[1] = string_new(parser, &node->as.operator_assignment.operator);
 
       // value
       argv[2] = node_new(parser, node->as.operator_assignment.value);
@@ -187,7 +194,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node->as.range.left == NULL ? Qnil : node_new(parser, node->as.range.left);
 
       // operator
-      argv[1] = token_new(parser, &node->as.range.operator);
+      argv[1] = string_new(parser, &node->as.range.operator);
 
       // right
       argv[2] = node->as.range.right == NULL ? Qnil : node_new(parser, node->as.range.right);
@@ -201,7 +208,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.rational_literal.value);
+      argv[0] = string_new(parser, &node->as.rational_literal.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -212,7 +219,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.redo.value);
+      argv[0] = string_new(parser, &node->as.redo.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -223,7 +230,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.retry.value);
+      argv[0] = string_new(parser, &node->as.retry.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -251,13 +258,13 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.ternary.predicate);
 
       // question_mark
-      argv[1] = token_new(parser, &node->as.ternary.question_mark);
+      argv[1] = string_new(parser, &node->as.ternary.question_mark);
 
       // true_expression
       argv[2] = node_new(parser, node->as.ternary.true_expression);
 
       // colon
-      argv[3] = token_new(parser, &node->as.ternary.colon);
+      argv[3] = string_new(parser, &node->as.ternary.colon);
 
       // false_expression
       argv[4] = node_new(parser, node->as.ternary.false_expression);
@@ -274,7 +281,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.unless_modifier.statement);
 
       // keyword
-      argv[1] = token_new(parser, &node->as.unless_modifier.keyword);
+      argv[1] = string_new(parser, &node->as.unless_modifier.keyword);
 
       // predicate
       argv[2] = node_new(parser, node->as.unless_modifier.predicate);
@@ -291,7 +298,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.until_modifier.statement);
 
       // keyword
-      argv[1] = token_new(parser, &node->as.until_modifier.keyword);
+      argv[1] = string_new(parser, &node->as.until_modifier.keyword);
 
       // predicate
       argv[2] = node_new(parser, node->as.until_modifier.predicate);
@@ -305,7 +312,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       VALUE argv[2];
 
       // value
-      argv[0] = token_new(parser, &node->as.variable_reference.value);
+      argv[0] = string_new(parser, &node->as.variable_reference.value);
 
       // location
       argv[1] = location_new(&node->location);
@@ -319,7 +326,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) {
       argv[0] = node_new(parser, node->as.while_modifier.statement);
 
       // keyword
-      argv[1] = token_new(parser, &node->as.while_modifier.keyword);
+      argv[1] = string_new(parser, &node->as.while_modifier.keyword);
 
       // predicate
       argv[2] = node_new(parser, node->as.while_modifier.predicate);