diff --git a/Gemfile.lock b/Gemfile.lock index bfb8e4425d..1d2b74cbc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -15,8 +15,7 @@ GEM power_assert PLATFORMS - arm64-darwin-21 - x86_64-linux + ruby DEPENDENCIES bundler (~> 2) diff --git a/bin/template b/bin/template index 2467e79a45..e8f1a8b8b8 100755 --- a/bin/template +++ b/bin/template @@ -47,12 +47,12 @@ class NodeListParam < Struct.new(:name) end # This represents a parameter to a node that is a token. We pass them as -# references and store them by copying. -class TokenParam < Struct.new(:name) - def decl = "yp_token_t #{name}" +# references and store by converting to a string. +class StringParam < Struct.new(:name) + def decl = "yp_string_t #{name}" def param = "yp_token_t *#{name}" - def assign = "*#{name}" - def rbs_class = "Token" + def assign = "token_to_string(*#{name})" + def rbs_class = "String" def child_nodes = nil def start_location(params) = "#{name}->start - parser->start" @@ -85,10 +85,10 @@ class NodeType OptionalNodeParam.new(name, $1) when "node[]" NodeListParam.new(name) - when "token" - TokenParam.new(name) + when "string" + StringParam.new(name) else - raise + raise "Unknown field type #{type.inspect}" end end diff --git a/bin/templates/extension.c.erb b/bin/templates/extension.c.erb index 51912d70db..70cebc374f 100644 --- a/bin/templates/extension.c.erb +++ b/bin/templates/extension.c.erb @@ -21,10 +21,10 @@ node_new(yp_parser_t *parser, yp_node_t *node) { for (size_t index = 0; index < node->as.<%= node.human %>.<%= param.name %>->size; index++) { rb_ary_push(argv[<%= index %>], node_new(parser, node->as.<%= node.human %>.<%= param.name %>->nodes[index])); } - <%- when TokenParam -%> - argv[<%= index %>] = token_new(parser, &node->as.<%= node.human %>.<%= param.name %>); + <%- when StringParam -%> + argv[<%= index %>] = string_new(parser, &node->as.<%= node.human %>.<%= param.name %>); <%- else -%> - <%- raise -%> + <%- raise "Unsupported param type #{param.class}" -%> <%- end -%> <%- end -%> diff --git a/bin/templates/yarp.c.erb b/bin/templates/yarp.c.erb index 48de922a4f..0b73a0c4f8 100644 --- a/bin/templates/yarp.c.erb +++ b/bin/templates/yarp.c.erb @@ -31,7 +31,7 @@ yp_node_dealloc(yp_parser_t *parser, yp_node_t *node) { case <%= node.type %>: <%- node.params.each do |param| -%> <%- case param -%> - <%- when TokenParam -%> + <%- when StringParam -%> <%- when NodeParam -%> yp_node_dealloc(parser, node->as.<%= node.human %>.<%= param.name %>); <%- when OptionalNodeParam -%> diff --git a/config.yml b/config.yml index 003cbc88c1..6b5771abfc 100644 --- a/config.yml +++ b/config.yml @@ -278,45 +278,45 @@ nodes: - name: Assignment child_nodes: - node target - - token operator + - string operator - node value location: target->value - name: Binary child_nodes: - node left - - token operator + - string operator - node right location: left->right - name: CharacterLiteral child_nodes: - - token value + - string value location: value - name: FloatLiteral child_nodes: - - token value + - string value location: value - name: Identifier child_nodes: - - token value + - string value location: value - name: IfModifier child_nodes: - node statement - - token keyword + - string keyword - node predicate location: statement->predicate - name: ImaginaryLiteral child_nodes: - - token value + - string value location: value - name: IntegerLiteral child_nodes: - - token value + - string value location: value - name: OperatorAssignment child_nodes: - node target - - token operator + - string operator - node value location: target->value - name: Program @@ -325,21 +325,21 @@ nodes: location: statements - name: RationalLiteral child_nodes: - - token value + - string value location: value - name: Range child_nodes: - node?operator left - - token operator + - string operator - node?operator right location: left->right - name: Redo child_nodes: - - token value + - string value location: value - name: Retry child_nodes: - - token value + - string value location: value - name: Statements child_nodes: @@ -348,30 +348,30 @@ nodes: - name: Ternary child_nodes: - node predicate - - token question_mark + - string question_mark - node true_expression - - token colon + - string colon - node false_expression location: predicate->false_expression - name: UnlessModifier child_nodes: - node statement - - token keyword + - string keyword - node predicate location: statement->predicate - name: UntilModifier child_nodes: - node statement - - token keyword + - string keyword - node predicate location: statement->predicate - name: VariableReference child_nodes: - - token value + - string value location: value - name: WhileModifier child_nodes: - node statement - - token keyword + - string keyword - node predicate location: statement->predicate diff --git a/ext/yarp/extconf.rb b/ext/yarp/extconf.rb index 553b529d42..5f6df316ae 100644 --- a/ext/yarp/extconf.rb +++ b/ext/yarp/extconf.rb @@ -2,4 +2,8 @@ require "mkmf" $INCFLAGS << " -I$(top_srcdir)" if $extmk + +# There are complains about missing memcpy and strlen +$CFLAGS << ' -Wno-implicit-function-declaration' + create_makefile "yarp/yarp" diff --git a/ext/yarp/extension.c b/ext/yarp/extension.c index b761a223ef..cfc23d1447 100644 --- a/ext/yarp/extension.c +++ b/ext/yarp/extension.c @@ -1,3 +1,5 @@ +#include + #include "gen_token_type.h" #include "yarp.h" #include @@ -39,6 +41,11 @@ token_new(yp_parser_t *parser, yp_token_t *token) { return rb_class_new_instance(3, argv, rb_cYARPToken); } +static VALUE +string_new(yp_parser_t *parser, yp_string_t *string) { + return rb_str_new(string_ptr(string), string_length(string)); +} + /******************************************************************************/ /* BEGIN TEMPLATE */ /******************************************************************************/ @@ -53,7 +60,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.assignment.target); // operator - argv[1] = token_new(parser, &node->as.assignment.operator); + argv[1] = string_new(parser, &node->as.assignment.operator); // value argv[2] = node_new(parser, node->as.assignment.value); @@ -70,7 +77,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.binary.left); // operator - argv[1] = token_new(parser, &node->as.binary.operator); + argv[1] = string_new(parser, &node->as.binary.operator); // right argv[2] = node_new(parser, node->as.binary.right); @@ -84,7 +91,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.character_literal.value); + argv[0] = string_new(parser, &node->as.character_literal.value); // location argv[1] = location_new(&node->location); @@ -95,7 +102,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.float_literal.value); + argv[0] = string_new(parser, &node->as.float_literal.value); // location argv[1] = location_new(&node->location); @@ -106,7 +113,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.identifier.value); + argv[0] = string_new(parser, &node->as.identifier.value); // location argv[1] = location_new(&node->location); @@ -120,7 +127,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.if_modifier.statement); // keyword - argv[1] = token_new(parser, &node->as.if_modifier.keyword); + argv[1] = string_new(parser, &node->as.if_modifier.keyword); // predicate argv[2] = node_new(parser, node->as.if_modifier.predicate); @@ -134,7 +141,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.imaginary_literal.value); + argv[0] = string_new(parser, &node->as.imaginary_literal.value); // location argv[1] = location_new(&node->location); @@ -145,7 +152,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.integer_literal.value); + argv[0] = string_new(parser, &node->as.integer_literal.value); // location argv[1] = location_new(&node->location); @@ -159,7 +166,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.operator_assignment.target); // operator - argv[1] = token_new(parser, &node->as.operator_assignment.operator); + argv[1] = string_new(parser, &node->as.operator_assignment.operator); // value argv[2] = node_new(parser, node->as.operator_assignment.value); @@ -187,7 +194,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node->as.range.left == NULL ? Qnil : node_new(parser, node->as.range.left); // operator - argv[1] = token_new(parser, &node->as.range.operator); + argv[1] = string_new(parser, &node->as.range.operator); // right argv[2] = node->as.range.right == NULL ? Qnil : node_new(parser, node->as.range.right); @@ -201,7 +208,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.rational_literal.value); + argv[0] = string_new(parser, &node->as.rational_literal.value); // location argv[1] = location_new(&node->location); @@ -212,7 +219,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.redo.value); + argv[0] = string_new(parser, &node->as.redo.value); // location argv[1] = location_new(&node->location); @@ -223,7 +230,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.retry.value); + argv[0] = string_new(parser, &node->as.retry.value); // location argv[1] = location_new(&node->location); @@ -251,13 +258,13 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.ternary.predicate); // question_mark - argv[1] = token_new(parser, &node->as.ternary.question_mark); + argv[1] = string_new(parser, &node->as.ternary.question_mark); // true_expression argv[2] = node_new(parser, node->as.ternary.true_expression); // colon - argv[3] = token_new(parser, &node->as.ternary.colon); + argv[3] = string_new(parser, &node->as.ternary.colon); // false_expression argv[4] = node_new(parser, node->as.ternary.false_expression); @@ -274,7 +281,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.unless_modifier.statement); // keyword - argv[1] = token_new(parser, &node->as.unless_modifier.keyword); + argv[1] = string_new(parser, &node->as.unless_modifier.keyword); // predicate argv[2] = node_new(parser, node->as.unless_modifier.predicate); @@ -291,7 +298,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.until_modifier.statement); // keyword - argv[1] = token_new(parser, &node->as.until_modifier.keyword); + argv[1] = string_new(parser, &node->as.until_modifier.keyword); // predicate argv[2] = node_new(parser, node->as.until_modifier.predicate); @@ -305,7 +312,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { VALUE argv[2]; // value - argv[0] = token_new(parser, &node->as.variable_reference.value); + argv[0] = string_new(parser, &node->as.variable_reference.value); // location argv[1] = location_new(&node->location); @@ -319,7 +326,7 @@ node_new(yp_parser_t *parser, yp_node_t *node) { argv[0] = node_new(parser, node->as.while_modifier.statement); // keyword - argv[1] = token_new(parser, &node->as.while_modifier.keyword); + argv[1] = string_new(parser, &node->as.while_modifier.keyword); // predicate argv[2] = node_new(parser, node->as.while_modifier.predicate); diff --git a/ext/yarp/gen_node.h b/ext/yarp/gen_node.h index e91a2a4db5..50f579b005 100644 --- a/ext/yarp/gen_node.h +++ b/ext/yarp/gen_node.h @@ -60,53 +60,53 @@ typedef struct yp_node { // Assignment struct { struct yp_node *target; - yp_token_t operator; + yp_string_t operator; struct yp_node *value; } assignment; // Binary struct { struct yp_node *left; - yp_token_t operator; + yp_string_t operator; struct yp_node *right; } binary; // CharacterLiteral struct { - yp_token_t value; + yp_string_t value; } character_literal; // FloatLiteral struct { - yp_token_t value; + yp_string_t value; } float_literal; // Identifier struct { - yp_token_t value; + yp_string_t value; } identifier; // IfModifier struct { struct yp_node *statement; - yp_token_t keyword; + yp_string_t keyword; struct yp_node *predicate; } if_modifier; // ImaginaryLiteral struct { - yp_token_t value; + yp_string_t value; } imaginary_literal; // IntegerLiteral struct { - yp_token_t value; + yp_string_t value; } integer_literal; // OperatorAssignment struct { struct yp_node *target; - yp_token_t operator; + yp_string_t operator; struct yp_node *value; } operator_assignment; @@ -118,23 +118,23 @@ typedef struct yp_node { // Range struct { struct yp_node *left; - yp_token_t operator; + yp_string_t operator; struct yp_node *right; } range; // RationalLiteral struct { - yp_token_t value; + yp_string_t value; } rational_literal; // Redo struct { - yp_token_t value; + yp_string_t value; } redo; // Retry struct { - yp_token_t value; + yp_string_t value; } retry; // Statements @@ -145,35 +145,35 @@ typedef struct yp_node { // Ternary struct { struct yp_node *predicate; - yp_token_t question_mark; + yp_string_t question_mark; struct yp_node *true_expression; - yp_token_t colon; + yp_string_t colon; struct yp_node *false_expression; } ternary; // UnlessModifier struct { struct yp_node *statement; - yp_token_t keyword; + yp_string_t keyword; struct yp_node *predicate; } unless_modifier; // UntilModifier struct { struct yp_node *statement; - yp_token_t keyword; + yp_string_t keyword; struct yp_node *predicate; } until_modifier; // VariableReference struct { - yp_token_t value; + yp_string_t value; } variable_reference; // WhileModifier struct { struct yp_node *statement; - yp_token_t keyword; + yp_string_t keyword; struct yp_node *predicate; } while_modifier; } as; diff --git a/ext/yarp/string.c b/ext/yarp/string.c new file mode 100644 index 0000000000..2678c609f5 --- /dev/null +++ b/ext/yarp/string.c @@ -0,0 +1,47 @@ +#include "string.h" +#include + +yp_string_t +new_string_shared(const char *start, const char *end) { + return (yp_string_t) { .type = STRING_SHARED, .as.shared = { .start = start, .end = end } }; +} + +yp_string_t +new_string_owned(char *ptr, size_t length) { + return (yp_string_t) { .type = STRING_OWNED, .as.owned = { .ptr = ptr, .length = length } }; +} + +bool +is_string_shared(const yp_string_t *string) { + return string->type == STRING_SHARED; +} + +bool +is_string_owned(const yp_string_t *string) { + return string->type == STRING_OWNED; +} + +size_t +string_length(const yp_string_t *string) { + if (string->type == STRING_SHARED) { + return string->as.shared.end - string->as.shared.start; + } else { + return string->as.owned.length; + } +} + +const char * +string_ptr(const yp_string_t *string) { + if (string->type == STRING_SHARED) { + return string->as.shared.start; + } else { + return string->as.owned.ptr; + } +} + +void +string_drop(yp_string_t *string) { + if (string->type == STRING_OWNED) { + free(string->as.owned.ptr); + } +} diff --git a/ext/yarp/string.h b/ext/yarp/string.h new file mode 100644 index 0000000000..686e043f42 --- /dev/null +++ b/ext/yarp/string.h @@ -0,0 +1,55 @@ +#ifndef YARP_STRING_H +#define YARP_STRING_H + +#include +#include + +typedef struct { + enum { + STRING_SHARED, + STRING_OWNED, + } type; + + union { + struct { + const char *start; + const char *end; + } shared; + + struct { + char *ptr; + size_t length; + } owned; + } as; +} yp_string_t; + +// Constructs shared string that is based on initial input +yp_string_t +new_string_shared(const char *start, const char *end); + +// Constructs owned string that contains owned value +yp_string_t +new_string_owned(char *ptr, size_t length); + +// Returns true if string is shared +bool +is_string_shared(const yp_string_t *string); + +// Returns true if string is owned +bool +is_string_owned(const yp_string_t *string); + +// Returns length of the string +size_t +string_length(const yp_string_t *string); + +// Returns read-only pointer of the string +const char * +string_ptr(const yp_string_t *string); + +// Destructor of a string, de-allocates internal state of the string +// if needed (i.e. if string is owned) +void +string_drop(yp_string_t *string); + +#endif // YARP_STRING_H diff --git a/ext/yarp/token.c b/ext/yarp/token.c new file mode 100644 index 0000000000..ac25622659 --- /dev/null +++ b/ext/yarp/token.c @@ -0,0 +1,6 @@ +#include "token.h" + +yp_string_t +token_to_string(yp_token_t token) { + return new_string_shared(token.start, token.end); +} diff --git a/ext/yarp/token.h b/ext/yarp/token.h index edc8cc4aa9..beeab29444 100644 --- a/ext/yarp/token.h +++ b/ext/yarp/token.h @@ -2,6 +2,7 @@ #define YARP_TOKEN_H #include "gen_token_type.h" +#include "string.h" // This struct represents a token in the Ruby source. We use it to track both // type and location information. @@ -11,4 +12,8 @@ typedef struct { const char *end; } yp_token_t; +// Creates a string based on token +yp_string_t +token_to_string(yp_token_t token); + #endif // YARP_TOKEN_H diff --git a/ext/yarp/yarp.c b/ext/yarp/yarp.c index b53d020808..88b6da9670 100644 --- a/ext/yarp/yarp.c +++ b/ext/yarp/yarp.c @@ -982,7 +982,7 @@ yp_node_alloc_assignment(yp_parser_t *parser, yp_node_t *target, yp_token_t *ope .location = { .start = target->location.start, .end = value->location.end }, .as.assignment = { .target = target, - .operator = *operator, + .operator = token_to_string(*operator), .value = value, }, }; @@ -998,7 +998,7 @@ yp_node_alloc_binary(yp_parser_t *parser, yp_node_t *left, yp_token_t *operator, .location = { .start = left->location.start, .end = right->location.end }, .as.binary = { .left = left, - .operator = *operator, + .operator = token_to_string(*operator), .right = right, }, }; @@ -1013,7 +1013,7 @@ yp_node_alloc_character_literal(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_CHARACTER_LITERAL, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.character_literal = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1027,7 +1027,7 @@ yp_node_alloc_float_literal(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_FLOAT_LITERAL, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.float_literal = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1041,7 +1041,7 @@ yp_node_alloc_identifier(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_IDENTIFIER, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.identifier = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1056,7 +1056,7 @@ yp_node_alloc_if_modifier(yp_parser_t *parser, yp_node_t *statement, yp_token_t .location = { .start = statement->location.start, .end = predicate->location.end }, .as.if_modifier = { .statement = statement, - .keyword = *keyword, + .keyword = token_to_string(*keyword), .predicate = predicate, }, }; @@ -1071,7 +1071,7 @@ yp_node_alloc_imaginary_literal(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_IMAGINARY_LITERAL, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.imaginary_literal = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1085,7 +1085,7 @@ yp_node_alloc_integer_literal(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_INTEGER_LITERAL, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.integer_literal = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1100,7 +1100,7 @@ yp_node_alloc_operator_assignment(yp_parser_t *parser, yp_node_t *target, yp_tok .location = { .start = target->location.start, .end = value->location.end }, .as.operator_assignment = { .target = target, - .operator = *operator, + .operator = token_to_string(*operator), .value = value, }, }; @@ -1130,7 +1130,7 @@ yp_node_alloc_range(yp_parser_t *parser, yp_node_t *left, yp_token_t *operator, .location = { .start = (left == NULL ? operator->start - parser->start : left->location.start), .end = (right == NULL ? operator->end - parser->start : right->location.end) }, .as.range = { .left = left, - .operator = *operator, + .operator = token_to_string(*operator), .right = right, }, }; @@ -1145,7 +1145,7 @@ yp_node_alloc_rational_literal(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_RATIONAL_LITERAL, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.rational_literal = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1159,7 +1159,7 @@ yp_node_alloc_redo(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_REDO, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.redo = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1173,7 +1173,7 @@ yp_node_alloc_retry(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_RETRY, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.retry = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1203,9 +1203,9 @@ yp_node_alloc_ternary(yp_parser_t *parser, yp_node_t *predicate, yp_token_t *que .location = { .start = predicate->location.start, .end = false_expression->location.end }, .as.ternary = { .predicate = predicate, - .question_mark = *question_mark, + .question_mark = token_to_string(*question_mark), .true_expression = true_expression, - .colon = *colon, + .colon = token_to_string(*colon), .false_expression = false_expression, }, }; @@ -1221,7 +1221,7 @@ yp_node_alloc_unless_modifier(yp_parser_t *parser, yp_node_t *statement, yp_toke .location = { .start = statement->location.start, .end = predicate->location.end }, .as.unless_modifier = { .statement = statement, - .keyword = *keyword, + .keyword = token_to_string(*keyword), .predicate = predicate, }, }; @@ -1237,7 +1237,7 @@ yp_node_alloc_until_modifier(yp_parser_t *parser, yp_node_t *statement, yp_token .location = { .start = statement->location.start, .end = predicate->location.end }, .as.until_modifier = { .statement = statement, - .keyword = *keyword, + .keyword = token_to_string(*keyword), .predicate = predicate, }, }; @@ -1252,7 +1252,7 @@ yp_node_alloc_variable_reference(yp_parser_t *parser, yp_token_t *value) { .type = YP_NODE_VARIABLE_REFERENCE, .location = { .start = value->start - parser->start, .end = value->end - parser->start }, .as.variable_reference = { - .value = *value, + .value = token_to_string(*value), }, }; return node; @@ -1267,7 +1267,7 @@ yp_node_alloc_while_modifier(yp_parser_t *parser, yp_node_t *statement, yp_token .location = { .start = statement->location.start, .end = predicate->location.end }, .as.while_modifier = { .statement = statement, - .keyword = *keyword, + .keyword = token_to_string(*keyword), .predicate = predicate, }, }; diff --git a/lib/yarp.rb b/lib/yarp.rb index e87e345477..623714debb 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -95,7 +95,7 @@ class Assignment < Node # attr_reader target: Node attr_reader :target - # attr_reader operator: Token + # attr_reader operator: String attr_reader :operator # attr_reader value: Node @@ -104,7 +104,7 @@ class Assignment < Node # attr_reader location: Location attr_reader :location - # def initialize: (target: Node, operator: Token, value: Node, location: Location) -> void + # def initialize: (target: Node, operator: String, value: Node, location: Location) -> void def initialize(target, operator, value, location) @target = target @operator = operator @@ -140,7 +140,7 @@ class Binary < Node # attr_reader left: Node attr_reader :left - # attr_reader operator: Token + # attr_reader operator: String attr_reader :operator # attr_reader right: Node @@ -149,7 +149,7 @@ class Binary < Node # attr_reader location: Location attr_reader :location - # def initialize: (left: Node, operator: Token, right: Node, location: Location) -> void + # def initialize: (left: Node, operator: String, right: Node, location: Location) -> void def initialize(left, operator, right, location) @left = left @operator = operator @@ -182,13 +182,13 @@ def ==(other) end class CharacterLiteral < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -219,13 +219,13 @@ def ==(other) end class FloatLiteral < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -256,13 +256,13 @@ def ==(other) end class Identifier < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -296,7 +296,7 @@ class IfModifier < Node # attr_reader statement: Node attr_reader :statement - # attr_reader keyword: Token + # attr_reader keyword: String attr_reader :keyword # attr_reader predicate: Node @@ -305,7 +305,7 @@ class IfModifier < Node # attr_reader location: Location attr_reader :location - # def initialize: (statement: Node, keyword: Token, predicate: Node, location: Location) -> void + # def initialize: (statement: Node, keyword: String, predicate: Node, location: Location) -> void def initialize(statement, keyword, predicate, location) @statement = statement @keyword = keyword @@ -338,13 +338,13 @@ def ==(other) end class ImaginaryLiteral < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -375,13 +375,13 @@ def ==(other) end class IntegerLiteral < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -415,7 +415,7 @@ class OperatorAssignment < Node # attr_reader target: Node attr_reader :target - # attr_reader operator: Token + # attr_reader operator: String attr_reader :operator # attr_reader value: Node @@ -424,7 +424,7 @@ class OperatorAssignment < Node # attr_reader location: Location attr_reader :location - # def initialize: (target: Node, operator: Token, value: Node, location: Location) -> void + # def initialize: (target: Node, operator: String, value: Node, location: Location) -> void def initialize(target, operator, value, location) @target = target @operator = operator @@ -497,7 +497,7 @@ class Range < Node # attr_reader left: Node? attr_reader :left - # attr_reader operator: Token + # attr_reader operator: String attr_reader :operator # attr_reader right: Node? @@ -506,7 +506,7 @@ class Range < Node # attr_reader location: Location attr_reader :location - # def initialize: (left: Node?, operator: Token, right: Node?, location: Location) -> void + # def initialize: (left: Node?, operator: String, right: Node?, location: Location) -> void def initialize(left, operator, right, location) @left = left @operator = operator @@ -539,13 +539,13 @@ def ==(other) end class RationalLiteral < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -576,13 +576,13 @@ def ==(other) end class Redo < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -613,13 +613,13 @@ def ==(other) end class Retry < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -690,13 +690,13 @@ class Ternary < Node # attr_reader predicate: Node attr_reader :predicate - # attr_reader question_mark: Token + # attr_reader question_mark: String attr_reader :question_mark # attr_reader true_expression: Node attr_reader :true_expression - # attr_reader colon: Token + # attr_reader colon: String attr_reader :colon # attr_reader false_expression: Node @@ -705,7 +705,7 @@ class Ternary < Node # attr_reader location: Location attr_reader :location - # def initialize: (predicate: Node, question_mark: Token, true_expression: Node, colon: Token, false_expression: Node, location: Location) -> void + # def initialize: (predicate: Node, question_mark: String, true_expression: Node, colon: String, false_expression: Node, location: Location) -> void def initialize(predicate, question_mark, true_expression, colon, false_expression, location) @predicate = predicate @question_mark = question_mark @@ -743,7 +743,7 @@ class UnlessModifier < Node # attr_reader statement: Node attr_reader :statement - # attr_reader keyword: Token + # attr_reader keyword: String attr_reader :keyword # attr_reader predicate: Node @@ -752,7 +752,7 @@ class UnlessModifier < Node # attr_reader location: Location attr_reader :location - # def initialize: (statement: Node, keyword: Token, predicate: Node, location: Location) -> void + # def initialize: (statement: Node, keyword: String, predicate: Node, location: Location) -> void def initialize(statement, keyword, predicate, location) @statement = statement @keyword = keyword @@ -788,7 +788,7 @@ class UntilModifier < Node # attr_reader statement: Node attr_reader :statement - # attr_reader keyword: Token + # attr_reader keyword: String attr_reader :keyword # attr_reader predicate: Node @@ -797,7 +797,7 @@ class UntilModifier < Node # attr_reader location: Location attr_reader :location - # def initialize: (statement: Node, keyword: Token, predicate: Node, location: Location) -> void + # def initialize: (statement: Node, keyword: String, predicate: Node, location: Location) -> void def initialize(statement, keyword, predicate, location) @statement = statement @keyword = keyword @@ -830,13 +830,13 @@ def ==(other) end class VariableReference < Node - # attr_reader value: Token + # attr_reader value: String attr_reader :value # attr_reader location: Location attr_reader :location - # def initialize: (value: Token, location: Location) -> void + # def initialize: (value: String, location: Location) -> void def initialize(value, location) @value = value @location = location @@ -870,7 +870,7 @@ class WhileModifier < Node # attr_reader statement: Node attr_reader :statement - # attr_reader keyword: Token + # attr_reader keyword: String attr_reader :keyword # attr_reader predicate: Node @@ -879,7 +879,7 @@ class WhileModifier < Node # attr_reader location: Location attr_reader :location - # def initialize: (statement: Node, keyword: Token, predicate: Node, location: Location) -> void + # def initialize: (statement: Node, keyword: String, predicate: Node, location: Location) -> void def initialize(statement, keyword, predicate, location) @statement = statement @keyword = keyword diff --git a/test-native/run-one.c b/test-native/run-one.c index a60a88fe70..beb05c20d3 100644 --- a/test-native/run-one.c +++ b/test-native/run-one.c @@ -1,4 +1,6 @@ #include "../ext/yarp/gen_token_type.c" +#include "../ext/yarp/string.c" +#include "../ext/yarp/token.c" #include "../ext/yarp/yarp.c" #include "file.h" #include "formatter.h" diff --git a/test/parse_test.rb b/test/parse_test.rb index 4009e311df..ebb99f6238 100644 --- a/test/parse_test.rb +++ b/test/parse_test.rb @@ -6,99 +6,99 @@ class ParseTest < Test::Unit::TestCase include YARP::DSL test "binary" do - assert_equal Binary(expression("1"), PLUS("+"), expression("2")), expression("1 + 2") + assert_equal Binary(expression("1"), "+", expression("2")), expression("1 + 2") end test "character literal" do - assert_equal CharacterLiteral(CHARACTER_LITERAL("?a")), expression("?a") + assert_equal CharacterLiteral("?a"), expression("?a") end test "class variable" do - assert_equal VariableReference(CLASS_VARIABLE("@@abc")), expression("@@abc") + assert_equal VariableReference("@@abc"), expression("@@abc") end test "false" do - assert_equal VariableReference(KEYWORD_FALSE("false")), expression("false") + assert_equal VariableReference("false"), expression("false") end test "float" do - assert_equal FloatLiteral(FLOAT("1.0")), expression("1.0") + assert_equal FloatLiteral("1.0"), expression("1.0") end test "global variable" do - assert_equal VariableReference(GLOBAL_VARIABLE("$abc")), expression("$abc") + assert_equal VariableReference("$abc"), expression("$abc") end test "identifier" do - assert_equal VariableReference(IDENTIFIER("a")), expression("a") + assert_equal VariableReference("a"), expression("a") end test "if modifier" do - assert_equal IfModifier(expression("1"), KEYWORD_IF("if"), expression("true")), expression("1 if true") + assert_equal IfModifier(expression("1"), "if", expression("true")), expression("1 if true") end test "imaginary" do - assert_equal ImaginaryLiteral(IMAGINARY_NUMBER("1i")), expression("1i") + assert_equal ImaginaryLiteral("1i"), expression("1i") end test "instance variable" do - assert_equal VariableReference(INSTANCE_VARIABLE("@abc")), expression("@abc") + assert_equal VariableReference("@abc"), expression("@abc") end test "nil" do - assert_equal VariableReference(KEYWORD_NIL("nil")), expression("nil") + assert_equal VariableReference("nil"), expression("nil") end test "rational" do - assert_equal RationalLiteral(RATIONAL_NUMBER("1r")), expression("1r") + assert_equal RationalLiteral("1r"), expression("1r") end test "redo" do - assert_equal Redo(KEYWORD_REDO("redo")), expression("redo") + assert_equal Redo("redo"), expression("redo") end test "retry" do - assert_equal Retry(KEYWORD_RETRY("retry")), expression("retry") + assert_equal Retry("retry"), expression("retry") end test "self" do - assert_equal VariableReference(KEYWORD_SELF("self")), expression("self") + assert_equal VariableReference("self"), expression("self") end test "ternary" do expected = Ternary( - VariableReference(IDENTIFIER("a")), - QUESTION_MARK("?"), - VariableReference(IDENTIFIER("b")), - COLON(":"), - VariableReference(IDENTIFIER("c")) + VariableReference("a"), + "?", + VariableReference("b"), + ":", + VariableReference("c") ) assert_equal expected, expression("a ? b : c") end test "true" do - assert_equal VariableReference(KEYWORD_TRUE("true")), expression("true") + assert_equal VariableReference("true"), expression("true") end test "unless modifier" do - assert_equal UnlessModifier(expression("1"), KEYWORD_UNLESS("unless"), expression("true")), expression("1 unless true") + assert_equal UnlessModifier(expression("1"), "unless", expression("true")), expression("1 unless true") end test "until modifier" do - assert_equal UntilModifier(expression("1"), KEYWORD_UNTIL("until"), expression("true")), expression("1 until true") + assert_equal UntilModifier(expression("1"), "until", expression("true")), expression("1 until true") end test "while modifier" do - assert_equal WhileModifier(expression("1"), KEYWORD_WHILE("while"), expression("true")), expression("1 while true") + assert_equal WhileModifier(expression("1"), "while", expression("true")), expression("1 while true") end test "TERM < FACTOR" do actual = expression("1 + 2 * 3") expected = Binary( expression("1"), - PLUS("+"), - Binary(expression("2"), STAR("*"), expression("3")) + "+", + Binary(expression("2"), "*", expression("3")) ) assert_equal expected, actual @@ -108,8 +108,8 @@ class ParseTest < Test::Unit::TestCase actual = expression("1 * 2 ** 3") expected = Binary( expression("1"), - STAR("*"), - Binary(expression("2"), STAR_STAR("**"), expression("3")) + "*", + Binary(expression("2"), "**", expression("3")) ) assert_equal expected, actual @@ -118,8 +118,8 @@ class ParseTest < Test::Unit::TestCase test "FACTOR > TERM" do actual = expression("1 * 2 + 3") expected = Binary( - Binary(expression("1"), STAR("*"), expression("2")), - PLUS("+"), + Binary(expression("1"), "*", expression("2")), + "+", expression("3") ) @@ -129,8 +129,8 @@ class ParseTest < Test::Unit::TestCase test "MODIFIER left associativity" do actual = expression("a if b if c") expected = IfModifier( - IfModifier(expression("a"), KEYWORD_IF("if"), expression("b")), - KEYWORD_IF("if"), + IfModifier(expression("a"), "if", expression("b")), + "if", expression("c") )