From ade088fb2d80bdd313c661b72a70120d5743eb37 Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 16:33:00 +0530 Subject: [PATCH 1/8] Moved Interpolation logic to logical_parser Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 5664986481..ec68a54dd4 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -155,11 +155,11 @@ def scan_nonquoted_string(boundary_charset = LINE_END) end def scan_embedded_code - src = '"#{'+@ss.rest+"\n=begin\n=end\n}" + src = '#{'+@ss.rest+"\n=begin\n=end\n}" seek = -1 while (seek = src.index('}', seek + 1)) - unless Ripper.sexp(src[0..seek] + '"').nil? # eager parsing until valid expression + unless Ripper.sexp(src[0..seek]).nil? # eager parsing until valid expression break end end @@ -168,7 +168,7 @@ def scan_embedded_code raise Fluent::ConfigParseError, @ss.rest end - code = src[3, seek-3] + code = src[2, seek-2] if @ss.rest.length < code.length @ss.pos += @ss.rest.length @@ -255,7 +255,12 @@ def scan_json(is_array) line_buffer = "" else # '#' is a char in json string - line_buffer << char + if skip(/\{/) + line_buffer << eval_embedded_code(scan_embedded_code) + skip(/\}/) + else + line_buffer << char + end end next # This char '#' MUST NOT terminate json object. @@ -280,6 +285,7 @@ def scan_json(is_array) end JSON.dump(result) + end end end From f4f5edc1b4b2362f455e7d71217c6e73c426fe75 Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 16:39:29 +0530 Subject: [PATCH 2/8] nit Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index ec68a54dd4..506ac49989 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -285,7 +285,6 @@ def scan_json(is_array) end JSON.dump(result) - end end end From 60ee369d76b59820b19c5acada3ec634cb9f139b Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 17:08:46 +0530 Subject: [PATCH 3/8] Added few comments Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 506ac49989..626fc19340 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -176,7 +176,7 @@ def scan_embedded_code end @ss.pos += code.length - + p code '"#{' + code + '}"' end @@ -254,11 +254,13 @@ def scan_json(is_array) buffer << line_buffer + "\n" line_buffer = "" else - # '#' is a char in json string - if skip(/\{/) + if @ss.exist?(/\{[^}]+\}/) + # if it's interpolated string + skip(/\{/) line_buffer << eval_embedded_code(scan_embedded_code) skip(/\}/) else + # '#' is a char in json string line_buffer << char end end From dab67464cb51f1193eb12e5c1bb5a976f0be3310 Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 17:18:49 +0530 Subject: [PATCH 4/8] Added few comments Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 626fc19340..afa3ff8669 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -176,7 +176,6 @@ def scan_embedded_code end @ss.pos += code.length - p code '"#{' + code + '}"' end From 12e53161a4397310fd494805b66aca530cc1389c Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 22:01:12 +0530 Subject: [PATCH 5/8] Fixed the bug in PR to pass UTs Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index afa3ff8669..9337f175f5 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -155,11 +155,11 @@ def scan_nonquoted_string(boundary_charset = LINE_END) end def scan_embedded_code - src = '#{'+@ss.rest+"\n=begin\n=end\n}" + src = '"#{'+@ss.rest+"\n=begin\n=end\n}" seek = -1 while (seek = src.index('}', seek + 1)) - unless Ripper.sexp(src[0..seek]).nil? # eager parsing until valid expression + unless Ripper.sexp(src[0..seek] + '"').nil? # eager parsing until valid expression break end end @@ -168,7 +168,7 @@ def scan_embedded_code raise Fluent::ConfigParseError, @ss.rest end - code = src[2, seek-2] + code = src[3, seek-3] if @ss.rest.length < code.length @ss.pos += @ss.rest.length @@ -176,6 +176,7 @@ def scan_embedded_code end @ss.pos += code.length + '"#{' + code + '}"' end From 2851b998f16a68d92ce5173494ebcf01a46303e3 Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Wed, 7 Aug 2024 10:35:29 +0530 Subject: [PATCH 6/8] Added UTs for the String interpolation in Hash and array forms Signed-off-by: Athish Pranav D --- test/config/test_literal_parser.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb index 9f6130ff2a..c5b9b18898 100644 --- a/test/config/test_literal_parser.rb +++ b/test/config/test_literal_parser.rb @@ -259,6 +259,17 @@ def test_falseX test('[ "a" , "b" ]') { assert_text_parsed_as_json(["a","b"], '[ "a" , "b" ]') } test("[\n\"a\"\n,\n\"b\"\n]") { assert_text_parsed_as_json(["a","b"], "[\n\"a\"\n,\n\"b\"\n]") } test('["ab","cd"]') { assert_text_parsed_as_json(["ab","cd"], '["ab","cd"]') } + test('["a","#{v1}"') { assert_text_parsed_as_json(["a","#{v1}"], '["a","#{v1}"]') } + test('["a","#{v1}","#{v2}"]') { assert_text_parsed_as_json(["a","#{v1}","#{v2}"], '["a","#{v1}","#{v2}"]') } + test('["a","#{v1} #{v2}"]') { assert_text_parsed_as_json(["a","#{v1} #{v2}"], '["a","#{v1} #{v2}"]') } + test('["a","#{hostname}"]') { assert_text_parsed_as_json(["a","#{Socket.gethostname}"], '["a","#{hostname}"]') } + test('["a","foo#{worker_id}"]') { + ENV.delete('SERVERENGINE_WORKER_ID') + assert_text_parsed_as('["a","foo"]', '["a","foo#{worker_id}"]') + ENV['SERVERENGINE_WORKER_ID'] = '1' + assert_text_parsed_as('["a","foo1"]', '["a","foo#{worker_id}"]') + ENV.delete('SERVERENGINE_WORKER_ID') + } json_array_with_js_comment = <"b","c"=>"d"}, '{"a":"b","c":"d"}') } test('{ "a" : "b" , "c" : "d" }') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, '{ "a" : "b" , "c" : "d" }') } test('{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, "{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}") } + test('{"a":"b","c":"#{v1}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"#{v1}"}, '{"a":"b","c":"#{v1}"}') } + test('{"a":"b","#{v1}":"d"}') { assert_text_parsed_as_json({"a"=>"b","#{v1}"=>"d"}, '{"a":"b","#{v1}":"d"}') } + test('{"a":"#{v1}","c":"#{v2}"}') { assert_text_parsed_as_json({"a"=>"#{v1}","c"=>"#{v2}"}, '{"a":"#{v1}","c":"#{v2}"}') } + test('{"a":"b","c":"d #{v1} #{v2}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d #{v1} #{v2}"}, '{"a":"b","c":"d #{v1} #{v2}"}') } + test('{"a":"#{hostname}"}') { assert_text_parsed_as_json({"a"=>"#{Socket.gethostname}"}, '{"a":"#{hostname}"}') } + test('{"a":"foo#{worker_id}"}') { + ENV.delete('SERVERENGINE_WORKER_ID') + assert_text_parsed_as('{"a":"foo"}', '{"a":"foo#{worker_id}"}') + ENV['SERVERENGINE_WORKER_ID'] = '1' + assert_text_parsed_as('{"a":"foo1"}', '{"a":"foo#{worker_id}"}') + ENV.delete('SERVERENGINE_WORKER_ID') + } json_hash_with_comment = < Date: Fri, 9 Aug 2024 09:23:20 +0530 Subject: [PATCH 7/8] Added tests for single vs double quote Signed-off-by: Athish Pranav D --- test/config/test_literal_parser.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb index c5b9b18898..275e468e3f 100644 --- a/test/config/test_literal_parser.rb +++ b/test/config/test_literal_parser.rb @@ -319,6 +319,9 @@ def test_falseX assert_text_parsed_as('{"a":"foo1"}', '{"a":"foo#{worker_id}"}') ENV.delete('SERVERENGINE_WORKER_ID') } + test('no quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '{"a":"b","c":"#{v1}"}') } + test('single quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'#{v1}'}, '\'{"a":"b","c":"#{v1}"}\'') } + test('double quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '"{\"a\":\"b\",\"c\":\"#{v1}\"}"') } json_hash_with_comment = < Date: Tue, 13 Aug 2024 11:50:18 +0530 Subject: [PATCH 8/8] Addressed comments Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 9337f175f5..d165c728ff 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -254,7 +254,7 @@ def scan_json(is_array) buffer << line_buffer + "\n" line_buffer = "" else - if @ss.exist?(/\{[^}]+\}/) + if @ss.exist?(/^\{[^}]+\}/) # if it's interpolated string skip(/\{/) line_buffer << eval_embedded_code(scan_embedded_code)