From 3802638f075b0b3e8b30a8c764f0ecc9608125e5 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Wed, 25 Apr 2018 17:46:03 -0400 Subject: [PATCH 01/10] Test to address regression in #339 --- spec/dotenv/parser_spec.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spec/dotenv/parser_spec.rb b/spec/dotenv/parser_spec.rb index af7a06d7..b689819d 100644 --- a/spec/dotenv/parser_spec.rb +++ b/spec/dotenv/parser_spec.rb @@ -155,6 +155,20 @@ def env(string) expect(env("# Uncomment to activate:\n")).to eql({}) end + it "includes variables without values" do + input = 'DATABASE_PASSWORD= +DATABASE_USERNAME=root +DATABASE_HOST=/tmp/mysql.sock' + + output = { + "DATABASE_PASSWORD" => "", + "DATABASE_USERNAME" => "root", + "DATABASE_HOST" => "/tmp/mysql.sock" + } + + expect(env(input)).to eql(output) + end + it "parses # in quoted values" do expect(env('foo="ba#r"')).to eql("foo" => "ba#r") expect(env("foo='ba#r'")).to eql("foo" => "ba#r") From 3ef2d9007e7f684090f889a2924cca3160206c68 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Fri, 12 Jan 2018 23:58:31 -0500 Subject: [PATCH 02/10] Parse multi-line values properly (e.g. Heroku config export) --- lib/dotenv/parser.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index 4451f17c..33cb296a 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -12,7 +12,6 @@ class Parser [Dotenv::Substitutions::Variable, Dotenv::Substitutions::Command] LINE = / - \A \s* (?:export\s+)? # optional export ([\w\.]+) # key @@ -22,11 +21,10 @@ class Parser | # or "(?:\"|[^"])*" # double quoted value | # or - [^#\n]+ # unquoted value + [^#\r\n]+ # unquoted value )? # value end \s* (?:\#.*)? # optional comment - \z /x class << self @@ -44,7 +42,12 @@ def initialize(string, is_load = false) end def call - @string.split(/[\n\r]+/).each do |line| + # Process matches + @string.scan(LINE).each do |key, value| + @hash[key] = parse_value(value || "") + end + # Process non-matches + @string.gsub(LINE, '').split(/[\n\r]+/).each do |line| parse_line(line) end @hash @@ -53,10 +56,7 @@ def call private def parse_line(line) - if (match = line.match(LINE)) - key, value = match.captures - @hash[key] = parse_value(value || "") - elsif line.split.first == "export" + if line.split.first == "export" if variable_not_set?(line) raise FormatError, "Line #{line.inspect} has an unset variable" end From 16e43686c6eea724831b85ed5e78b9a18673d780 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sat, 13 Jan 2018 00:46:57 -0500 Subject: [PATCH 03/10] Fix original regex to escape slash --- lib/dotenv/parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index 33cb296a..04153176 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -17,9 +17,9 @@ class Parser ([\w\.]+) # key (?:\s*=\s*|:\s+?) # separator ( # optional value begin - '(?:\'|[^'])*' # single quoted value + '(?:\\'|[^'])*' # single quoted value | # or - "(?:\"|[^"])*" # double quoted value + "(?:\\"|[^"])*" # double quoted value | # or [^#\r\n]+ # unquoted value )? # value end From af7efd9073d1ff872e14ff89358b08201bf7db3b Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sat, 13 Jan 2018 00:47:08 -0500 Subject: [PATCH 04/10] Match multiple lines --- lib/dotenv/parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index 04153176..b665d1b7 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -65,7 +65,7 @@ def parse_line(line) def parse_value(value) # Remove surrounding quotes - value = value.strip.sub(/\A(['"])(.*)\1\z/, '\2') + value = value.strip.sub(/\A(['"])(.*)\1\z/m, '\2') if Regexp.last_match(1) == '"' value = unescape_characters(expand_newlines(value)) From 59a1ca0e9ed31d9e9ac432b53f661a8e20b87548 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sat, 13 Jan 2018 00:47:23 -0500 Subject: [PATCH 05/10] Add tests for parsing multi-line values --- spec/dotenv/parser_spec.rb | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/spec/dotenv/parser_spec.rb b/spec/dotenv/parser_spec.rb index b689819d..0b822583 100644 --- a/spec/dotenv/parser_spec.rb +++ b/spec/dotenv/parser_spec.rb @@ -183,6 +183,32 @@ def env(string) expect(env("foo=")).to eql("foo" => "") end + it "allows multi-line values in single quotes" do + expect(env(%{ +OPTION_A=first line +export OPTION_B='line 1 +line 2 +line 3' +OPTION_C="last line" +OPTION_ESCAPED='line one +this is \\'quoted\\' +one more line' + })).to eql("OPTION_A" => "first line", "OPTION_B" => "line 1\nline 2\nline 3", "OPTION_C" => "last line", "OPTION_ESCAPED" => "line one\nthis is \\'quoted\\'\none more line") + end + + it "allows multi-line values in double quotes" do + expect(env(%{ +OPTION_A=first line +export OPTION_B="line 1 +line 2 +line 3" +OPTION_C="last line" +OPTION_ESCAPED="line one +this is \\"quoted\\" +one more line" + })).to eql("OPTION_A" => "first line", "OPTION_B" => "line 1\nline 2\nline 3", "OPTION_C" => "last line", "OPTION_ESCAPED" => "line one\nthis is \"quoted\"\none more line") + end + if RUBY_VERSION > "1.8.7" it "parses shell commands interpolated in $()" do expect(env("echo=$(echo hello)")).to eql("echo" => "hello") From edf934b56700b0dded133fd509aa869a8c8be6ea Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sat, 13 Jan 2018 00:59:03 -0500 Subject: [PATCH 06/10] Fix Rubocop offenses --- lib/dotenv/parser.rb | 2 +- spec/dotenv/parser_spec.rb | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index b665d1b7..27c0574c 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -47,7 +47,7 @@ def call @hash[key] = parse_value(value || "") end # Process non-matches - @string.gsub(LINE, '').split(/[\n\r]+/).each do |line| + @string.gsub(LINE, "").split(/[\n\r]+/).each do |line| parse_line(line) end @hash diff --git a/spec/dotenv/parser_spec.rb b/spec/dotenv/parser_spec.rb index 0b822583..b2ae94d3 100644 --- a/spec/dotenv/parser_spec.rb +++ b/spec/dotenv/parser_spec.rb @@ -184,29 +184,41 @@ def env(string) end it "allows multi-line values in single quotes" do - expect(env(%{ -OPTION_A=first line + env_file = %(OPTION_A=first line export OPTION_B='line 1 line 2 line 3' OPTION_C="last line" OPTION_ESCAPED='line one this is \\'quoted\\' -one more line' - })).to eql("OPTION_A" => "first line", "OPTION_B" => "line 1\nline 2\nline 3", "OPTION_C" => "last line", "OPTION_ESCAPED" => "line one\nthis is \\'quoted\\'\none more line") +one more line') + + expected_result = { + "OPTION_A" => "first line", + "OPTION_B" => "line 1\nline 2\nline 3", + "OPTION_C" => "last line", + "OPTION_ESCAPED" => "line one\nthis is \\'quoted\\'\none more line" + } + expect(env(env_file)).to eql(expected_result) end it "allows multi-line values in double quotes" do - expect(env(%{ -OPTION_A=first line + env_file = %(OPTION_A=first line export OPTION_B="line 1 line 2 line 3" OPTION_C="last line" OPTION_ESCAPED="line one this is \\"quoted\\" -one more line" - })).to eql("OPTION_A" => "first line", "OPTION_B" => "line 1\nline 2\nline 3", "OPTION_C" => "last line", "OPTION_ESCAPED" => "line one\nthis is \"quoted\"\none more line") +one more line") + + expected_result = { + "OPTION_A" => "first line", + "OPTION_B" => "line 1\nline 2\nline 3", + "OPTION_C" => "last line", + "OPTION_ESCAPED" => "line one\nthis is \"quoted\"\none more line" + } + expect(env(env_file)).to eql(expected_result) end if RUBY_VERSION > "1.8.7" From f5454081f1961fafcd9f83698586d8c155f4ba70 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Fri, 13 Apr 2018 13:25:23 -0400 Subject: [PATCH 07/10] Update README with multi-line value improvements --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d350361f..d621a980 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,21 @@ export SECRET_KEY=YOURSECRETKEYGOESHERE If you need multiline variables, for example private keys, you can double quote strings and use the `\n` character for newlines: ```shell -PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nHkVN9…\n-----END DSA PRIVATE KEY-----\n" +PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nHkVN9...\n-----END DSA PRIVATE KEY-----\n" ``` +Alternatively, multi-line values with line breaks are now supported for quoted values. + +```shell +PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY----- +... +HkVN9... +... +-----END DSA PRIVATE KEY-----" +``` + +This is particularly helpful when using the Heroku command line plugin [`heroku-config`](https://github.com/xavdid/heroku-config) to pull configuration variables down that may have line breaks. + ### Command Substitution You need to add the output of a command in one of your variables? Simply add it with `$(your_command)`: From 348ccf0ffc7b9436bc01edd5c8fe9c1a42b85f8b Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sun, 6 May 2018 23:14:34 -0400 Subject: [PATCH 08/10] Replace "\r" characters with "\n" for consistent new line matching --- lib/dotenv/parser.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index 27c0574c..dd85e171 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -42,12 +42,14 @@ def initialize(string, is_load = false) end def call + # Convert line breaks to same format + lines = @string.gsub(/\r\n?/, "\n") # Process matches - @string.scan(LINE).each do |key, value| + lines.scan(LINE).each do |key, value| @hash[key] = parse_value(value || "") end # Process non-matches - @string.gsub(LINE, "").split(/[\n\r]+/).each do |line| + lines.gsub(LINE, "").split(/[\n\r]+/).each do |line| parse_line(line) end @hash From 98378626dee610b86e38391e9804c7498fd2d3c8 Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sun, 6 May 2018 23:15:40 -0400 Subject: [PATCH 09/10] Match beginning of line to make sure the line is not commented out --- lib/dotenv/parser.rb | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index dd85e171..dffa048a 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -12,19 +12,22 @@ class Parser [Dotenv::Substitutions::Variable, Dotenv::Substitutions::Command] LINE = / - \s* - (?:export\s+)? # optional export - ([\w\.]+) # key (?:\s*=\s*|:\s+?) # separator ( # optional value begin + (?:^|\A) # beginning of line + \s* # leading whitespace + (?:export\s+)? # optional export + ([\w\.]+) # key '(?:\\'|[^'])*' # single quoted value | # or + | # or "(?:\\"|[^"])*" # double quoted value - | # or - [^#\r\n]+ # unquoted value - )? # value end - \s* - (?:\#.*)? # optional comment + | # or + [^\#\r\n]+ # unquoted value + )? # value end + \s* # trailing whitespace + (?:\#.*)? # optional comment + (?:$|\z) # end of line /x class << self From 6fd3d71e68c572bcd412da1708da6a2a5ee61a0f Mon Sep 17 00:00:00 2001 From: Joel Van Horn Date: Sun, 6 May 2018 23:16:12 -0400 Subject: [PATCH 10/10] Don't match greedy spaces to avoid matching a new line when there is no value --- lib/dotenv/parser.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/dotenv/parser.rb b/lib/dotenv/parser.rb index dffa048a..0d95dfa4 100644 --- a/lib/dotenv/parser.rb +++ b/lib/dotenv/parser.rb @@ -12,14 +12,13 @@ class Parser [Dotenv::Substitutions::Variable, Dotenv::Substitutions::Command] LINE = / - (?:\s*=\s*|:\s+?) # separator - ( # optional value begin (?:^|\A) # beginning of line \s* # leading whitespace (?:export\s+)? # optional export ([\w\.]+) # key + (?:\s*=\s*?|:\s+?) # separator + ( # optional value begin '(?:\\'|[^'])*' # single quoted value - | # or | # or "(?:\\"|[^"])*" # double quoted value | # or