Merge pull request #8 from BolunThompson/bash

Ignore binary characters in var file for libbash

sh_expand/env_vars_util.py

@@ -27,10 +27,10 @@ def read_vars_file(var_file_path, bash_version_tuple):
 ## strings, while this does not
 def read_vars_file_new(var_file_path):
     vars_dict = {}
-    with open(var_file_path) as f:
+    with open(var_file_path, "rb") as f:
         for line in f:
             # remove newline
-            line = line[:-1]
+            line = line[:-1].decode('utf-8', errors='replace')
             _, var_type, rest = line.split(" ", maxsplit=2)
 
             ## TODO: This doesn't handle associative arrays
@@ -61,9 +61,9 @@ def read_vars_file_old(var_file_path):
     # with open(var_file_path) as f:
     #     lines = [line.rstrip() for line in f.readlines()]
 
-    with open(var_file_path) as f:
+    with open(var_file_path, "rb") as f:
         variable_reading_start_time = datetime.now()
-        data = f.read()
+        data = f.read().decode('utf-8', errors='replace')
         variable_reading_end_time = datetime.now()
         print_time_delta("Variable Reading", variable_reading_start_time, variable_reading_end_time)
 

tests/variable_parse/accepted_byte_data.env

@@ -0,0 +1 @@
+declare -- bin="��"

tests/variable_parse/accepted_byte_data.json

@@ -0,0 +1,3 @@
+{
+  "bin": [null, "\ufffd\ufffd"]
+}