Fix expansion of multibyte IFS characters

Closes att#13. Previously, the `varsub` method used for the macro expansion of
`$param`, `${param}`, and `${param op word}` would incorrectly expand the
internal field separator (IFS) if it was a multibyte character. This was due to
truncation based on the incorrect assumption that the IFS would never be larger
than a single byte.

This change fixes this issue by carefully tracking the number of bytes that
should be persisted in the IFS case and ensuring that all bytes are written
during expansion and substitution.

Author: etscrivner

src/cmd/ksh93/meson.build

@@ -88,7 +88,7 @@ all_tests = [
     ['alias'], ['append'], ['arith'], ['arrays'], ['arrays2'], ['attributes'],
     ['basic', 90], ['bracket'], ['builtins'], ['case'], ['comvar'],
     ['comvario'], ['coprocess', 50], ['cubetype'], ['directoryfd'], ['enum'],
-    ['exit'], ['expand'], ['functions'], ['glob'], ['grep'], ['heredoc'],
+    ['exit'], ['expand'], ['functions'], ['glob'], ['grep'], ['heredoc'], ['ifs'],
     ['io'], ['leaks'], ['locale'], ['math', 50], ['nameref'], ['namespace'],
     ['modifiers'], ['options'], ['path'], ['pointtype'], ['quoting'],
     ['quoting2'], ['readcsv'], ['recttype'], ['restricted'], ['return'], ['select'],

src/cmd/ksh93/sh/macro.c

@@ -1792,10 +1792,21 @@ static_fn bool varsub(Mac_t *mp) {
                 mp->atmode = mode == '@';
                 mp->pattern = oldpat;
             } else if (d) {
-                if (mp->sp) {
-                    sfputc(mp->sp, d);
+                Sfio_t *sfio_ptr = (mp->sp) ? mp->sp : stkp;
+
+                // We know from above that if we are not performing @-expansion
+                // then we assigned `d` the value of `mp->ifs`, here we check
+                // whether or not we have a valid string of IFS characters to
+                // write as it is possible for `d` to be set to `mp->ifs` and
+                // yet `mp->ifsp` to be NULL.
+                if (mode != '@' && mp->ifsp) {
+                    // Handle multi-byte characters being used for the internal
+                    // field separator (IFS).
+                    for (int i = 0; i < mbsize(mp->ifsp); i++) {
+                        sfputc(sfio_ptr, mp->ifsp[i]);
+                    }
                 } else {
-                    sfputc(stkp, d);
+                    sfputc(sfio_ptr, d);
                 }
             }
         }

src/cmd/ksh93/tests/ifs.sh

@@ -0,0 +1,30 @@
+# These are the tests for the internal field separator (IFS).
+
+IFS=e
+set : :
+[[ "$*" == ":e:" ]] || log_error "IFS failed" ":e:" "$*"
+
+IFS='|' read -r first second third <<< 'one|two|three'
+[[ "${first}" == "one" ]] || log_error "IFS failed" "one" "${first}"
+[[ "${second}" == "two" ]] || log_error "IFS failed" "two" "${second}"
+[[ "${third}" == "three" ]] || log_error "IFS failed" "three" "${third}"
+
+# Multi-byte character checks will only work if UTF-8 inputs are enabled
+if [ "${LC_ALL}" = "en_US.UTF-8" ]
+then
+    # 2 byte latin accented e character
+    IFS=é
+    set : :
+    [[ "$*" == ":é:" ]] || log_error "IFS failed with multibyte character" ":é:" "$*"
+
+    # 4 byte roman sestertius character
+    IFS=𐆘 read -r first second third <<< 'one𐆘two𐆘three'
+    [[ "${first}" == "one" ]] || log_error "IFS failed" "one" "${first}"
+    [[ "${second}" == "two" ]] || log_error "IFS failed" "two" "${second}"
+    [[ "${third}" == "three" ]] || log_error "IFS failed" "three" "${third}"
+
+    # Ensure subshells don't get corrupted when IFS becomes multibyte character
+    expected_output=$(printf ":é:\\ntrap -- 'echo end' EXIT\\nend")
+    output=$(LANG=C.UTF-8; IFS=é; set : :; echo "$*"; trap "echo end" EXIT; trap)
+    [[ "${output}" == "${expected_output}" ]] || log_error "IFS subshell failed" "${expected_output}" "${output}"
+fi