Fix crash on reading a here-doc within a comsub within a here-doc

Minimal reproducer:

TEST_LINE="Test line"
cat <<EOF
$(
cat <<EOF2
${TEST_LINE}
EOF2
)
EOF

This script crashes on (at least) Alpine Linux with musl libc. I
have not been able to get it to crash on any other system, even
with ASan or valgrind, but still, this indicates a problem.

The relevant part of the gdb(1) stack trace on that system is:

    #0  memcpy () at src/string/aarch64/memcpy.S:145
    #1  0x0000aaaaaabd4d7c in sfwrite (f=0xaaaaaac813d8
        <_Stak_data>, buf=0xfffff7eb127b, n=225211) at
        /usr/local/src/ksh/src/lib/libast/sfio/sfwrite.c:130
    #2  0x0000aaaaaaae7f28 in lex_advance (iop=0xfffff7fe0600,
        buff=0xfffff7eb127b <error: Cannot access memory at address
        0xfffff7eb127b>, size=225211, context=0xaaaaaac8aae0) at
        /usr/local/src/ksh/src/cmd/ksh93/sh/lex.c:161
    #3  0x0000aaaaaaad10e8 in fcfill () at
        /usr/local/src/ksh/src/cmd/ksh93/sh/fcin.c:98
    #4  0x0000aaaaaaaf17a0 in sh_machere (infile=0xfffff7fe0600,
        outfile=0xfffff7fe3140, string=0xfffff7fe3c51 "EOF") at
        /usr/local/src/ksh/src/cmd/ksh93/sh/macro.c:319
    #5  0x0000aaaaaaadf64c in io_heredoc (iop=0xfffff7fe3c00,
        name=0xfffff7fe3c51 "EOF", traceon=1) at
        /usr/local/src/ksh/src/cmd/ksh93/sh/io.c:1632
    #6  0x0000aaaaaaade36c in sh_redirect (iop=0xfffff7fe3c00,
        flag=0) at /usr/local/src/ksh/src/cmd/ksh93/sh/io.c:1245
    #7  0x0000aaaaaab27898 in sh_exec (t=0xfffff7fe3bc0, flags=4)
        at /usr/local/src/ksh/src/cmd/ksh93/sh/xec.c:1176

So, as we can see in #2, the problem is an invalid pointer. It
becomes invalid in lex_advance() in lex.c on line 152, which is:

	 buff = lp->lexd.first;

So it looks like the lp->lexd.first pointer is somehow invalidated.

Analysis: The execution of both here-documents and command
substitutions involves parsing and lexical analysis at runtime. A
command substitution combined with nested execution of here-
documents causes lp->lexd.first to be changed, without being
restored when the nested execution returns.

So, this suggests that the fix would be to save that pointer before
calling comsubst() from sh_machere() to execute a command
substitution from within a here-document, and restore it after.

But, to avoid other potential or unforeseen problems, it is
probably best to save and restore the entire lexer state struct,
not just that one 'first' pointer.

src/cmd/ksh93/sh/macro.c: sh_machere():
- When executing a command substitution or arithmetic expression
  from a here-document, save and restore the lexer state.

Thanks to @stefanbidi for the bug report.
Resolves: #823

Author: McDutchie

NEWS

@@ -2,6 +2,11 @@ This documents significant changes in the dev branch of ksh 93u+m.
 For full details, see the git log at: https://github.com/ksh93/ksh
 Uppercase BUG_* IDs are shell bug IDs as used by the Modernish shell library.
 
+2025-03-23:
+
+- Fixed a crash that occurred in specific circumstances when processing a
+  here-documnet within a command subtitution within another here-document.
+
 2025-03-21:
 
 - Fixed a corner case bug in the "$*" expansion: spurious backslashes were

src/cmd/ksh93/include/version.h

@@ -18,7 +18,7 @@
 #include <ast_release.h>
 #include "git.h"
 
-#define SH_RELEASE_DATE	"2025-03-22"	/* must be in this format for $((.sh.version)) */
+#define SH_RELEASE_DATE	"2025-03-23"	/* must be in this format for $((.sh.version)) */
 /*
  * This comment keeps SH_RELEASE_DATE a few lines away from SH_RELEASE_SVER to avoid
  * merge conflicts when cherry-picking dev branch commits onto a release branch.

src/cmd/ksh93/sh/macro.c

@@ -381,8 +381,17 @@ void sh_machere(Sfio_t *infile, Sfio_t *outfile, char *string)
 				break;
 			    }
 			    case S_PAR:
+			    {
+				/*
+				 * Execute a command substitution or arithmetic expression from a here-document. As both
+				 * these and here-documents involve lexical analysis at runtime, save and restore the
+				 * lexer state to avoid hard-to-trace invalid pointers down the line in case of nesting.
+				 */
+				Lex_t sav = *lp;
 				comsubst(mp,NULL,1);
+				*lp = sav;
 				break;
+			    }
 			    case S_EOF:
 				if((c=fcfill()) > 0)
 					goto again;

src/cmd/ksh93/tests/heredoc.sh

@@ -554,5 +554,31 @@ _EOF"; } 2>&1 )
 	"(expected status 0, '$exp';" \
 	"got status $e$( ((e>128)) && print -n /SIG && kill -l "$e"), $(printf %q "$got"))"
 
+# ======
+# potential crash when here-documents and command substitutions are nested
+# https://github.com/ksh93/ksh/issues/823
+got=$(set +x; { "$SHELL" -c 'TEST_LINE1="Test line 1"
+TEST_LINE2="Test line 2"
+function1 () {
+cat << EOF
+	${TEST_LINE1}
+EOF
+}
+function2 () {
+cat << EOF
+	${TEST_LINE2}
+EOF
+}
+function3 () {
+cat << EOF
+	$(function1)
+EOF
+	function2
+}
+function3'; } 2>&1)
+exp=$'\t\tTest line 1\n\tTest line 2'
+[[ $got == "$exp" ]] || err_exit "processing a here-document from a command substitution in a here-document" \
+	"(expected $(printf %q "$exp"), got $(printf %q "$got"))"
+
 # ======
 exit $((Errors<125?Errors:125))