feat: handle command substitution and parameter expansion in shell scanner

data/syntax/braces.go

@@ -1,38 +1,152 @@
 package syntax
 
+// cmdSubFrame stores the state saved when entering a $(...) or ${...} substitution.
+type cmdSubFrame struct {
+	openChar         byte // '(' for $(...), '{' for ${...}
+	closeChar        byte // ')' for $(...), '}' for ${...}
+	depth            int  // nested open/close count within this level
+	savedDoubleQuote bool // inDoubleQuote at the moment the substitution was entered
+}
+
 // ScannerState tracks quote and escape state while scanning a command byte-by-byte.
 type ScannerState struct {
 	inSingleQuote bool
 	inDoubleQuote bool
 	escaped       bool
+	inBacktick    bool          // inside `...` command substitution
+	prevDollar    bool          // previous non-escaped char was $
+	cmdSubStack   []cmdSubFrame // $(...) and ${...} nesting stack
 }
 
 // Advance processes one byte and updates quote/escape state.
-// Returns true if the byte is "live" (outside quotes and not consumed by an escape sequence).
+// Returns true if the byte is "live" (outside quotes, command substitutions,
+// and not consumed by an escape sequence).
 // Quote characters and the backslash are consumed (return false) but callers that build
 // output strings must still WriteByte(ch) for those characters themselves.
 func (s *ScannerState) Advance(ch byte) bool {
 	if s.escaped {
 		s.escaped = false
+		s.prevDollar = false
+
 		return false // byte consumed by preceding backslash
 	}
 
+	wasDollar := s.prevDollar
+	s.prevDollar = false
+
 	if ch == '\\' && !s.inSingleQuote {
 		s.escaped = true
+
+		return false
+	}
+
+	// Backtick command substitution — toggle on ` outside single quotes.
+	if ch == '`' && !s.inSingleQuote {
+		s.inBacktick = !s.inBacktick
+
+		return false
+	}
+
+	// Inside backtick substitution — everything is non-live.
+	if s.inBacktick {
 		return false
 	}
 
+	// Inside a $(...) or ${...} substitution — track inner quotes and
+	// delimiters but always return false (non-live).
+	if len(s.cmdSubStack) > 0 {
+		top := &s.cmdSubStack[len(s.cmdSubStack)-1]
+
+		// Track quotes inside the substitution.
+		if ch == '\'' && !s.inDoubleQuote {
+			s.inSingleQuote = !s.inSingleQuote
+
+			return false
+		}
+
+		if ch == '"' && !s.inSingleQuote {
+			s.inDoubleQuote = !s.inDoubleQuote
+
+			return false
+		}
+
+		if !s.inSingleQuote && !s.inDoubleQuote {
+			switch {
+			case wasDollar && (ch == '(' || ch == '{'):
+				// Nested $( or ${ — push a new frame.
+				s.pushSubFrame(ch)
+			case ch == top.openChar:
+				top.depth++
+			case ch == top.closeChar:
+				if top.depth > 0 {
+					top.depth--
+				} else {
+					// Closing this substitution level — pop and restore state.
+					s.inDoubleQuote = top.savedDoubleQuote
+					s.cmdSubStack = s.cmdSubStack[:len(s.cmdSubStack)-1]
+				}
+			}
+
+			s.prevDollar = ch == '$'
+		}
+
+		return false
+	}
+
+	// Top-level quote handling.
 	if ch == '\'' && !s.inDoubleQuote {
 		s.inSingleQuote = !s.inSingleQuote
+
 		return false
 	}
 
 	if ch == '"' && !s.inSingleQuote {
 		s.inDoubleQuote = !s.inDoubleQuote
+
+		return false
+	}
+
+	// Inside quotes at the top level.
+	if s.inSingleQuote || s.inDoubleQuote {
+		// $( or ${ inside double quotes starts a substitution.
+		if s.inDoubleQuote && wasDollar && (ch == '(' || ch == '{') {
+			s.pushSubFrame(ch)
+
+			return false
+		}
+
+		if !s.inSingleQuote {
+			s.prevDollar = ch == '$'
+		}
+
+		return false
+	}
+
+	// Live context — check for $( or ${ to enter substitution.
+	if wasDollar && (ch == '(' || ch == '{') {
+		s.pushSubFrame(ch)
+
 		return false
 	}
 
-	return !s.inSingleQuote && !s.inDoubleQuote
+	s.prevDollar = ch == '$'
+
+	return true
+}
+
+// pushSubFrame pushes a new substitution frame for the given opening char ('(' or '{').
+func (s *ScannerState) pushSubFrame(openCh byte) {
+	closeCh := byte(')')
+	if openCh == '{' {
+		closeCh = '}'
+	}
+
+	s.cmdSubStack = append(s.cmdSubStack, cmdSubFrame{
+		openChar:         openCh,
+		closeChar:        closeCh,
+		savedDoubleQuote: s.inDoubleQuote,
+	})
+	s.inDoubleQuote = false
 }
 
 // IsBalancedBraces checks if braces are balanced in a command,

data/syntax/braces_test.go

@@ -59,6 +59,122 @@ func TestScannerStateAdvance(t *testing.T) {
 			input:    `'\\'`,
 			wantLive: []bool{false, false, false, false},
 		},
+		{
+			name: "Command substitution $(cmd)",
+			//           $ (  c  m  d  )
+			input:    "$(cmd)",
+			wantLive: []bool{true, false, false, false, false, false},
+		},
+		{
+			name: "Command substitution with operators $(a && b)",
+			//           $ (  a     &  &     b  )
+			input:    "$(a && b)",
+			wantLive: []bool{true, false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Nested command substitution $(echo $(date))",
+			//           $ (  e  c  h  o     $ (  d  a  t  e  )  )
+			input:    "$(echo $(date))",
+			wantLive: []bool{true, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Command sub inside double quotes",
+			//           "      $  (  c  m  d  )  "
+			input:    `"$(cmd)"`,
+			wantLive: []bool{false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Dollar-paren inside single quotes is literal",
+			//           '      $  (  c  m  d  )  '
+			input:    `'$(cmd)'`,
+			wantLive: []bool{false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Arithmetic expansion $((1+2))",
+			//           $ (  (  1  +  2  )  )
+			input:    "$((1+2))",
+			wantLive: []bool{true, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Escaped dollar prevents substitution",
+			//           \  $  (  c  m  d  )
+			input:    `\$(cmd)`,
+			wantLive: []bool{false, false, true, true, true, true, true},
+		},
+		{
+			name: "After substitution closes chars are live",
+			//           $ (  c  )     r
+			input:    "$(c) r",
+			wantLive: []bool{true, false, false, false, true, true},
+		},
+		{
+			name: "Quoted close paren inside substitution",
+			//           $ (  e  c  h  o     "     )     "     )
+			input:    `$(echo ")")`,
+			wantLive: []bool{true, false, false, false, false, false, false, false, false, false, false},
+		},
+		// Backtick command substitution tests.
+		{
+			name: "Backtick substitution",
+			//           `  c  m  d  `
+			input:    "`cmd`",
+			wantLive: []bool{false, false, false, false, false},
+		},
+		{
+			name: "Backtick with operators inside",
+			//           e  c  h  o     `  a     &  &     b  `
+			input:    "echo `a && b`",
+			wantLive: []bool{true, true, true, true, true, false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Backtick inside double quotes",
+			//           "     `  c  m  d  `     "
+			input:    "\"`cmd`\"",
+			wantLive: []bool{false, false, false, false, false, false, false},
+		},
+		{
+			name:  "Escaped backtick is literal",
+			input: "\\`cmd\\`",
+			//           \  `  c  m  d  \  `
+			wantLive: []bool{false, false, true, true, true, false, false},
+		},
+		{
+			name: "After backtick closes chars are live",
+			//           `  c  `     r
+			input:    "`c` r",
+			wantLive: []bool{false, false, false, true, true},
+		},
+		// ${...} parameter expansion tests.
+		{
+			name: "Parameter expansion ${var}",
+			//           $ {  v  a  r  }
+			input:    "${var}",
+			wantLive: []bool{true, false, false, false, false, false},
+		},
+		{
+			name: "Parameter expansion with default ${var:-default}",
+			//           $ {  v  a  r  :  -  d  e  f  a  u  l  t  }
+			input:    "${var:-default}",
+			wantLive: []bool{true, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "Nested $() inside ${}",
+			//           $ {  v  a  r  :  -  $ (  c  m  d  )  }
+			input:    "${var:-$(cmd)}",
+			wantLive: []bool{true, false, false, false, false, false, false, false, false, false, false, false, false, false},
+		},
+		{
+			name: "After ${} closes chars are live",
+			//           $ {  v  }     r
+			input:    "${v} r",
+			wantLive: []bool{true, false, false, false, true, true},
+		},
+		{
+			name: "${} inside double quotes",
+			//           "     $ {  v  }     "
+			input:    `"${v}"`,
+			wantLive: []bool{false, false, false, false, false, false},
+		},
 	}
 
 	for _, tt := range tests {

data/syntax/splitter_test.go

@@ -178,6 +178,42 @@ func TestSplitAtOperatorsWithInfo(t *testing.T) {
 			name:  "Empty input",
 			input: "",
 		},
+		{
+			name:          "Operators inside command substitution ignored",
+			input:         "echo $(echo a && echo b) && echo c",
+			expectedParts: []string{"echo $(echo a && echo b)", "echo c"},
+			expectedOps:   []OperatorInfo{{Operator: opAnd, Position: 25}},
+		},
+		{
+			name:          "Pipe inside command substitution ignored",
+			input:         "echo $(cat file | grep pattern) | wc",
+			expectedParts: []string{"echo $(cat file | grep pattern)", "wc"},
+			expectedOps:   []OperatorInfo{{Operator: opPipe, Position: 32, IsPipe: true}},
+		},
+		{
+			name:          "Semicolon inside command substitution ignored",
+			input:         "echo $(echo a; echo b); echo c",
+			expectedParts: []string{"echo $(echo a; echo b)", "echo c"},
+			expectedOps:   []OperatorInfo{{Operator: opSemicolon, Position: 22}},
+		},
+		{
+			name:          "Operators inside backtick substitution ignored",
+			input:         "echo `a && b` && echo c",
+			expectedParts: []string{"echo `a && b`", "echo c"},
+			expectedOps:   []OperatorInfo{{Operator: opAnd, Position: 14}},
+		},
+		{
+			name:          "Pipe inside backtick substitution ignored",
+			input:         "echo `cat file | grep x` | wc",
+			expectedParts: []string{"echo `cat file | grep x`", "wc"},
+			expectedOps:   []OperatorInfo{{Operator: opPipe, Position: 25, IsPipe: true}},
+		},
+		{
+			name:          "Operators inside parameter expansion ignored",
+			input:         "echo ${var:-a && b} && echo c",
+			expectedParts: []string{"echo ${var:-a && b}", "echo c"},
+			expectedOps:   []OperatorInfo{{Operator: opAnd, Position: 20}},
+		},
 	}
 
 	for _, tt := range tests {

data/syntax/tokenize_test.go

@@ -89,6 +89,56 @@ func TestTokenizeWithQuotes(t *testing.T) {
 			input:    `echo 'he said "hello"'`,
 			expected: []string{"echo", `'he said "hello"'`},
 		},
+		{
+			name:     "Command substitution kept as single token",
+			input:    "echo $(date)",
+			expected: []string{"echo", "$(date)"},
+		},
+		{
+			name:     "Command substitution with spaces inside",
+			input:    "echo $(echo hello | grep h)",
+			expected: []string{"echo", "$(echo hello | grep h)"},
+		},
+		{
+			name:     "Nested command substitution",
+			input:    "echo $(echo $(date))",
+			expected: []string{"echo", "$(echo $(date))"},
+		},
+		{
+			name:     "Command substitution as assignment prefix",
+			input:    "result=$(cmd) echo done",
+			expected: []string{"result=$(cmd)", "echo", "done"},
+		},
+		{
+			name:     "Arithmetic expansion kept as single token",
+			input:    "echo $((1+2))",
+			expected: []string{"echo", "$((1+2))"},
+		},
+		{
+			name:     "Command substitution inside double quotes",
+			input:    `echo "$(date)"`,
+			expected: []string{"echo", `"$(date)"`},
+		},
+		{
+			name:     "Dollar-paren inside single quotes is literal",
+			input:    `echo '$(date)'`,
+			expected: []string{"echo", `'$(date)'`},
+		},
+		{
+			name:     "Backtick substitution kept as single token",
+			input:    "echo `date -u`",
+			expected: []string{"echo", "`date -u`"},
+		},
+		{
+			name:     "Parameter expansion kept as single token",
+			input:    "echo ${var:-default value}",
+			expected: []string{"echo", "${var:-default value}"},
+		},
+		{
+			name:     "Parameter expansion without spaces",
+			input:    "echo ${HOME} done",
+			expected: []string{"echo", "${HOME}", "done"},
+		},
 	}
 
 	runTokenizeTests(t, tests, tokenizeWithQuotes)