diff --git a/CLAUDE.md b/CLAUDE.md index b00f6cf..47006ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,14 +55,7 @@ Tests use actual markdown files with docci tags: ### Key Design Patterns 1. **Result Aggregation**: `DocciResult` struct contains success status, exit codes, and output -2. **Tag-based Execution**: Code blocks use docci tags for advanced behavior: - - `docci-exec` - Execute the block - - `docci-background` - Run in background - - `docci-output-contains` - Validate output - - `docci-assert-failure` - Expect failure - - `docci-retry` - Retry on failure - - `docci-delay` - Wait before execution - - `docci-wait-for-endpoint` - Wait for service availability +2. **Tag-based Execution**: Code blocks use docci tags for advanced behavior. You can run the `docci tags` command to see all information on them and some more context. 3. **Multi-file Processing**: Blocks from multiple files are merged with global indexing 4. **Real-time Streaming**: Executor streams output in real-time while capturing for validation diff --git a/parser/codeblocks.go b/parser/codeblocks.go index 1cb8f2b..c825267 100644 --- a/parser/codeblocks.go +++ b/parser/codeblocks.go @@ -257,21 +257,16 @@ func BuildExecutableScriptWithOptions(blocks []CodeBlock, opts types.DocciOpts) validationMap := make(map[int]string) // maps block index to expected output assertFailureMap := make(map[int]bool) // maps block index to assert-failure flag var backgroundPIDs []string + debugEnabled := log.Level >= logrus.DebugLevel // Always generate markers for parsing, visibility controlled in executor // Add trap at the beginning to clean up background processes // Only set the trap if keepRunning is false if !opts.KeepRunning { - script.WriteString("# Cleanup function for background processes\n") - script.WriteString("cleanup_background_processes() {\n") - // higher numbers are actually more verbose in the logrus library - if log.Level >= logrus.DebugLevel { - script.WriteString(" echo 'Cleaning up background processes...'\n") - } - script.WriteString(" jobs -p | xargs -r kill 2>/dev/null\n") - script.WriteString("}\n") - script.WriteString("trap cleanup_background_processes EXIT\n\n") + script.WriteString(replaceTemplateVars(scriptCleanupTemplate, map[string]string{ + "DEBUG_CLEANUP": formatDebugCleanup(debugEnabled), + })) } var backgroundIndexes []int @@ -279,96 +274,58 @@ func BuildExecutableScriptWithOptions(blocks []CodeBlock, opts types.DocciOpts) for _, block := range blocks { // Handle background kill first if specified if block.BackgroundKill > 0 { - // Kill a previously started background process - fileInfo := "" - if block.FileName != "" { - fileInfo = fmt.Sprintf(" from %s", block.FileName) - } - script.WriteString(fmt.Sprintf("# Kill background process at index %d%s\n", block.BackgroundKill, fileInfo)) - script.WriteString(fmt.Sprintf("if [ -n \"$DOCCI_BG_PID_%d\" ]; then\n", block.BackgroundKill)) - script.WriteString(fmt.Sprintf(" echo 'Killing background process %d with PID '$DOCCI_BG_PID_%d\n", block.BackgroundKill, block.BackgroundKill)) - script.WriteString(fmt.Sprintf(" # Kill the entire process group\n")) - script.WriteString(fmt.Sprintf(" kill -TERM -$DOCCI_BG_PID_%d 2>/dev/null || kill $DOCCI_BG_PID_%d 2>/dev/null || true\n", block.BackgroundKill, block.BackgroundKill)) - script.WriteString(fmt.Sprintf(" wait $DOCCI_BG_PID_%d 2>/dev/null || true\n", block.BackgroundKill)) - script.WriteString(fmt.Sprintf(" unset DOCCI_BG_PID_%d\n", block.BackgroundKill)) - script.WriteString("else\n") - script.WriteString(fmt.Sprintf(" echo 'Warning: No background process found at index %d'\n", block.BackgroundKill)) - script.WriteString("fi\n\n") + script.WriteString(replaceTemplateVars(backgroundKillTemplate, map[string]string{ + "KILL_INDEX": strconv.Itoa(block.BackgroundKill), + "FILE_INFO": formatFileInfo(block.FileName), + })) } if block.Background { // For background blocks, wrap in { } & and redirect output - fileInfo := "" - if block.FileName != "" { - fileInfo = fmt.Sprintf(" from %s", block.FileName) - } - script.WriteString(fmt.Sprintf("# Background block %d%s\n", block.Index, fileInfo)) - script.WriteString("setsid bash -c '{\n") - script.WriteString(block.Content) - script.WriteString(fmt.Sprintf("}' > /tmp/docci_bg_%d.out 2>&1 &\n", block.Index)) - script.WriteString(fmt.Sprintf("DOCCI_BG_PID_%d=$!\n", block.Index)) - script.WriteString(fmt.Sprintf("echo 'Started background process %d with PID '$DOCCI_BG_PID_%d\n\n", block.Index, block.Index)) + script.WriteString(replaceTemplateVars(backgroundBlockTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "FILE_INFO": formatFileInfo(block.FileName), + "CONTENT": block.Content, + })) backgroundPIDs = append(backgroundPIDs, fmt.Sprintf("$DOCCI_BG_PID_%d", block.Index)) backgroundIndexes = append(backgroundIndexes, block.Index) } else { // Regular blocks with markers (always generated for parsing) - marker := fmt.Sprintf("### DOCCI_BLOCK_START_%d ###", block.Index) - script.WriteString(fmt.Sprintf("echo '%s'\n", marker)) + script.WriteString(replaceTemplateVars(blockStartMarkerTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + })) // Add the block header comment only in debug mode - if log.Level >= logrus.DebugLevel { - fileInfo := "" - if block.FileName != "" { - fileInfo = fmt.Sprintf(" from %s", block.FileName) - } - script.WriteString(fmt.Sprintf("### === Code Block %d (%s)%s ===\n", block.Index, block.Language, fileInfo)) + if debugEnabled { + script.WriteString(replaceTemplateVars(blockHeaderTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "LANGUAGE": block.Language, + "FILE_INFO": formatFileInfo(block.FileName), + })) } // Add delay before block if specified if block.DelayBeforeSecs > 0 { - script.WriteString(fmt.Sprintf("# Delay before block %d for %g seconds\n", block.Index, block.DelayBeforeSecs)) - script.WriteString(fmt.Sprintf("sleep %g\n", block.DelayBeforeSecs)) + script.WriteString(replaceTemplateVars(delayBeforeTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "DELAY": strconv.FormatFloat(block.DelayBeforeSecs, 'g', -1, 64), + })) } // Add wait-for-endpoint logic if needed if block.WaitForEndpoint != "" { - script.WriteString(fmt.Sprintf("# Waiting for endpoint %s (timeout: %d seconds)\n", block.WaitForEndpoint, block.WaitTimeoutSecs)) - script.WriteString(fmt.Sprintf("echo 'Waiting for endpoint %s to be ready...'\n", block.WaitForEndpoint)) - script.WriteString(fmt.Sprintf(` -timeout_secs=%d -endpoint_url="%s" -start_time=$(date +%%s) - -while true; do - current_time=$(date +%%s) - elapsed=$((current_time - start_time)) - - if [ $elapsed -ge $timeout_secs ]; then - echo "Timeout waiting for endpoint $endpoint_url after $timeout_secs seconds" - exit 1 - fi - - if curl -s -f --max-time 5 "$endpoint_url" > /dev/null 2>&1; then - echo "Endpoint $endpoint_url is ready" - break - fi - - echo "Endpoint not ready yet, retrying in 1 second... (elapsed: ${elapsed}s)" - sleep 1 -done - -`, block.WaitTimeoutSecs, block.WaitForEndpoint)) + script.WriteString(replaceTemplateVars(waitForEndpointTemplate, map[string]string{ + "ENDPOINT": block.WaitForEndpoint, + "TIMEOUT": strconv.Itoa(block.WaitTimeoutSecs), + })) } // Add file existence check as guard clause if needed if block.IfFileNotExists != "" { - script.WriteString(fmt.Sprintf("# Guard clause: check if file exists and skip if it does\n")) - script.WriteString(fmt.Sprintf("if [ -f \"%s\" ]; then\n", block.IfFileNotExists)) - script.WriteString(fmt.Sprintf(" echo \"Skipping block %d: file %s already exists\"\n", block.Index, block.IfFileNotExists)) - script.WriteString("else\n") - script.WriteString(fmt.Sprintf(" echo \"File %s does not exist, executing block %d\"\n", block.IfFileNotExists, block.Index)) - script.WriteString("fi\n") - script.WriteString(fmt.Sprintf("if [ ! -f \"%s\" ]; then\n", block.IfFileNotExists)) + script.WriteString(replaceTemplateVars(fileExistenceGuardStartTemplate, map[string]string{ + "FILE": block.IfFileNotExists, + "INDEX": strconv.Itoa(block.Index), + })) } // Apply text replacement if needed @@ -385,48 +342,24 @@ done // Prepare the code content with per-command delay and command display delaySeconds := block.DelayPerCmdSecs - bashFlags := "-eT" - if block.AssertFailure { - bashFlags = "-T" // Don't use -e for assert-failure blocks - } - codeContent := fmt.Sprintf(`# Enable per-command delay (%g seconds) and command display -set %s -trap 'echo -e "\n Executing CMD: $BASH_COMMAND" >&2; sleep %g' DEBUG - -%s - -# Disable trap -trap - DEBUG -`, delaySeconds, bashFlags, delaySeconds, blockContent) + codeContent := replaceTemplateVars(codeExecutionTemplate, map[string]string{ + "DELAY": strconv.FormatFloat(delaySeconds, 'g', -1, 64), + "BASH_FLAGS": formatBashFlags(block.AssertFailure), + "CONTENT": blockContent, + }) // Add the actual code with retry logic if needed if block.RetryCount > 0 { retryDelay := GetRetryDelay() - script.WriteString(fmt.Sprintf("# Retry logic for block %d (max attempts: %d)\n", block.Index, block.RetryCount)) - script.WriteString("retry_count=0\n") - script.WriteString(fmt.Sprintf("max_retries=%d\n", block.RetryCount)) - script.WriteString("while [ $retry_count -le $max_retries ]; do\n") - script.WriteString(" if [ $retry_count -gt 0 ]; then\n") - script.WriteString(fmt.Sprintf(" echo \"Retry attempt $retry_count/$max_retries for block %d\"\n", block.Index)) - if retryDelay > 0 { - script.WriteString(fmt.Sprintf(" sleep %d\n", retryDelay)) - } - script.WriteString(" fi\n") - script.WriteString(" \n") - script.WriteString(" # Execute the block content\n") - script.WriteString(" if (\n") + script.WriteString(replaceTemplateVars(retryWrapperStartTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "MAX_RETRIES": strconv.Itoa(block.RetryCount), + "RETRY_DELAY": strconv.Itoa(retryDelay), + })) script.WriteString(codeContent) - script.WriteString(" ); then\n") - script.WriteString(" break\n") - script.WriteString(" else\n") - script.WriteString(" exit_code=$?\n") - script.WriteString(" retry_count=$((retry_count + 1))\n") - script.WriteString(" if [ $retry_count -gt $max_retries ]; then\n") - script.WriteString(fmt.Sprintf(" echo \"Block %d failed after $max_retries retry attempts\"\n", block.Index)) - script.WriteString(" exit $exit_code\n") - script.WriteString(" fi\n") - script.WriteString(" fi\n") - script.WriteString("done\n") + script.WriteString(replaceTemplateVars(retryWrapperEndTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + })) } else { script.WriteString(codeContent) } @@ -438,13 +371,16 @@ trap - DEBUG // Add delay after block if specified if block.DelayAfterSecs > 0 { - script.WriteString(fmt.Sprintf("# Delay after block %d for %g seconds\n", block.Index, block.DelayAfterSecs)) - script.WriteString(fmt.Sprintf("sleep %g\n", block.DelayAfterSecs)) + script.WriteString(replaceTemplateVars(delayAfterTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "DELAY": strconv.FormatFloat(block.DelayAfterSecs, 'g', -1, 64), + })) } // Add a marker after the block - endMarker := fmt.Sprintf("### DOCCI_BLOCK_END_%d ###", block.Index) - script.WriteString(fmt.Sprintf("echo '%s'\n", endMarker)) + script.WriteString(replaceTemplateVars(blockEndMarkerTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + })) // Store validation requirement if present if block.OutputContains != "" { @@ -459,43 +395,31 @@ trap - DEBUG // Add section to display background logs at the end (unless hidden) if len(backgroundIndexes) > 0 && !opts.HideBackgroundLogs { - script.WriteString("\n# Display background process logs\n") - script.WriteString("echo -e '\\n=== Background Process Logs ==='\n") + var logEntries strings.Builder for _, bgIndex := range backgroundIndexes { - script.WriteString(fmt.Sprintf("if [ -f /tmp/docci_bg_%d.out ]; then\n", bgIndex)) - script.WriteString(fmt.Sprintf(" echo -e '\\n--- Background Block %d Output ---'\n", bgIndex)) - script.WriteString(fmt.Sprintf(" cat /tmp/docci_bg_%d.out\n", bgIndex)) - script.WriteString(fmt.Sprintf(" rm -f /tmp/docci_bg_%d.out\n", bgIndex)) - script.WriteString("else\n") - script.WriteString(fmt.Sprintf(" echo 'No output file found for background block %d'\n", bgIndex)) - script.WriteString("fi\n") + logEntries.WriteString(replaceTemplateVars(backgroundLogEntryTemplate, map[string]string{ + "INDEX": strconv.Itoa(bgIndex), + })) } + script.WriteString(replaceTemplateVars(backgroundLogsDisplayTemplate, map[string]string{ + "LOG_ENTRIES": logEntries.String(), + })) } else if len(backgroundIndexes) > 0 && opts.HideBackgroundLogs { // Still clean up the background output files even if we're not displaying them - script.WriteString("\n# Clean up background process logs (hidden)\n") + var cleanupCommands strings.Builder for _, bgIndex := range backgroundIndexes { - script.WriteString(fmt.Sprintf("rm -f /tmp/docci_bg_%d.out\n", bgIndex)) + cleanupCommands.WriteString(fmt.Sprintf("rm -f /tmp/docci_bg_%d.out\n", bgIndex)) } + script.WriteString(replaceTemplateVars(backgroundLogsCleanupTemplate, map[string]string{ + "CLEANUP_COMMANDS": cleanupCommands.String(), + })) } // Add infinite sleep if keepRunning is true (as a final block) if opts.KeepRunning { - script.WriteString("\n# Keep containers running with infinite sleep\n") - script.WriteString("echo '\\nšŸ”„ Keeping containers running. Press Ctrl+C to stop...'\n") - - // Add trap for cleanup when keepRunning is true - script.WriteString("\n# Cleanup function for background processes (on interrupt)\n") - script.WriteString("cleanup_on_interrupt() {\n") - // higher numbers are actually more verbose in the logrus library - if log.Level >= logrus.DebugLevel { - script.WriteString(" echo 'Cleaning up background processes...'\n") - } - script.WriteString(" jobs -p | xargs -r kill 2>/dev/null\n") - script.WriteString(" exit 0\n") - script.WriteString("}\n") - script.WriteString("trap cleanup_on_interrupt INT TERM\n\n") - - script.WriteString("sleep infinity\n") + script.WriteString(replaceTemplateVars(keepRunningTemplate, map[string]string{ + "DEBUG_CLEANUP": formatDebugCleanup(debugEnabled), + })) } return script.String(), validationMap, assertFailureMap diff --git a/parser/script_templates.go b/parser/script_templates.go new file mode 100644 index 0000000..d9dbe5a --- /dev/null +++ b/parser/script_templates.go @@ -0,0 +1,171 @@ +package parser + +// Script templates for bash code generation +const ( + // Main script template with cleanup trap + scriptCleanupTemplate = `# Cleanup function for background processes +cleanup_background_processes() { +{{DEBUG_CLEANUP}} jobs -p | xargs -r kill 2>/dev/null +} +trap cleanup_background_processes EXIT + +` + + // Background kill template + backgroundKillTemplate = `# Kill background process at index {{KILL_INDEX}}{{FILE_INFO}} +if [ -n "$DOCCI_BG_PID_{{KILL_INDEX}}" ]; then + echo 'Killing background process {{KILL_INDEX}} with PID '$DOCCI_BG_PID_{{KILL_INDEX}} + # Kill the entire process group + kill -TERM -$DOCCI_BG_PID_{{KILL_INDEX}} 2>/dev/null || kill $DOCCI_BG_PID_{{KILL_INDEX}} 2>/dev/null || true + wait $DOCCI_BG_PID_{{KILL_INDEX}} 2>/dev/null || true + unset DOCCI_BG_PID_{{KILL_INDEX}} +else + echo 'Warning: No background process found at index {{KILL_INDEX}}' +fi + +` + + // Background block template + backgroundBlockTemplate = `# Background block {{INDEX}}{{FILE_INFO}} +setsid bash -c '{ +{{CONTENT}}}' > /tmp/docci_bg_{{INDEX}}.out 2>&1 & +DOCCI_BG_PID_{{INDEX}}=$! +echo 'Started background process {{INDEX}} with PID '$DOCCI_BG_PID_{{INDEX}} + +` + + // Regular block start marker + blockStartMarkerTemplate = `echo '### DOCCI_BLOCK_START_{{INDEX}} ###' +` + + // Block header (debug mode only) + blockHeaderTemplate = `### === Code Block {{INDEX}} ({{LANGUAGE}}){{FILE_INFO}} === +` + + // Delay before template + delayBeforeTemplate = `# Delay before block {{INDEX}} for {{DELAY}} seconds +sleep {{DELAY}} +` + + // Wait for endpoint template + waitForEndpointTemplate = `# Waiting for endpoint {{ENDPOINT}} (timeout: {{TIMEOUT}} seconds) +echo 'Waiting for endpoint {{ENDPOINT}} to be ready...' + +timeout_secs={{TIMEOUT}} +endpoint_url="{{ENDPOINT}}" +start_time=$(date +%s) + +while true; do + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + + if [ $elapsed -ge $timeout_secs ]; then + echo "Timeout waiting for endpoint $endpoint_url after $timeout_secs seconds" + exit 1 + fi + + if curl -s -f --max-time 5 "$endpoint_url" > /dev/null 2>&1; then + echo "Endpoint $endpoint_url is ready" + break + fi + + echo "Endpoint not ready yet, retrying in 1 second... (elapsed: ${elapsed}s)" + sleep 1 +done + +` + + // File existence guard template + fileExistenceGuardStartTemplate = `# Guard clause: check if file exists and skip if it does +if [ -f "{{FILE}}" ]; then + echo "Skipping block {{INDEX}}: file {{FILE}} already exists" +else + echo "File {{FILE}} does not exist, executing block {{INDEX}}" +fi +if [ ! -f "{{FILE}}" ]; then +` + + // Code execution with per-command delay template + codeExecutionTemplate = `# Enable per-command delay ({{DELAY}} seconds) and command display +set {{BASH_FLAGS}} +trap 'echo -e "\n Executing CMD: $BASH_COMMAND" >&2; sleep {{DELAY}}' DEBUG + +{{CONTENT}} + +# Disable trap +trap - DEBUG +` + + // Retry wrapper start template + retryWrapperStartTemplate = `# Retry logic for block {{INDEX}} (max attempts: {{MAX_RETRIES}}) +retry_count=0 +max_retries={{MAX_RETRIES}} +while [ $retry_count -le $max_retries ]; do + if [ $retry_count -gt 0 ]; then + echo "Retry attempt $retry_count/$max_retries for block {{INDEX}}" + sleep {{RETRY_DELAY}} + fi + + # Execute the block content + if ( +` + + // Retry wrapper end template + retryWrapperEndTemplate = ` ); then + break + else + exit_code=$? + retry_count=$((retry_count + 1)) + if [ $retry_count -gt $max_retries ]; then + echo "Block {{INDEX}} failed after $max_retries retry attempts" + exit $exit_code + fi + fi +done +` + + // Delay after template + delayAfterTemplate = `# Delay after block {{INDEX}} for {{DELAY}} seconds +sleep {{DELAY}} +` + + // Block end marker + blockEndMarkerTemplate = `echo '### DOCCI_BLOCK_END_{{INDEX}} ###' +` + + // Background logs display template + backgroundLogsDisplayTemplate = ` +# Display background process logs +echo -e '\n=== Background Process Logs ===' +{{LOG_ENTRIES}}` + + // Single background log entry template + backgroundLogEntryTemplate = `if [ -f /tmp/docci_bg_{{INDEX}}.out ]; then + echo -e '\n--- Background Block {{INDEX}} Output ---' + cat /tmp/docci_bg_{{INDEX}}.out + rm -f /tmp/docci_bg_{{INDEX}}.out +else + echo 'No output file found for background block {{INDEX}}' +fi +` + + // Background logs cleanup template (hidden) + backgroundLogsCleanupTemplate = ` +# Clean up background process logs (hidden) +{{CLEANUP_COMMANDS}}` + + // Keep running template + keepRunningTemplate = ` +# Keep containers running with infinite sleep +echo '\nšŸ”„ Keeping containers running. Press Ctrl+C to stop...' + +# Cleanup function for background processes (on interrupt) +cleanup_on_interrupt() { +{{DEBUG_CLEANUP}} jobs -p | xargs -r kill 2>/dev/null + exit 0 +} +trap cleanup_on_interrupt INT TERM + +sleep infinity +` +) diff --git a/parser/template_helpers.go b/parser/template_helpers.go new file mode 100644 index 0000000..d3453bf --- /dev/null +++ b/parser/template_helpers.go @@ -0,0 +1,66 @@ +package parser + +import ( + "fmt" + "regexp" + "strings" +) + +// replaceTemplateVars replaces template variables with their values +func replaceTemplateVars(template string, vars map[string]string) string { + result := template + for key, value := range vars { + result = strings.ReplaceAll(result, "{{"+key+"}}", value) + } + + // Check for any remaining unreplaced variables + remaining := findUnreplacedVars(result) + if len(remaining) > 0 { + panic(fmt.Sprintf("Unreplaced template variables found: %v.\nTemplate:\n%v\n", remaining, result)) + } + + return result +} + +// findUnreplacedVars finds any remaining {{VARIABLE}} patterns in the text using regex +func findUnreplacedVars(text string) []string { + var unreplaced []string + seen := make(map[string]bool) + + // Regex to match {{VARIABLE}} patterns + re := regexp.MustCompile(`\{\{[^}]+\}\}`) + matches := re.FindAllString(text, -1) + + for _, match := range matches { + if !seen[match] { + unreplaced = append(unreplaced, match) + seen[match] = true + } + } + + return unreplaced +} + +// formatFileInfo returns a formatted file info string +func formatFileInfo(fileName string) string { + if fileName != "" { + return fmt.Sprintf(" from %s", fileName) + } + return "" +} + +// formatDebugCleanup returns debug cleanup message if debug level is enabled +func formatDebugCleanup(debugEnabled bool) string { + if debugEnabled { + return " echo 'Cleaning up background processes...'\n" + } + return "" +} + +// formatBashFlags returns appropriate bash flags based on assert failure setting +func formatBashFlags(assertFailure bool) string { + if assertFailure { + return "-T" // Don't use -e for assert-failure blocks + } + return "-eT" +}