From 22919f3e189e645c0c05cd8aa01088502ce517dc Mon Sep 17 00:00:00 2001 From: Jaakko Heusala Date: Tue, 29 Apr 2025 12:34:01 -0700 Subject: [PATCH 01/32] feat(tools): add base Tool interface and Registry --- pkg/tools/tool.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pkg/tools/tool.go diff --git a/pkg/tools/tool.go b/pkg/tools/tool.go new file mode 100644 index 0000000..81cb5d0 --- /dev/null +++ b/pkg/tools/tool.go @@ -0,0 +1,25 @@ +package tools + +// Tool represents a Gendo tool that can process input and produce output +type Tool interface { + // Process takes input text and returns output text and an optional error + Process(input string) (string, error) +} + +// Registry is a map of tool names to their implementations +type Registry map[string]Tool + +// NewRegistry creates a new empty tool registry +func NewRegistry() Registry { + return make(Registry) +} + +// Register adds a tool to the registry +func (r Registry) Register(name string, tool Tool) { + r[name] = tool +} + +// Get returns a tool by name, or nil if not found +func (r Registry) Get(name string) Tool { + return r[name] +} \ No newline at end of file From 7d3ebe25514be31b9ea37a574eea7ef4af27c691 Mon Sep 17 00:00:00 2001 From: Jaakko Heusala Date: Tue, 29 Apr 2025 12:34:09 -0700 Subject: [PATCH 02/32] feat(tools): implement file I/O tools with tests --- pkg/tools/io/io.go | 90 ++++++++++++++++++++ pkg/tools/io/io_test.go | 176 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 pkg/tools/io/io.go create mode 100644 pkg/tools/io/io_test.go diff --git a/pkg/tools/io/io.go b/pkg/tools/io/io.go new file mode 100644 index 0000000..c99abbc --- /dev/null +++ b/pkg/tools/io/io.go @@ -0,0 +1,90 @@ +package io + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gendo/pkg/log" +) + +// ReadTool implements the tools.Tool interface for file reading +type ReadTool struct { + basePath string +} + +// WriteTool implements the tools.Tool interface for file writing +type WriteTool struct { + basePath string +} + +// NewReadTool creates a new file reading tool +func NewReadTool(basePath string) *ReadTool { + log.Debug("Creating new read tool with base path: %q", basePath) + return &ReadTool{ + basePath: basePath, + } +} + +// NewWriteTool creates a new file writing tool +func NewWriteTool(basePath string) *WriteTool { + log.Debug("Creating new write tool with base path: %q", basePath) + return &WriteTool{ + basePath: basePath, + } +} + +// Process implements the tools.Tool interface for ReadTool +func (t *ReadTool) Process(input string) (string, error) { + log.Debug("Processing read input: %q", input) + + if input == "" { + log.Debug("Empty input provided") + return "", fmt.Errorf("no file path provided") + } + + filePath := input + if t.basePath != "" { + filePath = filepath.Join(t.basePath, input) + log.Debug("Using full file path: %q", filePath) + } + + content, err := os.ReadFile(filePath) + if err != nil { + log.Debug("Failed to read file %q: %v", filePath, err) + return "", fmt.Errorf("failed to read file: %v", err) + } + + log.Debug("Successfully read %d bytes from %q", len(content), filePath) + return string(content), nil +} + +// Process implements the tools.Tool interface for WriteTool +func (t *WriteTool) Process(input string) (string, error) { + log.Debug("Processing write input: %q", input) + + // Split input into file path and content + parts := strings.SplitN(input, " ", 2) + if len(parts) != 2 { + log.Debug("Invalid input format") + return "", fmt.Errorf("invalid input format: expected 'path content'") + } + + filePath := parts[0] + content := parts[1] + + if t.basePath != "" { + filePath = filepath.Join(t.basePath, filePath) + log.Debug("Using full file path: %q", filePath) + } + + err := os.WriteFile(filePath, []byte(content), 0644) + if err != nil { + log.Debug("Failed to write to file %q: %v", filePath, err) + return "", fmt.Errorf("failed to write file: %v", err) + } + + log.Debug("Successfully wrote %d bytes to %q", len(content), filePath) + return fmt.Sprintf("Successfully wrote to %s", filePath), nil +} \ No newline at end of file diff --git a/pkg/tools/io/io_test.go b/pkg/tools/io/io_test.go new file mode 100644 index 0000000..074c9e2 --- /dev/null +++ b/pkg/tools/io/io_test.go @@ -0,0 +1,176 @@ +package io + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadTool(t *testing.T) { + // Create a temporary directory for test files + tmpDir, err := os.MkdirTemp("", "gendo-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create a test file + testContent := "test content" + testFile := filepath.Join(tmpDir, "test.txt") + if err := os.WriteFile(testFile, []byte(testContent), 0644); err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + tests := []struct { + name string + input string + basePath string + want string + }{ + { + name: "read existing file", + input: "test.txt", + basePath: tmpDir, + want: testContent, + }, + { + name: "read non-existent file", + input: "nonexistent.txt", + basePath: tmpDir, + want: "ERROR: Failed to read file:", + }, + { + name: "invalid input", + input: "file1.txt file2.txt", + basePath: tmpDir, + want: "ERROR: Read tool requires a filename", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tool := NewReadTool(tt.basePath) + got := tool.Process(tt.input) + if !strings.HasPrefix(got, tt.want) { + t.Errorf("ReadTool.Process() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestWriteTool(t *testing.T) { + // Create a temporary directory for test files + tmpDir, err := os.MkdirTemp("", "gendo-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + tests := []struct { + name string + input string + basePath string + want string + check func(t *testing.T, path string) + }{ + { + name: "write new file", + input: "test.txt test content", + basePath: tmpDir, + want: "Written to", + check: func(t *testing.T, path string) { + content, err := os.ReadFile(filepath.Join(path, "test.txt")) + if err != nil { + t.Errorf("Failed to read written file: %v", err) + } + if string(content) != "test content" { + t.Errorf("Written content = %v, want %v", string(content), "test content") + } + }, + }, + { + name: "invalid input - no content", + input: "test.txt", + basePath: tmpDir, + want: "ERROR: Write tool requires a filename and content", + check: func(t *testing.T, path string) {}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tool := NewWriteTool(tt.basePath) + got := tool.Process(tt.input) + if !strings.HasPrefix(got, tt.want) { + t.Errorf("WriteTool.Process() = %v, want %v", got, tt.want) + } + tt.check(t, tt.basePath) + }) + } +} + +func BenchmarkReadTool(b *testing.B) { + // Create a temporary directory for benchmark files + tmpDir, err := os.MkdirTemp("", "gendo-bench-*") + if err != nil { + b.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create test files of different sizes + files := map[string]int{ + "small.txt": 100, // 100 bytes + "medium.txt": 10000, // 10KB + "large.txt": 1000000, // 1MB + } + + for filename, size := range files { + content := strings.Repeat("x", size) + filepath := filepath.Join(tmpDir, filename) + if err := os.WriteFile(filepath, []byte(content), 0644); err != nil { + b.Fatalf("Failed to create benchmark file: %v", err) + } + } + + tool := NewReadTool(tmpDir) + + for filename := range files { + b.Run(filename, func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + tool.Process(filename) + } + }) + } +} + +func BenchmarkWriteTool(b *testing.B) { + // Create a temporary directory for benchmark files + tmpDir, err := os.MkdirTemp("", "gendo-bench-*") + if err != nil { + b.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Test data sizes + sizes := map[string]int{ + "small": 100, // 100 bytes + "medium": 10000, // 10KB + "large": 1000000, // 1MB + } + + tool := NewWriteTool(tmpDir) + + for name, size := range sizes { + content := strings.Repeat("x", size) + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + filename := filepath.Join(tmpDir, fmt.Sprintf("bench_%d.txt", i)) + tool.Process(fmt.Sprintf("%s %s", filename, content)) + } + }) + } +} \ No newline at end of file From c6987f67150da952183f86a5e9589f467ee80777 Mon Sep 17 00:00:00 2001 From: Jaakko Heusala Date: Tue, 29 Apr 2025 12:40:45 -0700 Subject: [PATCH 03/32] feat: Initial project setup with core structure and documentation --- .cursor/rules/development-workflow.mdc | 5 + .cursor/rules/project-structure.mdc | 5 + LICENSE | 2 +- Makefile | 55 ++++++++ README.md | 167 ++++++++++++++++++++++++- go.mod | 3 + 6 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 .cursor/rules/development-workflow.mdc create mode 100644 .cursor/rules/project-structure.mdc create mode 100644 Makefile create mode 100644 go.mod diff --git a/.cursor/rules/development-workflow.mdc b/.cursor/rules/development-workflow.mdc new file mode 100644 index 0000000..b93c988 --- /dev/null +++ b/.cursor/rules/development-workflow.mdc @@ -0,0 +1,5 @@ +--- +description: +globs: +alwaysApply: false +--- diff --git a/.cursor/rules/project-structure.mdc b/.cursor/rules/project-structure.mdc new file mode 100644 index 0000000..b93c988 --- /dev/null +++ b/.cursor/rules/project-structure.mdc @@ -0,0 +1,5 @@ +--- +description: +globs: +alwaysApply: false +--- diff --git a/LICENSE b/LICENSE index 4d19e16..a2bbd6e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 HyperifyIO +Copyright (c) 2025 Jaakko Heusala Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..eb73d7f --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +.PHONY: all build test bench clean + +# Default target +all: build + +# Build the project +build: + @echo "Building..." + go build -o gendo ./cmd/gendo + +# Run all tests +test: + @echo "Running tests..." + go test -v ./... + +# Run benchmarks +bench: + @echo "Running benchmarks..." + go test -bench=. -benchmem ./... + +# Run tests with coverage +coverage: + @echo "Running tests with coverage..." + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out + +# Clean build artifacts +clean: + @echo "Cleaning..." + rm -f gendo + rm -f coverage.out + +# Install dependencies +deps: + @echo "Installing dependencies..." + go mod download + go mod tidy + +# Run linter +lint: + @echo "Running linter..." + golangci-lint run + +# Help target +help: + @echo "Available targets:" + @echo " all - Build the project (default)" + @echo " build - Build the project" + @echo " test - Run all tests" + @echo " bench - Run benchmarks" + @echo " coverage - Run tests with coverage report" + @echo " clean - Remove build artifacts" + @echo " deps - Install dependencies" + @echo " lint - Run linter" + @echo " help - Show this help message" \ No newline at end of file diff --git a/README.md b/README.md index a8dcb71..fb6a4b5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,165 @@ -# gendo -Gendo Programming Language +# Gendo Language Specification v0.2 + +## 1. Introduction + +Gendo is a minimalist, prompt-based programming language designed for live, incremental code generation and execution via small, local AI models. Programs consist of self-contained **nodes**—each defining behavior or invoking AI prompts—that pass plain-text streams to each other, enabling rapid composition of functionality without mutable global state or hidden dependencies. + +## 2. Core Concepts + +### 2.1 Nodes + +- **Definition**: `nodeID : refID refID … [: prompt text]` + - `nodeID`: unique integer identifier + - `refID`s: list of nodeIDs this node can call + - `prompt text` *(optional)*: instructions for the AI when this node is invoked +- **Invocation**: `[errorDest !] [dest <] src input text` + - Routes stdout (`dest`) and stderr (`errorDest`) to designated nodes + - Defaults: stdout→1, stderr→2 + +### 2.2 Streams + +- Nodes exchange plain-text. AI-enabled nodes transform input via their prompt; passthrough nodes output input verbatim. +- Errors are first-class data, buffered and routed like stdout. + +### 2.3 Default Handlers + +You can set **default** destinations for stdout and stderr across subsequent invocations by writing a line with only the handler syntax. Whitespace may be used to indent purely for readability; it has no semantic effect. + +```gendo +# Only redefine stdout default to node 3 (errors still go to node 2) +3 < + +# Errors still go to previously set default (node 2) +5 Another input + +# You can override the default by specifying both handlers on a command: +# Here, errors→5, stdout→6 for this line only +5 ! 6 < Overridden command text +``` + +You can individually redefine defaults: + +```gendo +# Only redefine stdout default to node 3 (errors still go to node 2) +3 < + + # Errors still go to previously set default (node 2) + 5 Another input + + # You can override the default by specifying it + 5 < 6 Second command text +``` + +The default handlers remain in effect until redefined or the script ends. + +## 3. Structured Control Flow + +*(Looping and conditionals TBD—let's agree on design here before fleshing out.)* + +## 4. Modular Units & Files + +*(Modular units, namespaces, and imports TBD—let's agree on design before fleshing this out.)* + +## 5. Built-in Utilities + +> **Note:** Each tool-backed node requires enabling the corresponding tool in the Gendo runtime configuration. If a tool (e.g., `math`, `rand`, `read`, `write`) is not enabled, attempting to invoke its node will result in an error. + + +### 5.1 Math + +Gendo uses explicit **tool nodes** for arithmetic. If a node’s ref list includes the special `tool` directive, the runtime connects it to the math evaluator. + +- **Definition Syntax**: `nodeID : tool : math [config...]` + - `tool` marks a tool-backed node. + - Optional `config` may specify precision or mode (e.g., `float`). + +**Example Definition** +```gendo +# Node 50 runs the host math evaluator +50 : tool : math +``` + +**Example Invocation** +```gendo +# Evaluate an expression +< 50 3 * (2 + 5) +# → 21 +``` + +Tool nodes are sandboxed and only execute their designated operation. + +### 5.2 Random + +Gendo defines **tool nodes** for randomness. Including `tool` with `rand` uses the host RNG. + +- **Definition Syntax**: `nodeID : tool : rand [config...]` + - `config` may specify distribution (`uniform`, `normal`) or bounds. + +**Example Definition** +```gendo +# Node 51 runs the host RNG +51 : tool : rand +``` + +**Example Invocation** +```gendo +# Generate a random integer in [1,100] +< 51 1 100 +# → 73 (example) +``` + +Tool nodes are sandboxed and only execute their designated operation. + +### 5.3 I/O & Persistence I/O & Persistence + +Gendo also uses **tool nodes** for safe, sandboxed file operations. Include `tool` in the ref list and specify `read` or `write` as the tool name. + +- **Definition Syntax**: `nodeID : tool : read|write [filename]` + - `read` nodes take no input arguments and output the contents of the named file. + - `write` nodes accept stdin and save it to the named file, returning a confirmation message. + +**Example Definitions** +```gendo +# Node 60 reads "config.json" +60 : tool : read config.json + +# Node 61 writes to "results.txt" +61 : tool : write results.txt +``` + +**Example Invocations** +```gendo +# Load configuration +< 60 +# → {"threshold":10} + +# Save results +Some computed output text +< 61 +# → "Written to results.txt" +``` + +- Filenames are sandboxed and isolated per program; no arbitrary paths allowed. + +## 6. Safety & Concurrency + +Gendo emphasizes reliability and performance: + +- **Stateless Nodes**: By default, nodes have no hidden state; all side effects occur through explicit tool nodes (e.g., I/O), ensuring predictable behavior. +- **Error Handling**: Errors are treated as first-class data. You choose where to route error messages via the `errorDest !` syntax; unhandled errors by default go to node 2. This allows logging, retries, or feeding errors into AI prompts for recovery. +- **Concurrency and Parallelism**: The runtime can execute independent node invocations in parallel when there are no data dependencies. This lets you leverage multi-core CPUs without adding complex syntax. +- **Sandboxing**: Tool nodes (math, rand, read, write) are isolated from arbitrary host resources. Filesystem and network access occur only through sandboxed APIs, preventing unauthorized operations. + +## 7. Data Model + +Gendo operates purely on plain text streams. Each node receives a string and returns a string. For structured data (e.g., JSON), simply define your prompts or AI nodes to parse and emit valid JSON. Gendo does not enforce data schemas, offering maximum flexibility. + +## 8. Community and Next Steps + +Gendo invites developers to build small, focused units that grow at runtime via AI. Its minimal core encourages experimentation: + +- **Extensibility**: Community-contributed tools and node libraries can add capabilities (e.g., HTTP, database connectors) without altering the core. +- **Safety**: All extensions must register as explicit tools and respect sandbox rules. +- **Example Library**: Curated sets of nodes for common tasks (e.g., data processing pipelines, chat bots). + +*Gendo makes it so.* diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d9bae79 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gendo + +go 1.21 From 86b21efd2c015266ef3fc3fba307f6e361feb933 Mon Sep 17 00:00:00 2001 From: Jaakko Heusala Date: Tue, 29 Apr 2025 12:40:55 -0700 Subject: [PATCH 04/32] feat: Add core engine implementation and parser --- cmd/gendo/main.go | 31 ++++++ internal/gendo/gendo.go | 209 +++++++++++++++++++++++++++++++++++ internal/gendo/gendo_test.go | 91 +++++++++++++++ pkg/log/log.go | 55 +++++++++ pkg/parser/parser.go | 136 +++++++++++++++++++++++ pkg/parser/parser_test.go | 173 +++++++++++++++++++++++++++++ 6 files changed, 695 insertions(+) create mode 100644 cmd/gendo/main.go create mode 100644 internal/gendo/gendo.go create mode 100644 internal/gendo/gendo_test.go create mode 100644 pkg/log/log.go create mode 100644 pkg/parser/parser.go create mode 100644 pkg/parser/parser_test.go diff --git a/cmd/gendo/main.go b/cmd/gendo/main.go new file mode 100644 index 0000000..f5714ec --- /dev/null +++ b/cmd/gendo/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "gendo/internal/gendo" + "gendo/pkg/log" +) + +func main() { + verbose := flag.Bool("verbose", false, "Enable verbose logging") + model := flag.String("model", "", "Model to use for LLM (overrides GENDO_MODEL environment variable)") + flag.StringVar(model, "m", "", "Model to use for LLM (shorthand)") + flag.Parse() + + args := flag.Args() + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "Usage: %s [-verbose] [-m model]