From 5bb3d1872c4db839b87682b6c7931f7f328ba7da Mon Sep 17 00:00:00 2001 From: Jacob Repp Date: Sun, 19 Oct 2025 11:31:23 -0700 Subject: [PATCH 1/4] Enhance validate_docs.py with improved code block validation and import handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User request: "look at the improvements made in ~/hc/cloud-agf-devportal/tooling/validate_docs.py and merge those changes into our local implementation create a feature branch for this work and make sure you're up to date with origin/main" Key improvements merged from cloud-agf-devportal: 1. **Better import handling**: Added fallback strategy to import doc_schemas - Try importing as package (tooling.doc_schemas) first - Fall back to module import (doc_schemas) if package import fails - Improves script portability and execution from different contexts 2. **Enhanced code block validation with blank line checking**: - Validates blank line before opening code fence (except after frontmatter/document start) - Validates blank line after closing code fence (except at document end) - Tracks frontmatter boundaries to handle edge cases correctly - Tracks previous line blank state for accurate validation - Detects closing fence followed immediately by content 3. **Enforced lowercase filename standard**: - Changed patterns to lowercase-only (adr-, rfc-, memo-) - Removed case-insensitive matching (no more ADR-, RFC-, MEMO-) - Updated error messages to indicate lowercase requirement - Aligns with project's lowercase naming standard Testing shows script correctly identifies: - 32 files with uppercase names needing lowercase conversion - Missing blank lines in code blocks in intro.md and analysis-summary.md - All existing validation checks continue to work correctly πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- tooling/validate_docs.py | 71 +++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/tooling/validate_docs.py b/tooling/validate_docs.py index 55402042e..957023397 100755 --- a/tooling/validate_docs.py +++ b/tooling/validate_docs.py @@ -42,7 +42,11 @@ import yaml from pydantic import ValidationError - from tooling.doc_schemas import ADRFrontmatter, GenericDocFrontmatter, MemoFrontmatter, RFCFrontmatter + # Try importing as package first, then as module + try: + from tooling.doc_schemas import ADRFrontmatter, GenericDocFrontmatter, MemoFrontmatter, RFCFrontmatter + except ModuleNotFoundError: + from doc_schemas import ADRFrontmatter, GenericDocFrontmatter, MemoFrontmatter, RFCFrontmatter ENHANCED_VALIDATION = True except ImportError as e: @@ -144,11 +148,11 @@ def scan_documents(self): """Scan all markdown files""" self.log("\nπŸ“‚ Scanning documents...") - # Filename patterns (adr-NNN-name-with-dashes.md or ADR-NNN-name-with-dashes.md) - # Accept both lowercase (new standard) and uppercase (legacy) formats - adr_pattern = re.compile(r"^(adr|ADR)-(\d{3})-(.+)\.md$", re.IGNORECASE) - rfc_pattern = re.compile(r"^(rfc|RFC)-(\d{3})-(.+)\.md$", re.IGNORECASE) - memo_pattern = re.compile(r"^(memo|MEMO)-(\d{3})-(.+)\.md$", re.IGNORECASE) + # Filename patterns (adr-NNN-name-with-dashes.md) + # ENFORCE lowercase only - uppercase is deprecated + adr_pattern = re.compile(r"^(adr)-(\d{3})-(.+)\.md$") + rfc_pattern = re.compile(r"^(rfc)-(\d{3})-(.+)\.md$") + memo_pattern = re.compile(r"^(memo)-(\d{3})-(.+)\.md$") # Scan ADRs adr_dir = self.repo_root / "docs-cms" / "adr" @@ -161,9 +165,9 @@ def scan_documents(self): match = adr_pattern.match(md_file.name) if not match: self.errors.append( - f"Invalid ADR filename: {md_file.name} (expected: adr-NNN-name-with-dashes.md or ADR-NNN-name-with-dashes.md)" + f"Invalid ADR filename: {md_file.name} (expected: adr-NNN-name-with-dashes.md - lowercase only)" ) - self.log(f" βœ— {md_file.name}: Invalid filename format") + self.log(f" βœ— {md_file.name}: Invalid filename format (must be lowercase)") continue _prefix, num, _slug = match.groups() @@ -188,9 +192,9 @@ def scan_documents(self): match = rfc_pattern.match(md_file.name) if not match: self.errors.append( - f"Invalid RFC filename: {md_file.name} (expected: rfc-NNN-name-with-dashes.md or RFC-NNN-name-with-dashes.md)" + f"Invalid RFC filename: {md_file.name} (expected: rfc-NNN-name-with-dashes.md - lowercase only)" ) - self.log(f" βœ— {md_file.name}: Invalid filename format") + self.log(f" βœ— {md_file.name}: Invalid filename format (must be lowercase)") continue _prefix, num, _slug = match.groups() @@ -215,9 +219,9 @@ def scan_documents(self): match = memo_pattern.match(md_file.name) if not match: self.errors.append( - f"Invalid MEMO filename: {md_file.name} (expected: memo-NNN-name-with-dashes.md or MEMO-NNN-name-with-dashes.md)" + f"Invalid MEMO filename: {md_file.name} (expected: memo-NNN-name-with-dashes.md - lowercase only)" ) - self.log(f" βœ— {md_file.name}: Invalid filename format") + self.log(f" βœ— {md_file.name}: Invalid filename format (must be lowercase)") continue _prefix, num, _slug = match.groups() @@ -723,6 +727,8 @@ def check_code_blocks(self): Rules (per CommonMark/MDX spec): - Opening fence: ``` followed by optional language (e.g., ```python, ```text) - Closing fence: ``` with NO language or other text + - Blank line required before opening fence (except at document start or after frontmatter) + - Blank line required after closing fence (except at document end) - Content: Everything between opening and closing is treated as content """ self.log("\nπŸ“ Checking code blocks...") @@ -742,6 +748,9 @@ def check_code_blocks(self): opening_language = None doc_valid_blocks = 0 doc_invalid_blocks = 0 + closing_fence_line = None # Track last closing fence for newline check + previous_line_blank = True # Track if previous line was blank + frontmatter_end_line = None # Track where frontmatter ends for line_num, line in enumerate(lines, start=1): stripped = line.strip() @@ -753,6 +762,7 @@ def check_code_blocks(self): in_frontmatter = True elif frontmatter_count == 2: in_frontmatter = False + frontmatter_end_line = line_num continue # Skip frontmatter content @@ -763,6 +773,19 @@ def check_code_blocks(self): # Per CommonMark: fence must be at least 3 backticks fence_match = re.match(r"^(`{3,})", stripped) if not fence_match: + # Check if previous line was a closing fence and this line is not blank + if closing_fence_line is not None and closing_fence_line == line_num - 1: + if stripped and line_num <= len(lines): + # Non-blank line immediately after closing fence + error = f"Line {closing_fence_line}: Missing blank line after closing code fence (found content at line {line_num})" + doc.errors.append(error) + self.log(f" βœ— {doc.file_path.name}:{closing_fence_line} - No blank line after closing fence") + doc_invalid_blocks += 1 + total_invalid += 1 + closing_fence_line = None # Reset after check + + # Track if this line is blank for next iteration + previous_line_blank = not stripped continue # This is a fence line @@ -770,7 +793,19 @@ def check_code_blocks(self): remainder = stripped[len(fence_backticks) :].strip() if not in_code_block: - # Opening fence + # Opening fence - check for blank line before + # Exception: first line after frontmatter or very beginning of content + content_start = (frontmatter_end_line + 1) if frontmatter_end_line else 1 + is_after_frontmatter = frontmatter_end_line and line_num == frontmatter_end_line + 1 + is_document_start = line_num == content_start + + if not previous_line_blank and not is_after_frontmatter and not is_document_start: + error = f"Line {line_num}: Missing blank line before opening code fence" + doc.errors.append(error) + self.log(f" βœ— {doc.file_path.name}:{line_num} - No blank line before opening fence") + doc_invalid_blocks += 1 + total_invalid += 1 + if not remainder: # Bare opening fence - INVALID (must have language) error = f"Line {line_num}: Opening code fence missing language (use ```text for plain text)" @@ -787,6 +822,9 @@ def check_code_blocks(self): in_code_block = True opening_line = line_num opening_language = remainder.split()[0] if remainder else "" + + # Reset blank line tracking when entering code block + previous_line_blank = False else: # Closing fence if remainder: @@ -801,11 +839,14 @@ def check_code_blocks(self): # Valid closing fence doc_valid_blocks += 1 total_valid += 1 + closing_fence_line = line_num # Mark for newline check # Mark block as closed regardless in_code_block = False opening_line = None opening_language = None + # Reset blank line tracking - fence line itself is not blank + previous_line_blank = False # Check for unclosed code block if in_code_block: @@ -880,8 +921,8 @@ def check_ids(self): # Extract expected ID from filename filename = doc.file_path.name - # Match adr-XXX, rfc-XXX, or memo-XXX pattern (case insensitive) - filename_pattern = re.compile(r"^(adr|rfc|memo)-(\d{3})-", re.IGNORECASE) + # Match adr-XXX, rfc-XXX, or memo-XXX pattern (lowercase only) + filename_pattern = re.compile(r"^(adr|rfc|memo)-(\d{3})-") match = filename_pattern.match(filename) if not match: From 9e5cd02914e3fbe5be136e6d86fb7a91f474c6c0 Mon Sep 17 00:00:00 2001 From: Jacob Repp Date: Sun, 19 Oct 2025 11:37:29 -0700 Subject: [PATCH 2/4] Rename uppercase documentation files to lowercase and add code block spacing fixer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User request: "use the validate_docs script to begin fixing our documentation" Fixed 32 filename violations identified by enhanced validate_docs.py: **ADR Files (2)**: - ADR-000-template.md β†’ adr-000-template.md - ADR-058-proxy-drain-on-shutdown.md β†’ adr-058-proxy-drain-on-shutdown.md **RFC Files (11)**: - RFC-028 through RFC-040 β†’ rfc-028 through rfc-040 **MEMO Files (19)**: - MEMO-010 through MEMO-036 β†’ memo-010 through memo-036 All renames follow lowercase naming convention (adr-, rfc-, memo-) as enforced by the enhanced validation script. Also added: - tooling/fix_code_block_spacing.py: Script to automatically fix missing blank lines before/after code fences (for future use) This fixes all 32 filename validation errors. Remaining issues are code block blank line formatting (87 documents) which will be addressed separately. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- ...DR-000-template.md => adr-000-template.md} | 0 ....md => adr-058-proxy-drain-on-shutdown.md} | 0 ...md => memo-010-poc1-edge-case-analysis.md} | 0 ...ce.md => memo-012-developer-experience.md} | 0 ... memo-013-poc1-infrastructure-analysis.md} | 0 ...memo-014-pattern-sdk-shared-complexity.md} | 0 ...ross-backend-acceptance-test-framework.md} | 0 ...observability-lifecycle-implementation.md} | 0 ... memo-017-message-schema-configuration.md} | 0 ...y.md => memo-018-poc4-complete-summary.md} | 0 ...md => memo-019-loadtest-results-100rps.md} | 0 ...020-parallel-testing-and-build-hygiene.md} | 0 ...md => memo-021-parallel-linting-system.md} | 0 ... memo-022-prismctl-integration-testing.md} | 0 ...o-030-pattern-based-acceptance-testing.md} | 0 ...031-rfc031-security-performance-review.md} | 0 ... => memo-032-driver-test-consolidation.md} | 0 ...033-process-isolation-bulkhead-pattern.md} | 0 ...> memo-034-pattern-launcher-quickstart.md} | 0 ...memo-035-nomad-local-development-setup.md} | 0 ...mo-036-kubernetes-operator-development.md} | 0 ...t.md => rfc-028-prism-probe-cli-client.md} | 0 ...030-schema-evolution-pubsub-validation.md} | 0 ...d => rfc-031-message-envelope-protocol.md} | 0 ...fc-032-minimal-schema-registry-standin.md} | 0 ...tern.md => rfc-033-claim-check-pattern.md} | 0 ...r.md => rfc-034-robust-process-manager.md} | 0 ...md => rfc-035-pattern-process-launcher.md} | 0 ...mailbox-pattern-searchable-event-store.md} | 0 ... => rfc-038-admin-leader-election-raft.md} | 0 ...rfc-039-backend-configuration-registry.md} | 0 ....md => rfc-040-client-sdk-architecture.md} | 0 tooling/fix_code_block_spacing.py | 152 ++++++++++++++++++ 33 files changed, 152 insertions(+) rename docs-cms/adr/{ADR-000-template.md => adr-000-template.md} (100%) rename docs-cms/adr/{ADR-058-proxy-drain-on-shutdown.md => adr-058-proxy-drain-on-shutdown.md} (100%) rename docs-cms/memos/{MEMO-010-poc1-edge-case-analysis.md => memo-010-poc1-edge-case-analysis.md} (100%) rename docs-cms/memos/{MEMO-012-developer-experience.md => memo-012-developer-experience.md} (100%) rename docs-cms/memos/{MEMO-013-poc1-infrastructure-analysis.md => memo-013-poc1-infrastructure-analysis.md} (100%) rename docs-cms/memos/{MEMO-014-pattern-sdk-shared-complexity.md => memo-014-pattern-sdk-shared-complexity.md} (100%) rename docs-cms/memos/{MEMO-015-cross-backend-acceptance-test-framework.md => memo-015-cross-backend-acceptance-test-framework.md} (100%) rename docs-cms/memos/{MEMO-016-observability-lifecycle-implementation.md => memo-016-observability-lifecycle-implementation.md} (100%) rename docs-cms/memos/{MEMO-017-message-schema-configuration.md => memo-017-message-schema-configuration.md} (100%) rename docs-cms/memos/{MEMO-018-poc4-complete-summary.md => memo-018-poc4-complete-summary.md} (100%) rename docs-cms/memos/{MEMO-019-loadtest-results-100rps.md => memo-019-loadtest-results-100rps.md} (100%) rename docs-cms/memos/{MEMO-020-parallel-testing-and-build-hygiene.md => memo-020-parallel-testing-and-build-hygiene.md} (100%) rename docs-cms/memos/{MEMO-021-parallel-linting-system.md => memo-021-parallel-linting-system.md} (100%) rename docs-cms/memos/{MEMO-022-prismctl-integration-testing.md => memo-022-prismctl-integration-testing.md} (100%) rename docs-cms/memos/{MEMO-030-pattern-based-acceptance-testing.md => memo-030-pattern-based-acceptance-testing.md} (100%) rename docs-cms/memos/{MEMO-031-rfc031-security-performance-review.md => memo-031-rfc031-security-performance-review.md} (100%) rename docs-cms/memos/{MEMO-032-driver-test-consolidation.md => memo-032-driver-test-consolidation.md} (100%) rename docs-cms/memos/{MEMO-033-process-isolation-bulkhead-pattern.md => memo-033-process-isolation-bulkhead-pattern.md} (100%) rename docs-cms/memos/{MEMO-034-pattern-launcher-quickstart.md => memo-034-pattern-launcher-quickstart.md} (100%) rename docs-cms/memos/{MEMO-035-nomad-local-development-setup.md => memo-035-nomad-local-development-setup.md} (100%) rename docs-cms/memos/{MEMO-036-kubernetes-operator-development.md => memo-036-kubernetes-operator-development.md} (100%) rename docs-cms/rfcs/{RFC-028-prism-probe-cli-client.md => rfc-028-prism-probe-cli-client.md} (100%) rename docs-cms/rfcs/{RFC-030-schema-evolution-pubsub-validation.md => rfc-030-schema-evolution-pubsub-validation.md} (100%) rename docs-cms/rfcs/{RFC-031-message-envelope-protocol.md => rfc-031-message-envelope-protocol.md} (100%) rename docs-cms/rfcs/{RFC-032-minimal-schema-registry-standin.md => rfc-032-minimal-schema-registry-standin.md} (100%) rename docs-cms/rfcs/{RFC-033-claim-check-pattern.md => rfc-033-claim-check-pattern.md} (100%) rename docs-cms/rfcs/{RFC-034-robust-process-manager.md => rfc-034-robust-process-manager.md} (100%) rename docs-cms/rfcs/{RFC-035-pattern-process-launcher.md => rfc-035-pattern-process-launcher.md} (100%) rename docs-cms/rfcs/{RFC-037-mailbox-pattern-searchable-event-store.md => rfc-037-mailbox-pattern-searchable-event-store.md} (100%) rename docs-cms/rfcs/{RFC-038-admin-leader-election-raft.md => rfc-038-admin-leader-election-raft.md} (100%) rename docs-cms/rfcs/{RFC-039-backend-configuration-registry.md => rfc-039-backend-configuration-registry.md} (100%) rename docs-cms/rfcs/{RFC-040-client-sdk-architecture.md => rfc-040-client-sdk-architecture.md} (100%) create mode 100755 tooling/fix_code_block_spacing.py diff --git a/docs-cms/adr/ADR-000-template.md b/docs-cms/adr/adr-000-template.md similarity index 100% rename from docs-cms/adr/ADR-000-template.md rename to docs-cms/adr/adr-000-template.md diff --git a/docs-cms/adr/ADR-058-proxy-drain-on-shutdown.md b/docs-cms/adr/adr-058-proxy-drain-on-shutdown.md similarity index 100% rename from docs-cms/adr/ADR-058-proxy-drain-on-shutdown.md rename to docs-cms/adr/adr-058-proxy-drain-on-shutdown.md diff --git a/docs-cms/memos/MEMO-010-poc1-edge-case-analysis.md b/docs-cms/memos/memo-010-poc1-edge-case-analysis.md similarity index 100% rename from docs-cms/memos/MEMO-010-poc1-edge-case-analysis.md rename to docs-cms/memos/memo-010-poc1-edge-case-analysis.md diff --git a/docs-cms/memos/MEMO-012-developer-experience.md b/docs-cms/memos/memo-012-developer-experience.md similarity index 100% rename from docs-cms/memos/MEMO-012-developer-experience.md rename to docs-cms/memos/memo-012-developer-experience.md diff --git a/docs-cms/memos/MEMO-013-poc1-infrastructure-analysis.md b/docs-cms/memos/memo-013-poc1-infrastructure-analysis.md similarity index 100% rename from docs-cms/memos/MEMO-013-poc1-infrastructure-analysis.md rename to docs-cms/memos/memo-013-poc1-infrastructure-analysis.md diff --git a/docs-cms/memos/MEMO-014-pattern-sdk-shared-complexity.md b/docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md similarity index 100% rename from docs-cms/memos/MEMO-014-pattern-sdk-shared-complexity.md rename to docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md diff --git a/docs-cms/memos/MEMO-015-cross-backend-acceptance-test-framework.md b/docs-cms/memos/memo-015-cross-backend-acceptance-test-framework.md similarity index 100% rename from docs-cms/memos/MEMO-015-cross-backend-acceptance-test-framework.md rename to docs-cms/memos/memo-015-cross-backend-acceptance-test-framework.md diff --git a/docs-cms/memos/MEMO-016-observability-lifecycle-implementation.md b/docs-cms/memos/memo-016-observability-lifecycle-implementation.md similarity index 100% rename from docs-cms/memos/MEMO-016-observability-lifecycle-implementation.md rename to docs-cms/memos/memo-016-observability-lifecycle-implementation.md diff --git a/docs-cms/memos/MEMO-017-message-schema-configuration.md b/docs-cms/memos/memo-017-message-schema-configuration.md similarity index 100% rename from docs-cms/memos/MEMO-017-message-schema-configuration.md rename to docs-cms/memos/memo-017-message-schema-configuration.md diff --git a/docs-cms/memos/MEMO-018-poc4-complete-summary.md b/docs-cms/memos/memo-018-poc4-complete-summary.md similarity index 100% rename from docs-cms/memos/MEMO-018-poc4-complete-summary.md rename to docs-cms/memos/memo-018-poc4-complete-summary.md diff --git a/docs-cms/memos/MEMO-019-loadtest-results-100rps.md b/docs-cms/memos/memo-019-loadtest-results-100rps.md similarity index 100% rename from docs-cms/memos/MEMO-019-loadtest-results-100rps.md rename to docs-cms/memos/memo-019-loadtest-results-100rps.md diff --git a/docs-cms/memos/MEMO-020-parallel-testing-and-build-hygiene.md b/docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md similarity index 100% rename from docs-cms/memos/MEMO-020-parallel-testing-and-build-hygiene.md rename to docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md diff --git a/docs-cms/memos/MEMO-021-parallel-linting-system.md b/docs-cms/memos/memo-021-parallel-linting-system.md similarity index 100% rename from docs-cms/memos/MEMO-021-parallel-linting-system.md rename to docs-cms/memos/memo-021-parallel-linting-system.md diff --git a/docs-cms/memos/MEMO-022-prismctl-integration-testing.md b/docs-cms/memos/memo-022-prismctl-integration-testing.md similarity index 100% rename from docs-cms/memos/MEMO-022-prismctl-integration-testing.md rename to docs-cms/memos/memo-022-prismctl-integration-testing.md diff --git a/docs-cms/memos/MEMO-030-pattern-based-acceptance-testing.md b/docs-cms/memos/memo-030-pattern-based-acceptance-testing.md similarity index 100% rename from docs-cms/memos/MEMO-030-pattern-based-acceptance-testing.md rename to docs-cms/memos/memo-030-pattern-based-acceptance-testing.md diff --git a/docs-cms/memos/MEMO-031-rfc031-security-performance-review.md b/docs-cms/memos/memo-031-rfc031-security-performance-review.md similarity index 100% rename from docs-cms/memos/MEMO-031-rfc031-security-performance-review.md rename to docs-cms/memos/memo-031-rfc031-security-performance-review.md diff --git a/docs-cms/memos/MEMO-032-driver-test-consolidation.md b/docs-cms/memos/memo-032-driver-test-consolidation.md similarity index 100% rename from docs-cms/memos/MEMO-032-driver-test-consolidation.md rename to docs-cms/memos/memo-032-driver-test-consolidation.md diff --git a/docs-cms/memos/MEMO-033-process-isolation-bulkhead-pattern.md b/docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md similarity index 100% rename from docs-cms/memos/MEMO-033-process-isolation-bulkhead-pattern.md rename to docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md diff --git a/docs-cms/memos/MEMO-034-pattern-launcher-quickstart.md b/docs-cms/memos/memo-034-pattern-launcher-quickstart.md similarity index 100% rename from docs-cms/memos/MEMO-034-pattern-launcher-quickstart.md rename to docs-cms/memos/memo-034-pattern-launcher-quickstart.md diff --git a/docs-cms/memos/MEMO-035-nomad-local-development-setup.md b/docs-cms/memos/memo-035-nomad-local-development-setup.md similarity index 100% rename from docs-cms/memos/MEMO-035-nomad-local-development-setup.md rename to docs-cms/memos/memo-035-nomad-local-development-setup.md diff --git a/docs-cms/memos/MEMO-036-kubernetes-operator-development.md b/docs-cms/memos/memo-036-kubernetes-operator-development.md similarity index 100% rename from docs-cms/memos/MEMO-036-kubernetes-operator-development.md rename to docs-cms/memos/memo-036-kubernetes-operator-development.md diff --git a/docs-cms/rfcs/RFC-028-prism-probe-cli-client.md b/docs-cms/rfcs/rfc-028-prism-probe-cli-client.md similarity index 100% rename from docs-cms/rfcs/RFC-028-prism-probe-cli-client.md rename to docs-cms/rfcs/rfc-028-prism-probe-cli-client.md diff --git a/docs-cms/rfcs/RFC-030-schema-evolution-pubsub-validation.md b/docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md similarity index 100% rename from docs-cms/rfcs/RFC-030-schema-evolution-pubsub-validation.md rename to docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md diff --git a/docs-cms/rfcs/RFC-031-message-envelope-protocol.md b/docs-cms/rfcs/rfc-031-message-envelope-protocol.md similarity index 100% rename from docs-cms/rfcs/RFC-031-message-envelope-protocol.md rename to docs-cms/rfcs/rfc-031-message-envelope-protocol.md diff --git a/docs-cms/rfcs/RFC-032-minimal-schema-registry-standin.md b/docs-cms/rfcs/rfc-032-minimal-schema-registry-standin.md similarity index 100% rename from docs-cms/rfcs/RFC-032-minimal-schema-registry-standin.md rename to docs-cms/rfcs/rfc-032-minimal-schema-registry-standin.md diff --git a/docs-cms/rfcs/RFC-033-claim-check-pattern.md b/docs-cms/rfcs/rfc-033-claim-check-pattern.md similarity index 100% rename from docs-cms/rfcs/RFC-033-claim-check-pattern.md rename to docs-cms/rfcs/rfc-033-claim-check-pattern.md diff --git a/docs-cms/rfcs/RFC-034-robust-process-manager.md b/docs-cms/rfcs/rfc-034-robust-process-manager.md similarity index 100% rename from docs-cms/rfcs/RFC-034-robust-process-manager.md rename to docs-cms/rfcs/rfc-034-robust-process-manager.md diff --git a/docs-cms/rfcs/RFC-035-pattern-process-launcher.md b/docs-cms/rfcs/rfc-035-pattern-process-launcher.md similarity index 100% rename from docs-cms/rfcs/RFC-035-pattern-process-launcher.md rename to docs-cms/rfcs/rfc-035-pattern-process-launcher.md diff --git a/docs-cms/rfcs/RFC-037-mailbox-pattern-searchable-event-store.md b/docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md similarity index 100% rename from docs-cms/rfcs/RFC-037-mailbox-pattern-searchable-event-store.md rename to docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md diff --git a/docs-cms/rfcs/RFC-038-admin-leader-election-raft.md b/docs-cms/rfcs/rfc-038-admin-leader-election-raft.md similarity index 100% rename from docs-cms/rfcs/RFC-038-admin-leader-election-raft.md rename to docs-cms/rfcs/rfc-038-admin-leader-election-raft.md diff --git a/docs-cms/rfcs/RFC-039-backend-configuration-registry.md b/docs-cms/rfcs/rfc-039-backend-configuration-registry.md similarity index 100% rename from docs-cms/rfcs/RFC-039-backend-configuration-registry.md rename to docs-cms/rfcs/rfc-039-backend-configuration-registry.md diff --git a/docs-cms/rfcs/RFC-040-client-sdk-architecture.md b/docs-cms/rfcs/rfc-040-client-sdk-architecture.md similarity index 100% rename from docs-cms/rfcs/RFC-040-client-sdk-architecture.md rename to docs-cms/rfcs/rfc-040-client-sdk-architecture.md diff --git a/tooling/fix_code_block_spacing.py b/tooling/fix_code_block_spacing.py new file mode 100755 index 000000000..bdebde6bd --- /dev/null +++ b/tooling/fix_code_block_spacing.py @@ -0,0 +1,152 @@ +#!/usr/bin/env -S uv run python3 +"""Fix code block spacing issues in markdown files + +Automatically adds missing blank lines: +- Before opening code fences (except at document start or after frontmatter) +- After closing code fences (except at document end) + +Usage: + uv run tooling/fix_code_block_spacing.py [file2.md ...] + uv run tooling/fix_code_block_spacing.py docs-cms/adr/*.md +""" + +import re +import sys +from pathlib import Path + + +def fix_code_block_spacing(file_path: Path) -> tuple[bool, int]: + """Fix code block spacing in a markdown file + + Returns: + (changed, num_fixes): Whether file was modified and number of fixes applied + """ + try: + content = file_path.read_text(encoding="utf-8") + lines = content.split("\n") + + fixed_lines = [] + in_code_block = False + in_frontmatter = False + frontmatter_count = 0 + num_fixes = 0 + + for i, line in enumerate(lines): + stripped = line.strip() + + # Track frontmatter (first --- to second ---) + if stripped == "---": + frontmatter_count += 1 + if frontmatter_count == 1: + in_frontmatter = True + elif frontmatter_count == 2: + in_frontmatter = False + fixed_lines.append(line) + continue + + # Skip frontmatter content + if in_frontmatter: + fixed_lines.append(line) + continue + + # Check if this line is a code fence + fence_match = re.match(r"^(`{3,})", stripped) + + if fence_match: + fence_backticks = fence_match.group(1) + remainder = stripped[len(fence_backticks):].strip() + + if not in_code_block: + # Opening fence - check if we need blank line before + if len(fixed_lines) > 0: + prev_line = fixed_lines[-1] if fixed_lines else "" + + # Check if previous line is blank + if prev_line.strip() != "": + # Check if we're right after frontmatter + is_after_frontmatter = (frontmatter_count == 2 and + len(fixed_lines) >= 1 and + fixed_lines[-1].strip() == "---") + + if not is_after_frontmatter: + # Add blank line before opening fence + fixed_lines.append("") + num_fixes += 1 + + in_code_block = True + fixed_lines.append(line) + else: + # Closing fence + fixed_lines.append(line) + in_code_block = False + + # Check if we need blank line after (look ahead) + if i + 1 < len(lines): + next_line = lines[i + 1] + if next_line.strip() != "": + # Need to add blank line after this closing fence + # We'll mark this by checking on next iteration + pass + else: + # Not a fence line + # Check if previous line was a closing fence and this line is not blank + if (len(fixed_lines) > 0 and + not in_code_block and + stripped != ""): + + # Check if previous line was a closing fence + prev_line = fixed_lines[-1] + if re.match(r"^```\s*$", prev_line.strip()): + # Previous was closing fence, this is content, need blank line + fixed_lines.append("") + num_fixes += 1 + + fixed_lines.append(line) + + # Join and compare + new_content = "\n".join(fixed_lines) + + if new_content != content: + file_path.write_text(new_content, encoding="utf-8") + return True, num_fixes + + return False, 0 + + except Exception as e: + print(f"Error processing {file_path}: {e}", file=sys.stderr) + return False, 0 + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + + files = [Path(f) for f in sys.argv[1:]] + + total_changed = 0 + total_fixes = 0 + + for file_path in files: + if not file_path.exists(): + print(f"⚠️ File not found: {file_path}") + continue + + if not file_path.suffix == ".md": + print(f"⚠️ Skipping non-markdown file: {file_path}") + continue + + changed, num_fixes = fix_code_block_spacing(file_path) + + if changed: + print(f"βœ“ {file_path.name}: Fixed {num_fixes} spacing issue(s)") + total_changed += 1 + total_fixes += num_fixes + else: + print(f" {file_path.name}: No changes needed") + + print(f"\nβœ… Fixed {total_fixes} spacing issues in {total_changed}/{len(files)} files") + + +if __name__ == "__main__": + main() From d070725fe5cea975ab076a771145854b47dfb9a2 Mon Sep 17 00:00:00 2001 From: Jacob Repp Date: Sun, 19 Oct 2025 11:47:13 -0700 Subject: [PATCH 3/4] Fix linting errors in tooling scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User request: "merge with main, re-run lint" Merged main branch and fixed linting issues found by ruff and black: - Remove unused variables (fence_backticks, remainder) in fix_code_block_spacing.py - Replace 'not x == y' with 'x != y' for cleaner comparison - Apply ruff format to ensure consistent formatting All ruff checks now pass. Pylint shows 9.99/10 rating with only nested block complexity warnings (acceptable). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- tooling/fix_code_block_spacing.py | 16 +++++----------- tooling/validate_docs.py | 4 +++- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tooling/fix_code_block_spacing.py b/tooling/fix_code_block_spacing.py index bdebde6bd..5dba84eb7 100755 --- a/tooling/fix_code_block_spacing.py +++ b/tooling/fix_code_block_spacing.py @@ -53,9 +53,6 @@ def fix_code_block_spacing(file_path: Path) -> tuple[bool, int]: fence_match = re.match(r"^(`{3,})", stripped) if fence_match: - fence_backticks = fence_match.group(1) - remainder = stripped[len(fence_backticks):].strip() - if not in_code_block: # Opening fence - check if we need blank line before if len(fixed_lines) > 0: @@ -64,9 +61,9 @@ def fix_code_block_spacing(file_path: Path) -> tuple[bool, int]: # Check if previous line is blank if prev_line.strip() != "": # Check if we're right after frontmatter - is_after_frontmatter = (frontmatter_count == 2 and - len(fixed_lines) >= 1 and - fixed_lines[-1].strip() == "---") + is_after_frontmatter = ( + frontmatter_count == 2 and len(fixed_lines) >= 1 and fixed_lines[-1].strip() == "---" + ) if not is_after_frontmatter: # Add blank line before opening fence @@ -90,10 +87,7 @@ def fix_code_block_spacing(file_path: Path) -> tuple[bool, int]: else: # Not a fence line # Check if previous line was a closing fence and this line is not blank - if (len(fixed_lines) > 0 and - not in_code_block and - stripped != ""): - + if len(fixed_lines) > 0 and not in_code_block and stripped != "": # Check if previous line was a closing fence prev_line = fixed_lines[-1] if re.match(r"^```\s*$", prev_line.strip()): @@ -132,7 +126,7 @@ def main(): print(f"⚠️ File not found: {file_path}") continue - if not file_path.suffix == ".md": + if file_path.suffix != ".md": print(f"⚠️ Skipping non-markdown file: {file_path}") continue diff --git a/tooling/validate_docs.py b/tooling/validate_docs.py index 957023397..e564d9d1c 100755 --- a/tooling/validate_docs.py +++ b/tooling/validate_docs.py @@ -779,7 +779,9 @@ def check_code_blocks(self): # Non-blank line immediately after closing fence error = f"Line {closing_fence_line}: Missing blank line after closing code fence (found content at line {line_num})" doc.errors.append(error) - self.log(f" βœ— {doc.file_path.name}:{closing_fence_line} - No blank line after closing fence") + self.log( + f" βœ— {doc.file_path.name}:{closing_fence_line} - No blank line after closing fence" + ) doc_invalid_blocks += 1 total_invalid += 1 closing_fence_line = None # Reset after check From d3e142aa8408758b0ef18aad3efcd015cc397e4b Mon Sep 17 00:00:00 2001 From: Jacob Repp Date: Sun, 19 Oct 2025 14:10:21 -0700 Subject: [PATCH 4/4] Fix code block spacing in 111 documentation files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automatically fixed 1519 spacing issues across ADRs, RFCs, and MEMOs: - Added missing blank lines before opening code fences - Added missing blank lines after closing code fences - Ensures MDX compatibility and proper rendering in Docusaurus Fixed files: - 58 ADRs: adr-002 through adr-057 - 35 RFCs: rfc-001 through rfc-040 - 18 MEMOs: memo-002 through memo-036 All documentation now passes validation with zero errors. User request: "switch to PR 18, update it to latest by merging main and address any CI failures including doc validation" πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docs-cms/adr/README.md | 1 + ...adr-002-client-originated-configuration.md | 11 +++ ...adr-003-protobuf-single-source-of-truth.md | 11 +++ docs-cms/adr/adr-004-local-first-testing.md | 11 +++ .../adr-005-backend-plugin-architecture.md | 3 + .../adr/adr-006-namespace-multi-tenancy.md | 18 +++++ .../adr-007-authentication-authorization.md | 1 + .../adr/adr-008-observability-strategy.md | 7 ++ .../adr/adr-009-shadow-traffic-migrations.md | 15 ++++ docs-cms/adr/adr-010-caching-layer.md | 7 ++ .../adr/adr-011-implementation-roadmap.md | 51 +++++++++++++ docs-cms/adr/adr-012-go-for-tooling.md | 9 +++ docs-cms/adr/adr-015-go-testing-strategy.md | 8 +++ docs-cms/adr/adr-016-go-cli-configuration.md | 3 + docs-cms/adr/adr-017-go-structured-logging.md | 9 +++ .../adr-018-rust-error-handling-strategy.md | 13 ++++ docs-cms/adr/adr-020-rust-testing-strategy.md | 10 +++ .../adr/adr-021-rust-structured-logging.md | 19 +++++ .../adr-022-dynamic-client-configuration.md | 12 ++++ .../adr-023-grpc-first-interface-design.md | 31 ++++++++ .../adr/adr-025-container-plugin-model.md | 25 +++++++ .../adr-026-distroless-container-images.md | 5 ++ .../adr/adr-028-admin-ui-fastapi-grpc-web.md | 7 ++ ...dr-029-protocol-recording-protobuf-tags.md | 12 ++++ .../adr-030-schema-recording-protobuf-tags.md | 10 +++ .../adr/adr-031-ttl-defaults-client-data.md | 23 ++++++ .../adr/adr-032-object-storage-pattern.md | 1 + docs-cms/adr/adr-033-capability-api.md | 3 + docs-cms/adr/adr-034-sharding-strategy.md | 13 ++++ docs-cms/adr/adr-035-connection-pooling.md | 3 + docs-cms/adr/adr-036-sqlite-config-storage.md | 9 +++ docs-cms/adr/adr-037-kubernetes-operator.md | 5 ++ .../adr/adr-038-backend-connector-buffers.md | 16 +++++ .../adr/adr-039-cli-acceptance-testing.md | 13 ++++ docs-cms/adr/adr-040-go-binary-admin-cli.md | 23 ++++++ docs-cms/adr/adr-042-sqs-queue-backend.md | 2 + .../adr-043-plugin-capability-discovery.md | 1 + .../adr/adr-044-tinkerpop-gremlin-plugin.md | 25 +++++++ .../adr/adr-045-prismctl-stack-management.md | 33 +++++++++ ...r-047-opentelemetry-tracing-integration.md | 4 ++ .../adr/adr-048-local-signoz-observability.md | 15 ++++ .../adr-049-podman-container-optimization.md | 8 +++ .../adr/adr-051-minio-claim-check-testing.md | 6 ++ .../adr/adr-052-object-store-interface.md | 15 ++++ ...-053-claim-check-ttl-garbage-collection.md | 8 +++ .../adr/adr-055-proxy-admin-control-plane.md | 2 + .../adr-056-launcher-admin-control-plane.md | 1 + .../adr/adr-057-prism-launcher-refactoring.md | 8 +++ docs-cms/analysis-summary.md | 3 + .../memos/memo-002-admin-protocol-review.md | 18 +++++ ...emo-003-documentation-first-development.md | 12 ++++ ...004-backend-plugin-implementation-guide.md | 32 +++++++++ ...o-005-client-protocol-design-philosophy.md | 16 +++++ ...interface-decomposition-schema-registry.md | 5 ++ .../memo-007-podman-scratch-container-demo.md | 10 +++ .../memo-008-vault-token-exchange-flow.md | 1 + ...09-topaz-local-authorizer-configuration.md | 7 ++ .../memos/memo-010-poc1-edge-case-analysis.md | 11 +++ .../memo-011-error-handling-best-practices.md | 13 ++++ .../memos/memo-012-developer-experience.md | 3 + .../memo-013-poc1-infrastructure-analysis.md | 6 ++ .../memo-014-pattern-sdk-shared-complexity.md | 6 ++ ...-observability-lifecycle-implementation.md | 27 +++++++ .../memo-017-message-schema-configuration.md | 6 ++ .../memos/memo-019-loadtest-results-100rps.md | 2 + ...-020-parallel-testing-and-build-hygiene.md | 10 +++ .../memo-022-prismctl-integration-testing.md | 16 +++++ docs-cms/memos/memo-023-tenancy-models.md | 20 ++++++ ...mo-030-pattern-based-acceptance-testing.md | 13 ++++ .../memo-032-driver-test-consolidation.md | 17 +++++ ...-033-process-isolation-bulkhead-pattern.md | 1 + .../memo-034-pattern-launcher-quickstart.md | 4 ++ .../memo-035-nomad-local-development-setup.md | 6 ++ ...emo-036-kubernetes-operator-development.md | 21 ++++++ docs-cms/prd.md | 5 ++ docs-cms/rfcs/rfc-001-prism-architecture.md | 12 ++++ docs-cms/rfcs/rfc-002-data-layer-interface.md | 61 ++++++++++++++++ docs-cms/rfcs/rfc-003-admin-interface.md | 40 +++++++++++ docs-cms/rfcs/rfc-004-redis-integration.md | 2 + .../rfcs/rfc-005-clickhouse-integration.md | 7 ++ docs-cms/rfcs/rfc-006-python-admin-cli.md | 62 ++++++++++++++++ docs-cms/rfcs/rfc-007-cache-strategies.md | 1 + .../rfcs/rfc-008-proxy-plugin-architecture.md | 29 ++++++++ ...fc-009-distributed-reliability-patterns.md | 32 +++++++++ docs-cms/rfcs/rfc-010-admin-protocol-oidc.md | 46 ++++++++++++ .../rfcs/rfc-011-data-proxy-authentication.md | 43 +++++++++++ .../rfcs/rfc-012-prism-netgw-control-plane.md | 31 ++++++++ .../rfcs/rfc-013-neptune-graph-backend.md | 9 +++ .../rfc-014-layered-data-access-patterns.md | 71 +++++++++++++++++++ ...fc-015-plugin-acceptance-test-framework.md | 7 ++ ...fc-016-local-development-infrastructure.md | 51 +++++++++++++ .../rfc-017-multicast-registry-pattern.md | 7 ++ .../rfc-018-poc-implementation-strategy.md | 14 ++++ .../rfc-019-plugin-sdk-authorization-layer.md | 3 + ...020-streaming-http-listener-api-adapter.md | 9 +++ ...c-021-poc1-three-plugins-implementation.md | 8 +++ .../rfc-022-core-plugin-sdk-code-layout.md | 23 ++++++ .../rfc-023-publish-snapshotter-plugin.md | 3 + ...c-024-distributed-session-store-pattern.md | 13 ++++ .../rfcs/rfc-025-pattern-sdk-architecture.md | 7 ++ ...rfc-026-poc1-keyvalue-memstore-original.md | 22 ++++++ ...espace-configuration-client-perspective.md | 6 ++ ...c-029-load-testing-framework-evaluation.md | 10 +++ ...-030-schema-evolution-pubsub-validation.md | 2 + .../rfcs/rfc-031-message-envelope-protocol.md | 23 ++++++ .../rfcs/rfc-034-robust-process-manager.md | 3 + .../rfcs/rfc-035-pattern-process-launcher.md | 3 + ...-mailbox-pattern-searchable-event-store.md | 4 ++ .../rfc-038-admin-leader-election-raft.md | 1 + .../rfc-039-backend-configuration-registry.md | 2 + .../rfcs/rfc-040-client-sdk-architecture.md | 15 ++++ 111 files changed, 1519 insertions(+) diff --git a/docs-cms/adr/README.md b/docs-cms/adr/README.md index cb2e27b09..c9397be04 100644 --- a/docs-cms/adr/README.md +++ b/docs-cms/adr/README.md @@ -329,6 +329,7 @@ graph TB 6. Update status when accepted **Frontmatter Format:** + ```markdown --- title: "ADR-XXX: Descriptive Title" diff --git a/docs-cms/adr/adr-002-client-originated-configuration.md b/docs-cms/adr/adr-002-client-originated-configuration.md index b1bbac245..c0fc2a601 100644 --- a/docs-cms/adr/adr-002-client-originated-configuration.md +++ b/docs-cms/adr/adr-002-client-originated-configuration.md @@ -187,6 +187,7 @@ As organizations grow, traditional manual provisioning breaks down: - ❌ **Restricted**: Replication factors, partition counts **Example**: + ```protobuf message UserEvents { option (prism.backend) = "kafka"; // βœ… Allowed @@ -205,6 +206,7 @@ message UserEvents { - ❌ **Restricted**: Encryption key management overrides **Example**: + ```protobuf message HighThroughputLogs { option (prism.backend) = "kafka"; @@ -361,12 +363,14 @@ teams: - Will this impact cluster capacity? (check resource availability) - Is the backend choice optimal? (suggest alternatives if not) 5. If approved, update `policies/teams.yaml`: + ```yaml - name: user-platform-team permission_level: guided limits: max_write_rps: 100000 # ← Increased ``` + 6. Team redeploys successfully **Key Benefits**: @@ -378,6 +382,7 @@ teams: ### Common Configuration Mistakes Prevented **1. Excessive Retention Leading to Cost Overruns** + ```protobuf // ❌ Rejected at deploy time message DebugLogs { @@ -387,6 +392,7 @@ message DebugLogs { ``` **2. Wrong Backend for Access Pattern** + ```protobuf // ⚠️ Warning at deploy time message HighThroughputEvents { @@ -398,6 +404,7 @@ message HighThroughputEvents { ``` **3. Over-Provisioning Resources** + ```protobuf // ❌ Rejected at deploy time message UserSessions { @@ -428,6 +435,7 @@ message UserSessions { ### Future Enhancements **Automated Permission Elevation**: + ```yaml auto_approve_conditions: - if: team.track_record > 6_months && team.incidents == 0 @@ -438,6 +446,7 @@ auto_approve_conditions: ``` **Cost Budgeting Integration**: + ```protobuf message ExpensiveData { option (prism.estimated_cost_per_month) = 5000; // $5k/month @@ -550,11 +559,13 @@ impl CapacityPlanner { ### Evolution Strategy **Phase 1** (MVP): Support explicit backend selection + ```protobuf option (prism.backend) = "postgres"; ``` **Phase 2**: Add access pattern hints + ```protobuf option (prism.access_pattern) = "read_heavy"; option (prism.estimated_read_rps) = "10000"; diff --git a/docs-cms/adr/adr-003-protobuf-single-source-of-truth.md b/docs-cms/adr/adr-003-protobuf-single-source-of-truth.md index 7dfa11317..2e0f81648 100644 --- a/docs-cms/adr/adr-003-protobuf-single-source-of-truth.md +++ b/docs-cms/adr/adr-003-protobuf-single-source-of-truth.md @@ -127,11 +127,13 @@ proto/*.proto └──> Deployment configs β”œβ”€β”€ Capacity specs └── Backend routing + ```text ### Example: Complete Data Model ``` + // user_profile.proto syntax = "proto3"; @@ -183,12 +185,14 @@ message ProfileSettings { string timezone = 2; string language = 3; } + ```text This **single file** generates: 1. **Rust structs** with validation: ``` + #[derive(Clone, PartialEq, Message)] pub struct UserProfile { #[prost(string, tag = "1")] @@ -205,10 +209,12 @@ impl UserProfile { // ... } } + ```text 2. **Postgres schema**: ``` + CREATE TABLE user_profile ( user_id UUID PRIMARY KEY, email VARCHAR(255) NOT NULL, @@ -220,10 +226,12 @@ CREATE TABLE user_profile ( ); CREATE INDEX idx_user_profile_email ON user_profile(email); + ```text 3. **TypeScript types** for admin UI: ``` + export interface UserProfile { userId: string; email: string; @@ -233,10 +241,12 @@ export interface UserProfile { updatedAt: number; settings?: ProfileSettings; } + ```text 4. **Deployment config** (auto-generated): ``` + name: user-profile backend: postgres capacity: @@ -246,6 +256,7 @@ capacity: cache: enabled: true ttl_seconds: 300 + ```text ### Alternatives Considered diff --git a/docs-cms/adr/adr-004-local-first-testing.md b/docs-cms/adr/adr-004-local-first-testing.md index 81a00f8cd..366dd797c 100644 --- a/docs-cms/adr/adr-004-local-first-testing.md +++ b/docs-cms/adr/adr-004-local-first-testing.md @@ -61,11 +61,13 @@ Developer Laptop β”‚ β”‚ β€’ Neptune (localstack) β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Local Stack Configuration ``` + # docker-compose.test.yml version: '3.9' @@ -136,11 +138,13 @@ services: - "8182:8182" volumes: - /var/run/docker.sock:/var/run/docker.sock + ```text ### Python Tooling ``` + # tooling/test/local_stack.py import subprocess @@ -211,11 +215,13 @@ if __name__ == "__main__": stack.down() elif args.command == "reset": stack.reset() + ```text ### Test Structure ``` + // proxy/tests/integration/keyvalue_test.rs use prism_proxy::*; @@ -282,6 +288,7 @@ async fn load_test_keyvalue_writes() { assert!(throughput > 500.0, "Throughput too low: {}", throughput); } + ```text ### Alternatives Considered @@ -367,6 +374,7 @@ async fn load_test_keyvalue_writes() { ### CI Configuration ``` + # .github/workflows/test.yml name: Tests @@ -391,11 +399,13 @@ jobs: - name: Stop local stack if: always() run: python -m tooling.test.local-stack down + ```text ### Developer Workflow ``` + # One-time setup curl -LsSf https://astral.sh/uv/install.sh | sh uv sync @@ -409,6 +419,7 @@ cargo test --ignored # Load tests # Stop when done python -m tooling.test.local-stack down + ```text ## References diff --git a/docs-cms/adr/adr-005-backend-plugin-architecture.md b/docs-cms/adr/adr-005-backend-plugin-architecture.md index daecf4ac1..5f0217d13 100644 --- a/docs-cms/adr/adr-005-backend-plugin-architecture.md +++ b/docs-cms/adr/adr-005-backend-plugin-architecture.md @@ -216,6 +216,7 @@ impl Router { ### Backend Interface Per Abstraction **KeyValue**: + ```rust #[async_trait] pub trait KeyValueBackend: Send + Sync { @@ -228,6 +229,7 @@ pub trait KeyValueBackend: Send + Sync { ``` **TimeSeries**: + ```rust #[async_trait] pub trait TimeSeriesBackend: Send + Sync { @@ -239,6 +241,7 @@ pub trait TimeSeriesBackend: Send + Sync { ``` **Graph**: + ```rust #[async_trait] pub trait GraphBackend: Send + Sync { diff --git a/docs-cms/adr/adr-006-namespace-multi-tenancy.md b/docs-cms/adr/adr-006-namespace-multi-tenancy.md index df897bfd4..3f41edbd7 100644 --- a/docs-cms/adr/adr-006-namespace-multi-tenancy.md +++ b/docs-cms/adr/adr-006-namespace-multi-tenancy.md @@ -44,6 +44,7 @@ Examples: - video-view-events (TimeSeries, analytics) - social-graph (Graph, relationships) - payment-transactions (KeyValue, financial data) + ```text **Properties**: @@ -55,6 +56,7 @@ Examples: ### Namespace Configuration ``` + namespace: user-profiles # What abstraction? @@ -90,6 +92,7 @@ backend_config: connection_string: postgres://prod-postgres-1/prism pool_size: 20 table_name: user_profiles + ```text ### Multi-Tenancy Strategies @@ -115,6 +118,7 @@ Netflix uses **sharded deployments** (single-tenant architecture): - **Capacity**: Add shards independently **Shard Assignment**: + ```rust // Deterministic shard selection fn select_shard(namespace: &str, shards: &[Shard]) -> &Shard { @@ -130,6 +134,7 @@ Namespace: user-profiles Backend: postgres ↓ Physical: prism_db.user_profiles table + ```text Namespace: video-events @@ -144,6 +149,7 @@ Namespace: social-graph Backend: neptune ↓ Physical: social-graph-prod instance + ```text ### Alternatives Considered @@ -196,6 +202,7 @@ Physical: social-graph-prod instance 1. **Creation**: ``` + # Via protobuf definition message UserProfile { option (prism.namespace) = "user-profiles"; @@ -209,6 +216,7 @@ Physical: social-graph-prod instance --abstraction keyvalue \ --backend postgres \ --capacity-estimate-rps 5000 + ```text 2. **Provisioning**: @@ -219,10 +227,12 @@ Physical: social-graph-prod instance 3. **Access Control**: ``` + // Check if service can access namespace if !authz.can_access(service_id, namespace, AccessLevel::ReadWrite) { return Err(Error::Forbidden); } + ```text 4. **Deletion**: @@ -235,6 +245,7 @@ Physical: social-graph-prod instance ### Namespace Metadata Store ``` + pub struct NamespaceMetadata { pub name: String, pub abstraction: AbstractionType, @@ -255,6 +266,7 @@ pub enum NamespaceStatus { Deleting, Deleted, } + ```text Stored in: @@ -265,6 +277,7 @@ Stored in: ### Namespace Discovery ``` + // Client discovers which shard serves a namespace pub struct DiscoveryClient { control_plane_url: String, @@ -284,6 +297,7 @@ impl DiscoveryClient { }) } } + ```text ### Co-Location Strategy @@ -291,19 +305,23 @@ impl DiscoveryClient { Small namespaces can share a shard: ``` + shard: prod-shard-1 namespaces: - user-profiles (5000 RPS) - user-preferences (500 RPS) # Co-located - user-settings (200 RPS) # Co-located + ```text Large namespaces get dedicated shards: ``` + shard: prod-shard-video-events namespaces: - video-events (200,000 RPS) # Dedicated shard + ```text ## References diff --git a/docs-cms/adr/adr-007-authentication-authorization.md b/docs-cms/adr/adr-007-authentication-authorization.md index 8a20ef9cf..39314edf4 100644 --- a/docs-cms/adr/adr-007-authentication-authorization.md +++ b/docs-cms/adr/adr-007-authentication-authorization.md @@ -39,6 +39,7 @@ Certificate contains: - Service name (CN: user-api.prod.company.com) - Environment (prod, staging, dev) - Expiry (auto-rotated) + ```text **User-Facing APIs** (if exposed): diff --git a/docs-cms/adr/adr-008-observability-strategy.md b/docs-cms/adr/adr-008-observability-strategy.md index e2bfbbb4c..86b5b2da6 100644 --- a/docs-cms/adr/adr-008-observability-strategy.md +++ b/docs-cms/adr/adr-008-observability-strategy.md @@ -51,6 +51,7 @@ Adopt **OpenTelemetry** from day one for metrics, logs, and traces. Use **Promet β”‚ β”œβ”€ Logs ────────► Loki β”‚ └─ Traces ──────► Jaeger/Tempo β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Metrics @@ -58,6 +59,7 @@ Adopt **OpenTelemetry** from day one for metrics, logs, and traces. Use **Promet **Key Metrics** (Prometheus format): ``` + use prometheus::{ register_histogram_vec, register_counter_vec, HistogramVec, CounterVec, @@ -99,6 +101,7 @@ timer.observe_duration(); REQUEST_COUNT .with_label_values(&[namespace, "get", "success"]) .inc(); + ```text **Dashboards**: @@ -111,6 +114,7 @@ REQUEST_COUNT **Format**: JSON for machine parsing ``` + use tracing::{info, error, instrument}; use tracing_subscriber::fmt::format::json; @@ -144,10 +148,12 @@ async fn handle_request(request: Request) -> Result { } } } + ```text **Log Output**: ``` + { "timestamp": "2025-10-05T12:34:56.789Z", "level": "INFO", @@ -162,6 +168,7 @@ async fn handle_request(request: Request) -> Result { "trace_id": "0af7651916cd43dd8448eb211c80319c" } } + ```text **Log Levels**: diff --git a/docs-cms/adr/adr-009-shadow-traffic-migrations.md b/docs-cms/adr/adr-009-shadow-traffic-migrations.md index 61bb74212..05834c8e6 100644 --- a/docs-cms/adr/adr-009-shadow-traffic-migrations.md +++ b/docs-cms/adr/adr-009-shadow-traffic-migrations.md @@ -46,6 +46,7 @@ Client Request β”œβ”€β”€β–Ί Primary Backend (old) ──► Response to client β”‚ └──► Shadow Backend (new) ──► Log comparison + ```text **Phases**: @@ -61,6 +62,7 @@ Client Request **Phase 1: Setup Shadow (Week 1)** ``` + namespace: user-profiles backends: @@ -72,10 +74,12 @@ backends: type: postgres-new connection: postgres://new-cluster/prism mode: shadow_write # Write only, don't read + ```text All writes go to both: ``` + async fn put(&self, request: PutRequest) -> Result { // Write to primary (blocking) let primary_result = self.primary_backend.put(&request).await?; @@ -96,12 +100,14 @@ async fn put(&self, request: PutRequest) -> Result { Ok(primary_result) } + ```text **Phase 2: Backfill (Week 2-3)** Copy existing data: ``` + # Scan all data from primary prism-cli backfill \ --namespace user-profiles \ @@ -109,9 +115,11 @@ prism-cli backfill \ --to postgres-new \ --parallelism 10 \ --throttle-rps 1000 + ```text ``` + async fn backfill( from: &dyn Backend, to: &dyn Backend, @@ -138,12 +146,14 @@ async fn backfill( Ok(BackfillStats { items_copied: total_copied }) } + ```text **Phase 3: Shadow Read (Week 4)** Read from both, compare: ``` + namespace: user-profiles backends: @@ -153,9 +163,11 @@ backends: shadow: type: postgres-new mode: shadow_read # Read and compare + ```text ``` + async fn get(&self, request: GetRequest) -> Result { // Read from primary (blocking) let primary_response = self.primary_backend.get(&request).await?; @@ -187,6 +199,7 @@ async fn get(&self, request: GetRequest) -> Result { Ok(primary_response) } + ```text **Monitor mismatch rate**: @@ -199,6 +212,7 @@ Target: < 0.1% (1 in 1000) **Phase 4: Promote (Week 5)** Flip primary when confident: + ```yaml namespace: user-profiles @@ -216,6 +230,7 @@ Monitor for issues. If problems, flip back instantly. **Phase 5: Decommission (Week 6+)** After confidence period (e.g., 2 weeks): + ```yaml namespace: user-profiles diff --git a/docs-cms/adr/adr-010-caching-layer.md b/docs-cms/adr/adr-010-caching-layer.md index be7169b2c..abc10688f 100644 --- a/docs-cms/adr/adr-010-caching-layer.md +++ b/docs-cms/adr/adr-010-caching-layer.md @@ -57,11 +57,13 @@ Write Path: β”‚ β”‚ β”‚ Invalidate └───────────▢│ + ```text ### Cache Configuration ``` + namespace: user-profiles cache: @@ -77,11 +79,13 @@ cache: connection: endpoints: [redis://cache-cluster-1:6379] pool_size: 50 + ```text ### Implementation ``` + #[async_trait] pub trait CacheBackend: Send + Sync { async fn get(&self, key: &str) -> Result>>; @@ -113,11 +117,13 @@ impl CacheBackend for RedisCache { Ok(()) } } + ```text ### Cache-Aware Backend Wrapper ``` + pub struct CachedBackend { backend: B, cache: Option>, @@ -196,6 +202,7 @@ impl KeyValueBackend for CachedBackend { Ok(()) } } + ```text ### Cache Key Design diff --git a/docs-cms/adr/adr-011-implementation-roadmap.md b/docs-cms/adr/adr-011-implementation-roadmap.md index 6f576af9a..41e6f3756 100644 --- a/docs-cms/adr/adr-011-implementation-roadmap.md +++ b/docs-cms/adr/adr-011-implementation-roadmap.md @@ -43,6 +43,7 @@ proto/ β”‚ └── metadata.proto # Item metadata β”œβ”€β”€ buf.yaml # Buf configuration └── buf.lock + ```text 2. **Implement Prism custom options**: @@ -52,6 +53,7 @@ proto/ 3. **Set up code generation**: ``` + # Install buf brew install bufbuild/buf/buf @@ -60,12 +62,15 @@ buf generate --template buf.gen.rust.yaml # Generate Python code buf generate --template buf.gen.python.yaml + ```text 4. **Create `tooling/codegen`** module: ``` + python -m tooling.codegen generate # β†’ Generates Rust, Python, TypeScript from proto + ```text ### Success Criteria @@ -91,12 +96,15 @@ Create minimal gRPC server in Rust that can accept requests and return dummy res 1. **Initialize Rust workspace**: ``` + cargo new --lib proxy cd proxy + ```text 2. **Add dependencies** (`Cargo.toml`): ``` + [dependencies] tokio = { version = "1.35", features = ["full"] } tonic = "0.10" @@ -104,10 +112,12 @@ prost = "0.12" tower = "0.4" tracing = "0.1" tracing-subscriber = "0.3" + ```text 3. **Implement health check service**: ``` + // proxy/src/health.rs pub struct HealthService; @@ -117,10 +127,12 @@ impl HealthCheck for HealthService { Ok(Response::new(HealthCheckResponse { status: "healthy" })) } } + ```text 4. **Create main server**: ``` + // proxy/src/main.rs #[tokio::main] async fn main() -> Result<()> { @@ -134,14 +146,17 @@ async fn main() -> Result<()> { Ok(()) } + ```text 5. **Add basic logging**: ``` + tracing_subscriber::fmt() .with_target(false) .compact() .init(); + ```text ### Success Criteria @@ -166,6 +181,7 @@ Define complete KeyValue protobuf API and generate server stubs. 1. **Create KeyValue proto**: ``` + // proto/prism/keyvalue/v1/keyvalue.proto service KeyValueService { rpc Put(PutRequest) returns (PutResponse); @@ -173,10 +189,12 @@ service KeyValueService { rpc Delete(DeleteRequest) returns (DeleteResponse); rpc Scan(ScanRequest) returns (stream ScanResponse); } + ```text 2. **Create KeyValue types**: ``` + // proto/prism/keyvalue/v1/types.proto message Item { bytes key = 1; @@ -190,15 +208,19 @@ message PutRequest { repeated Item items = 3; } // ... etc + ```text 3. **Regenerate Rust code**: ``` + buf generate + ```text 4. **Implement stub service** (returns errors): ``` + // proxy/src/keyvalue/service.rs pub struct KeyValueService; @@ -209,15 +231,18 @@ impl KeyValue for KeyValueService { } // ... etc } + ```text 5. **Wire into server**: ``` + Server::builder() .add_service(HealthServer::new(health_svc)) .add_service(KeyValueServer::new(kv_svc)) // ← New! .serve(addr) .await?; + ```text ### Success Criteria @@ -243,6 +268,7 @@ Implement working KeyValue backend using SQLite for local testing. 1. **Define backend trait**: ``` + // proxy/src/backend/mod.rs #[async_trait] pub trait KeyValueBackend: Send + Sync { @@ -251,10 +277,12 @@ pub trait KeyValueBackend: Send + Sync { async fn delete(&self, namespace: &str, id: &str, keys: Vec<&[u8]>) -> Result<()>; async fn scan(&self, namespace: &str, id: &str) -> Result>; } + ```text 2. **Implement SQLite backend**: ``` + // proxy/src/backend/sqlite.rs pub struct SqliteBackend { pool: SqlitePool, @@ -282,10 +310,12 @@ impl KeyValueBackend for SqliteBackend { } // ... etc } + ```text 3. **Create schema migration**: ``` + -- proxy/migrations/001_create_kv_table.sql CREATE TABLE IF NOT EXISTS kv ( namespace TEXT NOT NULL, @@ -298,10 +328,12 @@ CREATE TABLE IF NOT EXISTS kv ( ); CREATE INDEX idx_kv_namespace ON kv(namespace); + ```text 4. **Wire backend into service**: ``` + // proxy/src/keyvalue/service.rs pub struct KeyValueService { backend: Arc, @@ -316,10 +348,12 @@ impl KeyValue for KeyValueService { } // ... etc } + ```text 5. **Add configuration**: ``` + # proxy/config.yaml database: type: sqlite @@ -328,6 +362,7 @@ database: logging: level: debug format: json + ```text ### Success Criteria @@ -354,6 +389,7 @@ Validate end-to-end functionality with automated tests using real local backends 1. **Create integration test**: ``` + // proxy/tests/integration_test.rs #[tokio::test] async fn test_put_get_roundtrip() { @@ -386,10 +422,12 @@ async fn test_put_get_roundtrip() { assert_eq!(response.items[0].key, b"profile"); assert_eq!(response.items[0].value, b"Alice"); } + ```text 2. **Enhance `docker-compose.test.yml`**: ``` + services: prism-proxy: build: ./proxy @@ -402,10 +440,12 @@ services: postgres: # ... existing config ... + ```text 3. **Create test helper**: ``` + // proxy/tests/common/mod.rs pub struct TestFixture { pub client: KeyValueClient, @@ -423,10 +463,12 @@ impl TestFixture { Self { client } } } + ```text 4. **Add CI workflow**: ``` + # .github/workflows/test.yml name: Tests @@ -446,6 +488,7 @@ jobs: - name: Run integration tests run: cargo test --test integration_test + ```text ### Success Criteria @@ -471,6 +514,7 @@ Production-ready Postgres backend with complete documentation. 1. **Implement Postgres backend**: ``` + // proxy/src/backend/postgres.rs pub struct PostgresBackend { pool: PgPool, @@ -501,18 +545,22 @@ impl KeyValueBackend for PostgresBackend { } // ... etc (similar to SQLite but with Postgres-specific SQL) } + ```text 2. **Add connection pooling**: ``` + let pool = PgPoolOptions::new() .max_connections(20) .connect(&database_url) .await?; + ```text 3. **Create Postgres migrations**: ``` + -- proxy/migrations/postgres/001_create_kv_table.sql CREATE TABLE IF NOT EXISTS kv ( namespace VARCHAR(255) NOT NULL, @@ -526,10 +574,12 @@ CREATE TABLE IF NOT EXISTS kv ( CREATE INDEX idx_kv_namespace ON kv(namespace); CREATE INDEX idx_kv_id ON kv(namespace, id); + ```text 4. **Add integration tests for Postgres**: ``` + #[tokio::test] async fn test_postgres_backend() { let pool = PgPool::connect("postgres://prism:prism_test_password@localhost/prism_test") @@ -541,6 +591,7 @@ async fn test_postgres_backend() { // Run same tests as SQLite // ... test put, get, delete, scan } + ```text 5. **Write documentation**: diff --git a/docs-cms/adr/adr-012-go-for-tooling.md b/docs-cms/adr/adr-012-go-for-tooling.md index 2504048d3..9c25742ad 100644 --- a/docs-cms/adr/adr-012-go-for-tooling.md +++ b/docs-cms/adr/adr-012-go-for-tooling.md @@ -118,11 +118,13 @@ prism/ β”‚ β”‚ └── util/ β”‚ β”œβ”€β”€ go.mod β”‚ └── go.sum + ```text ### Key Libraries ``` + // Protobuf import "google.golang.org/protobuf/proto" @@ -137,6 +139,7 @@ import "log/slog" // Concurrency patterns import "golang.org/x/sync/errgroup" + ```text ### Protobuf Sharing @@ -144,12 +147,15 @@ import "golang.org/x/sync/errgroup" Generate Go code from the same proto definitions: ``` + # Generate Go protobuf code buf generate --template tools/buf.gen.go.yaml + ```text `tools/buf.gen.go.yaml`: ``` + version: v1 plugins: - plugin: go @@ -160,11 +166,13 @@ plugins: out: internal/proto opt: - paths=source_relative + ```text ### Example Tool: prism-cli ``` + package main import ( @@ -199,6 +207,7 @@ func main() { os.Exit(1) } } + ```text ## References diff --git a/docs-cms/adr/adr-015-go-testing-strategy.md b/docs-cms/adr/adr-015-go-testing-strategy.md index c81d6a1b0..c9eccc88c 100644 --- a/docs-cms/adr/adr-015-go-testing-strategy.md +++ b/docs-cms/adr/adr-015-go-testing-strategy.md @@ -49,6 +49,7 @@ Implement **three-tier testing strategy**: **Location**: `*_test.go` files alongside source **Pattern**: + ```go package config @@ -86,6 +87,7 @@ func TestValidateConfig_MissingNamespace(t *testing.T) { **Location**: `*_integration_test.go` **Pattern**: + ```go package migrate_test @@ -130,6 +132,7 @@ func TestMigrate_PostgresToSqlite(t *testing.T) { **Location**: `cmd/*/e2e_test.go` **Pattern**: + ```go package main_test @@ -251,11 +254,13 @@ tools/ └── testutil/ # Test harness β”œβ”€β”€ proxy.go └── fixtures.go + ```text ### Running Tests ``` + # Unit tests only (fast) go test ./... -short @@ -271,11 +276,13 @@ go test ./cmd/... -run E2E # Specific package go test ./internal/migrate -v + ```text ### CI Configuration ``` + # .github/workflows/go-test.yml jobs: test: @@ -299,6 +306,7 @@ jobs: run: | go tool cover -func=coverage.out | grep total | \ awk '{if ($3 < 80.0) {print "Coverage below 80%"; exit 1}}' + ```text ## References diff --git a/docs-cms/adr/adr-016-go-cli-configuration.md b/docs-cms/adr/adr-016-go-cli-configuration.md index 0e121fb3b..37f48dc67 100644 --- a/docs-cms/adr/adr-016-go-cli-configuration.md +++ b/docs-cms/adr/adr-016-go-cli-configuration.md @@ -177,11 +177,13 @@ tools/ β”‚ └── config/ β”‚ β”œβ”€β”€ config.go # Config types β”‚ └── loader.go # Viper integration + ```text ### Example Implementation ``` + // cmd/prism-cli/root.go package main @@ -238,6 +240,7 @@ func main() { os.Exit(1) } } + ```text ## Consequences diff --git a/docs-cms/adr/adr-017-go-structured-logging.md b/docs-cms/adr/adr-017-go-structured-logging.md index 84bea18f1..b514817e2 100644 --- a/docs-cms/adr/adr-017-go-structured-logging.md +++ b/docs-cms/adr/adr-017-go-structured-logging.md @@ -119,11 +119,13 @@ tools/internal/ log.go # slog wrapper with context helpers context.go # Context management log_test.go # Tests + ```text ### Core API ``` + package log import ( @@ -182,11 +184,13 @@ func With(ctx context.Context, args ...any) context.Context { } type loggerKey struct{} + ```text ### Usage Examples ``` + // Initialize at startup if err := log.Init(slog.LevelInfo, "json"); err != nil { panic(err) @@ -218,6 +222,7 @@ logger.Debug("worker started", "worker_id", workerID, "queue_size", queueSize, ) + ```text ### Performance-Critical Paths @@ -225,17 +230,20 @@ logger.Debug("worker started", For hot paths, use conditional logging: ``` + if logger.Enabled(ctx, slog.LevelDebug) { logger.DebugContext(ctx, "processing item", "item_id", id, "batch", batchNum, ) } + ```text ### Testing Pattern ``` + // Custom handler for testing type TestHandler struct { logs []slog.Record @@ -263,6 +271,7 @@ func TestMigrate_Logging(t *testing.T) { t.Error("expected at least 1 log entry") } } + ```text ## Logging Guidelines diff --git a/docs-cms/adr/adr-018-rust-error-handling-strategy.md b/docs-cms/adr/adr-018-rust-error-handling-strategy.md index cbc0bc0f0..7ddc2503e 100644 --- a/docs-cms/adr/adr-018-rust-error-handling-strategy.md +++ b/docs-cms/adr/adr-018-rust-error-handling-strategy.md @@ -38,6 +38,7 @@ Adopt modern Rust error handling with `thiserror` and `anyhow`: ### Why thiserror + anyhow **thiserror** for library/domain errors: + ```rust use thiserror::Error; @@ -58,6 +59,7 @@ pub enum BackendError { ``` **anyhow** for application/handler errors: + ```rust use anyhow::{Context, Result}; @@ -165,11 +167,13 @@ proxy/src/ β”‚ β”œβ”€β”€ mod.rs β”‚ └── error.rs # KeyValue-specific errors └── main.rs + ```text ### Top-Level Error Types ``` + // proxy/src/error.rs use thiserror::Error; @@ -201,11 +205,13 @@ impl From for tonic::Status { } } } + ```text ### Handler Pattern ``` + use anyhow::{Context, Result}; use tonic::{Request, Response, Status}; @@ -243,11 +249,13 @@ impl KeyValueService for KeyValueHandler { } } } + ```text ### Backend Error Definition ``` + // proxy/src/backend/error.rs use thiserror::Error; @@ -268,11 +276,13 @@ pub enum BackendError { #[error("io error: {0}")] Io(#[from] std::io::Error), } + ```text ### Testing Error Conditions ``` + #[cfg(test)] mod tests { use super::*; @@ -302,6 +312,7 @@ mod tests { assert_eq!(status.code(), tonic::Code::NotFound); } } + ```text ### Error Logging @@ -309,6 +320,7 @@ mod tests { Integrate with structured logging: ``` + use tracing::error; match do_operation().await { @@ -323,6 +335,7 @@ match do_operation().await { return Err(e); } } + ```text ## References diff --git a/docs-cms/adr/adr-020-rust-testing-strategy.md b/docs-cms/adr/adr-020-rust-testing-strategy.md index 6b833266c..6828ad9dd 100644 --- a/docs-cms/adr/adr-020-rust-testing-strategy.md +++ b/docs-cms/adr/adr-020-rust-testing-strategy.md @@ -49,6 +49,7 @@ Implement **three-tier testing strategy** with Rust best practices: **Location**: `#[cfg(test)] mod tests` in same file or `tests/` subdirectory **Pattern**: + ```rust // src/backend/postgres.rs #[cfg(test)] @@ -97,6 +98,7 @@ mod tests { **Location**: `tests/` directory (separate from source) **Pattern**: + ```rust // tests/integration_test.rs use prism_proxy::{Backend, KeyValueBackend, SqliteBackend}; @@ -160,6 +162,7 @@ async fn test_postgres_backend() { **Location**: `tests/e2e/` **Pattern**: + ```rust // tests/e2e/keyvalue_test.rs use prism_proto::keyvalue::v1::{ @@ -364,11 +367,13 @@ proxy/ β”‚ └── keyvalue_test.rs └── benches/ └── keyvalue_bench.rs + ```text ### Running Tests ``` + # Unit tests only (fast) cargo test --lib @@ -389,11 +394,13 @@ cargo bench # Property tests (more iterations) PROPTEST_CASES=10000 cargo test + ```text ### CI Configuration ``` + # .github/workflows/rust-test.yml jobs: test: @@ -418,16 +425,19 @@ jobs: - name: Benchmarks (ensure no regression) run: cargo bench --no-fail-fast + ```text ### Dependencies ``` + [dev-dependencies] tokio-test = "0.4" proptest = "1.4" criterion = { version = "0.5", features = ["async_tokio"] } testcontainers = "0.15" + ```text ## References diff --git a/docs-cms/adr/adr-021-rust-structured-logging.md b/docs-cms/adr/adr-021-rust-structured-logging.md index 39e45091d..6ec5994b9 100644 --- a/docs-cms/adr/adr-021-rust-structured-logging.md +++ b/docs-cms/adr/adr-021-rust-structured-logging.md @@ -81,11 +81,13 @@ Application Code β”œβ”€ Layer: fmt (console) β”€β”˜ β”œβ”€ Layer: json (file) └─ Layer: opentelemetry (Jaeger/Tempo) + ```text ### Subscriber Configuration ``` + use tracing_subscriber::{fmt, EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}; fn init_tracing() -> Result<()> { @@ -106,11 +108,13 @@ fn init_tracing() -> Result<()> { Ok(()) } + ```text ### Structured Events ``` + use tracing::{info, warn, error, debug}; // Structured fields @@ -135,11 +139,13 @@ debug!( items = ?items, // Debug representation "processing items" ); + ```text ### Span Instrumentation ``` + use tracing::{info_span, instrument, Instrument}; // Automatic instrumentation with #[instrument] @@ -168,11 +174,13 @@ async fn manual_span_example() { // Span fields can be set dynamically let span = info_span!("request", user_id = tracing::field::Empty); span.record("user_id", &user_id); + ```text ### OpenTelemetry Integration ``` + use opentelemetry::global; use tracing_opentelemetry::OpenTelemetryLayer; use opentelemetry_jaeger::JaegerPipeline; @@ -197,11 +205,13 @@ async fn init_tracing_with_otel() -> Result<()> { Ok(()) } + ```text ### Log Levels and Filtering ``` + // Set via environment variable // RUST_LOG=prism_proxy=debug,sqlx=warn @@ -209,11 +219,13 @@ async fn init_tracing_with_otel() -> Result<()> { let filter = EnvFilter::new("prism_proxy=debug") .add_directive("sqlx=warn".parse()?) .add_directive("tonic=info".parse()?); + ```text ### Performance: Conditional Logging ``` + use tracing::Level; // Only evaluate expensive computation if debug enabled @@ -224,6 +236,7 @@ if tracing::level_enabled!(Level::DEBUG) { // Or use span guards for hot paths let _span = info_span!("hot_path").entered(); // Span only recorded if info level enabled + ```text ### Alternatives Considered @@ -270,12 +283,14 @@ let _span = info_span!("hot_path").entered(); ### Dependencies ``` + [dependencies] tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } tracing-opentelemetry = "0.22" opentelemetry = { version = "0.21", features = ["trace"] } opentelemetry-jaeger = { version = "0.20", features = ["rt-tokio"] } + ```text ### Standard Fields @@ -286,6 +301,7 @@ Always include: - `environment`: "production", "staging", "development" ``` + fn init_tracing() -> Result<()> { tracing_subscriber::registry() .with(EnvFilter::from_default_env()) @@ -299,6 +315,7 @@ fn init_tracing() -> Result<()> { Ok(()) } + ```text ### Logging Guidelines @@ -319,6 +336,7 @@ fn init_tracing() -> Result<()> { ### Testing with Tracing ``` + #[cfg(test)] mod tests { use tracing_subscriber::layer::SubscriberExt; @@ -335,6 +353,7 @@ mod tests { }); } } + ```text ## References diff --git a/docs-cms/adr/adr-022-dynamic-client-configuration.md b/docs-cms/adr/adr-022-dynamic-client-configuration.md index 8559df551..8ef9f07b3 100644 --- a/docs-cms/adr/adr-022-dynamic-client-configuration.md +++ b/docs-cms/adr/adr-022-dynamic-client-configuration.md @@ -68,11 +68,13 @@ Implement **Dynamic Client Configuration** with protobuf descriptors: β”‚ Uses named config β”‚ β”‚ Provides custom β”‚ β”‚ "user-profiles" β”‚ β”‚ inline config β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Client Configuration Descriptor (Protobuf) ``` + // proto/prism/config/v1/client_config.proto syntax = "proto3"; @@ -210,11 +212,13 @@ message RateLimitConfig { int32 requests_per_second = 1; int32 burst = 2; } + ```text ### Configuration Service (gRPC) ``` + // proto/prism/config/v1/config_service.proto syntax = "proto3"; @@ -277,6 +281,7 @@ message ValidateConfigResponse { repeated string errors = 2; repeated string warnings = 3; } + ```text ### Client Connection Flow @@ -426,6 +431,7 @@ Client configuration messages use protobuf custom options for schema evolution a - Field-level metadata drives validation and storage optimization **Example: Recording Configuration Request** + ```rust // Proxy automatically records configuration requests let entry = ProtocolEntry { @@ -447,6 +453,7 @@ recorder.record(entry).await?; ``` **Example: Schema Registry Integration** + ```bash # Schemas automatically registered during build prism-admin schema register \ @@ -522,6 +529,7 @@ config/ β”œβ”€β”€ key-value.yaml β”œβ”€β”€ queue.yaml └── pubsub.yaml + ```text ### Configuration Validation @@ -529,6 +537,7 @@ config/ Server validates all configurations: ``` + impl ConfigValidator { fn validate(&self, config: &ClientConfig) -> Result<(), Vec> { let mut errors = Vec::new(); @@ -552,6 +561,7 @@ impl ConfigValidator { if errors.is_empty() { Ok(()) } else { Err(errors) } } } + ```text ### Configuration Caching @@ -559,6 +569,7 @@ impl ConfigValidator { Client caches configurations locally: ``` + type ConfigCache struct { cache map[string]*ClientConfig ttl time.Duration @@ -578,6 +589,7 @@ func (c *ConfigCache) Get(name string) (*ClientConfig, error) { c.cache[name] = config return config, nil } + ```text ## References diff --git a/docs-cms/adr/adr-023-grpc-first-interface-design.md b/docs-cms/adr/adr-023-grpc-first-interface-design.md index a59653d55..694c28c03 100644 --- a/docs-cms/adr/adr-023-grpc-first-interface-design.md +++ b/docs-cms/adr/adr-023-grpc-first-interface-design.md @@ -94,6 +94,7 @@ Use **gRPC as the primary interface** for Prism data access layer: β”‚ Go Client β”‚ β”‚ Rust Client β”‚ β”‚ (generated code) β”‚ β”‚ (generated code) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Service Organization @@ -101,6 +102,7 @@ Use **gRPC as the primary interface** for Prism data access layer: Each access pattern gets its own service: ``` + // proto/prism/session/v1/session_service.proto service SessionService { rpc CreateSession(CreateSessionRequest) returns (CreateSessionResponse); @@ -134,12 +136,14 @@ service TransactService { rpc Write(WriteRequest) returns (WriteResponse); rpc Transaction(stream TransactRequest) returns (stream TransactResponse); } + ```text ### Streaming Patterns **Server streaming** (pagination, pub/sub): ``` + service ReaderService { // Server streams pages to client rpc Read(ReadRequest) returns (stream Page) { @@ -149,9 +153,11 @@ service ReaderService { }; } } + ```text ``` + // Server implementation async fn read(&self, req: Request) -> Result, Status> { let (tx, rx) = mpsc::channel(100); @@ -170,22 +176,27 @@ async fn read(&self, req: Request) -> Result metadata = 3; } + ```text ### Metadata and Context @@ -228,6 +243,7 @@ message ErrorInfo { Use gRPC metadata for cross-cutting concerns: ``` + // Server: extract session token from metadata let session_token = req.metadata() .get("x-session-token") @@ -240,6 +256,7 @@ request.metadata_mut().insert( "x-session-token", session_token.parse().unwrap(), ); + ```text Common metadata: @@ -252,29 +269,35 @@ Common metadata: **Connection pooling:** ``` + // Reuse connections let channel = Channel::from_static("http://localhost:8980") .connect_lazy(); let client = QueueServiceClient::new(channel.clone()); + ```text **Compression:** ``` + // Enable gzip compression let channel = Channel::from_static("http://localhost:8980") .http2_keep_alive_interval(Duration::from_secs(30)) .http2_adaptive_window(true) .connect_lazy(); + ```text **Timeouts:** ``` + service QueueService { rpc Publish(PublishRequest) returns (PublishResponse) { option (google.api.method_signature) = "timeout=5s"; } } + ```text ### Alternatives Considered @@ -328,6 +351,7 @@ service QueueService { ### Server Implementation (Rust) ``` + // proxy/src/main.rs use tonic::transport::Server; @@ -348,11 +372,13 @@ async fn main() -> Result<()> { Ok(()) } + ```text ### Client Implementation (Go) ``` + // Client connection conn, err := grpc.Dial( "localhost:8980", @@ -372,11 +398,13 @@ resp, err := client.Publish(ctx, &queue.PublishRequest{ Topic: "events", Payload: data, }) + ```text ### Testing with grpcurl ``` + # List services grpcurl -plaintext localhost:8980 list @@ -386,11 +414,13 @@ grpcurl -plaintext localhost:8980 describe prism.queue.v1.QueueService # Make request grpcurl -plaintext -d '{"topic":"events","payload":"dGVzdA=="}' \ localhost:8980 prism.queue.v1.QueueService/Publish + ```text ### Code Generation ``` + # Generate Rust code buf generate --template proxy/buf.gen.rust.yaml @@ -399,6 +429,7 @@ buf generate --template tools/buf.gen.go.yaml # Generate Python code buf generate --template clients/python/buf.gen.python.yaml + ```text ## References diff --git a/docs-cms/adr/adr-025-container-plugin-model.md b/docs-cms/adr/adr-025-container-plugin-model.md index 5c994d095..4c72ab76a 100644 --- a/docs-cms/adr/adr-025-container-plugin-model.md +++ b/docs-cms/adr/adr-025-container-plugin-model.md @@ -81,6 +81,7 @@ Implement **container plugin model** with standardized contracts: β”‚ β”‚ β”‚ β”‚ Listener β”‚ β”‚ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Plugin Contract @@ -88,6 +89,7 @@ Implement **container plugin model** with standardized contracts: All container plugins implement standard interface: ``` + // proto/prism/plugin/v1/plugin.proto syntax = "proto3"; @@ -146,6 +148,7 @@ message InfoResponse { string backend = 4; // "kafka", "nats", "postgres", etc. map capabilities = 5; } + ```text ### Environment Configuration @@ -153,6 +156,7 @@ message InfoResponse { All plugins configured via environment variables: ``` + # Common to all plugins PRISM_PROXY_ENDPOINT=localhost:8980 PRISM_PLUGIN_ROLE=publisher @@ -184,6 +188,7 @@ DATABASE_TABLE=events MAILBOX_TABLE=mailbox MAILBOX_POLL_INTERVAL=1s MAILBOX_BATCH_SIZE=100 + ```text ### Kafka Plugin Containers @@ -191,6 +196,7 @@ MAILBOX_BATCH_SIZE=100 #### Kafka Publisher ``` + // containers/kafka-publisher/src/main.rs use rdkafka::producer::{FutureProducer, FutureRecord}; @@ -235,11 +241,13 @@ async fn main() -> Result<()> { let publisher = KafkaPublisher::new()?; publisher.run().await } + ```text #### Kafka Consumer ``` + // containers/kafka-consumer/src/main.rs use rdkafka::consumer::{Consumer, StreamConsumer}; @@ -276,6 +284,7 @@ impl KafkaConsumer { } } } + ```text ### NATS Plugin Containers @@ -283,6 +292,7 @@ impl KafkaConsumer { #### NATS Publisher ``` + // containers/nats-publisher/src/main.rs use async_nats::Client; @@ -308,11 +318,13 @@ impl NatsPublisher { Ok(()) } } + ```text #### NATS Consumer ``` + // containers/nats-consumer/src/main.rs use async_nats::{Client, jetstream}; @@ -355,11 +367,13 @@ impl NatsConsumer { Ok(()) } } + ```text ### Paged Reader Plugin ``` + // containers/indexed-reader/src/main.rs use sqlx::PgPool; @@ -397,6 +411,7 @@ impl IndexedReader { } } } + ```text ### Transact Writer Plugins @@ -404,6 +419,7 @@ impl IndexedReader { #### Transaction Processor ``` + // containers/transact-processor/src/main.rs use sqlx::{PgPool, Transaction}; @@ -460,11 +476,13 @@ impl TransactProcessor { }) } } + ```text #### Mailbox Listener ``` + // containers/mailbox-listener/src/main.rs use sqlx::PgPool; @@ -514,6 +532,7 @@ impl MailboxListener { Ok(()) } } + ```text ### Docker Deployment @@ -521,6 +540,7 @@ impl MailboxListener { Each plugin is a separate Docker image: ``` + # Dockerfile.kafka-publisher FROM rust:1.75 as builder WORKDIR /app @@ -531,11 +551,13 @@ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/kafka-publisher /usr/local/bin/ ENTRYPOINT ["kafka-publisher"] + ```text ### Docker Compose Example ``` + # docker-compose.plugins.yml version: '3.8' @@ -595,11 +617,13 @@ services: - DATABASE_URL=postgres://prism:password@postgres/prism - MAILBOX_ID=system - MAILBOX_POLL_INTERVAL=1s + ```text ### Kubernetes Deployment ``` + # k8s/kafka-consumer-deployment.yaml apiVersion: apps/v1 kind: Deployment @@ -647,6 +671,7 @@ spec: limits: memory: "512Mi" cpu: "500m" + ```text ### Alternatives Considered diff --git a/docs-cms/adr/adr-026-distroless-container-images.md b/docs-cms/adr/adr-026-distroless-container-images.md index 951d697af..de12e7807 100644 --- a/docs-cms/adr/adr-026-distroless-container-images.md +++ b/docs-cms/adr/adr-026-distroless-container-images.md @@ -161,6 +161,7 @@ ENTRYPOINT ["/usr/local/bin/prism-proxy"] ``` **Access debug shell:** + ```bash # Override entrypoint to get shell docker run -it --entrypoint /busybox/sh prism/proxy:debug @@ -221,6 +222,7 @@ ENTRYPOINT ["/usr/local/bin/prism-proxy"] ``` **Build both variants:** + ```bash # Production docker build --target production -t prism/proxy:latest . @@ -257,6 +259,7 @@ ENTRYPOINT ["/usr/local/bin/kafka-publisher"] - Never run as root **Read-only filesystem:** + ```yaml # Kubernetes pod spec securityContext: @@ -275,6 +278,7 @@ securityContext: ### CI/CD Integration **Build pipeline:** + ```yaml # .github/workflows/docker-build.yml - name: Build production image @@ -354,6 +358,7 @@ prism/proxy:latest # Production prism/proxy:v1.2.3 # Specific version (production) prism/proxy:debug # Debug variant (latest) prism/proxy:v1.2.3-debug # Debug variant (specific version) + ```text ### File Structure diff --git a/docs-cms/adr/adr-028-admin-ui-fastapi-grpc-web.md b/docs-cms/adr/adr-028-admin-ui-fastapi-grpc-web.md index 150df49c8..608ccd32b 100644 --- a/docs-cms/adr/adr-028-admin-ui-fastapi-grpc-web.md +++ b/docs-cms/adr/adr-028-admin-ui-fastapi-grpc-web.md @@ -75,6 +75,7 @@ Build **Admin UI with FastAPI + gRPC-Web**: β”‚ Prism Admin API (:8981) β”‚ β”‚ - prism.admin.v1.AdminService β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Why FastAPI @@ -111,15 +112,18 @@ Build **Admin UI with FastAPI + gRPC-Web**: **Generated gRPC-Web client:** ``` + # Generate JavaScript client from proto protoc --js_out=import_style=commonjs,binary:./admin-ui/static/js \ --grpc-web_out=import_style=commonjs,mode=grpcwebtext:./admin-ui/static/js \ proto/prism/admin/v1/admin.proto + ```text ### FastAPI Implementation ``` + # admin-ui/main.py from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -150,6 +154,7 @@ async def grpc_proxy(method: str, request: bytes): @app.get("/health") async def health(): return {"status": "healthy"} + ```text ### Frontend Structure @@ -292,6 +297,7 @@ services: ### Security **Authentication:** + ```python from fastapi import Header, HTTPException @@ -312,6 +318,7 @@ async def grpc_proxy( ``` **CORS** (if needed): + ```python from fastapi.middleware.cors import CORSMiddleware diff --git a/docs-cms/adr/adr-029-protocol-recording-protobuf-tags.md b/docs-cms/adr/adr-029-protocol-recording-protobuf-tags.md index e22528330..f9e93e607 100644 --- a/docs-cms/adr/adr-029-protocol-recording-protobuf-tags.md +++ b/docs-cms/adr/adr-029-protocol-recording-protobuf-tags.md @@ -97,6 +97,7 @@ enum RecordingLevel { ### Tagged Message Examples **Queue Protocol:** + ```protobuf // proto/prism/queue/v1/queue.proto import "prism/options.proto"; @@ -145,6 +146,7 @@ message Message { ``` **Transaction Protocol:** + ```protobuf // proto/prism/transact/v1/transact.proto message WriteRequest { @@ -178,6 +180,7 @@ message TransactionStarted { ### Recording Infrastructure **Protocol Recorder Interface:** + ```rust // proxy/src/protocol/recorder.rs use prost::Message; @@ -212,6 +215,7 @@ pub struct ProtocolFilter { ``` **Interceptor for Recording:** + ```rust // proxy/src/protocol/interceptor.rs pub struct RecordingInterceptor { @@ -264,6 +268,7 @@ impl Interceptor for RecordingInterceptor { ``` **Sampling Logic:** + ```rust pub struct Sampler { rng: ThreadRng, @@ -295,6 +300,7 @@ fn should_record(options: &ProtocolOptions, sampler: &Sampler) -> bool { ### Storage Backends **File Storage:** + ```rust pub struct FileProtocolRecorder { path: PathBuf, @@ -314,6 +320,7 @@ impl ProtocolRecorder for FileProtocolRecorder { ``` **PostgreSQL Storage:** + ```rust pub struct PostgresProtocolRecorder { pool: PgPool, @@ -375,6 +382,7 @@ impl ProtocolRecorder for PostgresProtocolRecorder { ### Query Interface **CLI Tool:** + ```bash # Query protocol recordings prism-admin protocol query \ @@ -392,6 +400,7 @@ prism-admin protocol replay \ ``` **gRPC Admin API:** + ```protobuf service AdminService { // Query protocol recordings @@ -415,6 +424,7 @@ message QueryProtocolRequest { ### Privacy Considerations **PII in Protocol Messages:** + ```protobuf message UserProfile { option (prism.protocol) = { @@ -431,6 +441,7 @@ message UserProfile { ``` **Automatic PII Scrubbing:** + ```rust fn scrub_pii(entry: &mut ProtocolEntry) { if entry.tags.contains(&"pii".to_string()) { @@ -519,6 +530,7 @@ protocol_recording: ### Code Generation Extract protocol options in build: + ```rust // build.rs fn main() { diff --git a/docs-cms/adr/adr-030-schema-recording-protobuf-tags.md b/docs-cms/adr/adr-030-schema-recording-protobuf-tags.md index 4d6c445ca..40933bb2c 100644 --- a/docs-cms/adr/adr-030-schema-recording-protobuf-tags.md +++ b/docs-cms/adr/adr-030-schema-recording-protobuf-tags.md @@ -158,6 +158,7 @@ enum PIIType { ### Tagged Schema Examples **Entity Schema:** + ```protobuf // proto/prism/data/v1/user.proto import "prism/options.proto"; @@ -220,6 +221,7 @@ message UserProfile { ``` **Event Schema:** + ```protobuf // proto/prism/events/v1/user_events.proto message UserCreatedEvent { @@ -258,6 +260,7 @@ message UserCreatedEvent { ``` **Deprecated Schema:** + ```protobuf message UserProfileV1 { option (prism.schema) = { @@ -282,6 +285,7 @@ message UserProfileV1 { ### Schema Registry **Schema Registry Service:** + ```protobuf // proto/prism/schema/v1/registry.proto syntax = "proto3"; @@ -410,6 +414,7 @@ message SearchSchemasResponse { ### Schema Registry Implementation **Registry Storage:** + ```rust // proxy/src/schema/registry.rs use prost::Message; @@ -493,6 +498,7 @@ impl SchemaRegistry for PostgresSchemaRegistry { ``` **Compatibility Checker:** + ```rust // proxy/src/schema/compatibility.rs use prost_types::FileDescriptorSet; @@ -590,6 +596,7 @@ impl CompatibilityChecker { ### Build-time Schema Registration **Automatic registration during build:** + ```rust // build.rs fn main() { @@ -714,6 +721,7 @@ prism-admin schema migrate \ ### Migration Generation **Automatic migration script generation:** + ```rust // proxy/src/schema/migration.rs pub struct MigrationGenerator; @@ -842,6 +850,7 @@ impl MigrationGenerator { ### Code Generation Extract schema options during build: + ```rust // build.rs fn extract_schema_metadata(descriptor_set: &FileDescriptorSet) -> SchemaMetadata { @@ -853,6 +862,7 @@ fn extract_schema_metadata(descriptor_set: &FileDescriptorSet) -> SchemaMetadata ### Integration with Admin API Schema registry accessible via Admin API (ADR-027): + ```protobuf service AdminService { // Existing admin operations... diff --git a/docs-cms/adr/adr-031-ttl-defaults-client-data.md b/docs-cms/adr/adr-031-ttl-defaults-client-data.md index 510323930..5e43f6459 100644 --- a/docs-cms/adr/adr-031-ttl-defaults-client-data.md +++ b/docs-cms/adr/adr-031-ttl-defaults-client-data.md @@ -65,11 +65,13 @@ Namespace-Level Default TTL Pattern-Specific Default TTL ↓ (fallback) System-Wide Default TTL (30 days) + ```text ### Implementation in Protobuf ``` + // Client-specified TTL in requests message SetRequest { string namespace = 1; @@ -99,11 +101,13 @@ message NamespaceConfig { // Warn when data approaches expiration bool enable_ttl_warnings = 6 [default = true]; } + ```text ### Configuration YAML Example ``` + namespaces: - name: user-sessions backend: redis @@ -121,6 +125,7 @@ namespaces: backend: postgres pattern: keyvalue allow_infinite_ttl: true # Explicit opt-in for infinite TTL + ```text ## Backend-Specific Implementation @@ -128,6 +133,7 @@ namespaces: ### Redis (Cache, Session, Vector) ``` + // Rust implementation in Prism proxy async fn set_with_ttl( &self, @@ -144,11 +150,13 @@ async fn set_with_ttl( .set_ex(key, value, effective_ttl.as_secs() as usize) .await } + ```text ### PostgreSQL (KeyValue) ``` + -- Table schema with TTL support CREATE TABLE keyvalue ( namespace VARCHAR(255) NOT NULL, @@ -167,11 +175,13 @@ WHERE expires_at IS NOT NULL; DELETE FROM keyvalue WHERE expires_at < NOW() AND expires_at IS NOT NULL; + ```text ### ClickHouse (TimeSeries) ``` + -- ClickHouse table with TTL CREATE TABLE events ( timestamp DateTime64(9), @@ -183,11 +193,13 @@ ENGINE = MergeTree() PARTITION BY toYYYYMMDD(timestamp) ORDER BY (namespace, event_type, timestamp) TTL timestamp + INTERVAL 90 DAY; -- Default 90 days + ```text ### MinIO (Object Storage) ``` + # MinIO lifecycle policy lifecycle_config = { "Rules": [ @@ -204,6 +216,7 @@ lifecycle_config = { ] } minio_client.set_bucket_lifecycle("prism-objects", lifecycle_config) + ```text ## Monitoring and Observability @@ -211,6 +224,7 @@ minio_client.set_bucket_lifecycle("prism-objects", lifecycle_config) ### Metrics ``` + message TTLMetrics { string namespace = 1; @@ -228,11 +242,13 @@ message TTLMetrics { int64 total_bytes = 8; int64 bytes_to_expire_soon = 9; } + ```text ### Alerting Rules ``` + # Prometheus alert rules groups: - name: ttl_alerts @@ -251,11 +267,13 @@ groups: for: 6h annotations: summary: "Storage growing without expiration" + ```text ### Admin CLI Commands ``` + # Show TTL distribution for namespace prism namespace describe my-app --show-ttl-stats @@ -267,6 +285,7 @@ prism data set-ttl my-app key123 --ttl 1h # Disable expiration for specific item (requires permission) prism data set-ttl my-app key456 --infinite + ```text ## Client SDK Examples @@ -274,6 +293,7 @@ prism data set-ttl my-app key456 --infinite ### Python SDK ``` + from prism_sdk import PrismClient client = PrismClient(namespace="user-sessions") @@ -286,11 +306,13 @@ client.set("session:def456", session_data) # Infinite TTL (requires namespace config: allow_infinite_ttl=true) client.set("permanent:record", data, no_expiration=True) + ```text ### Go SDK ``` + client := prism.NewClient("user-sessions") // Explicit TTL @@ -303,6 +325,7 @@ client.Set(ctx, "session:def456", sessionData) // Infinite TTL (opt-in required) client.Set(ctx, "permanent:record", data, prism.WithNoExpiration()) + ```text ## Migration Path diff --git a/docs-cms/adr/adr-032-object-storage-pattern.md b/docs-cms/adr/adr-032-object-storage-pattern.md index f15a66595..1cade9ab5 100644 --- a/docs-cms/adr/adr-032-object-storage-pattern.md +++ b/docs-cms/adr/adr-032-object-storage-pattern.md @@ -748,6 +748,7 @@ What's the lifecycle? β”œβ”€ Temporary (< 7 days) β†’ Object Storage with short TTL β”œβ”€ Medium-term (7-90 days) β†’ Object Storage with default TTL └─ Permanent β†’ Object Storage with infinite TTL (explicit opt-in) + ```text --- diff --git a/docs-cms/adr/adr-033-capability-api.md b/docs-cms/adr/adr-033-capability-api.md index 4bc659189..27fda00a1 100644 --- a/docs-cms/adr/adr-033-capability-api.md +++ b/docs-cms/adr/adr-033-capability-api.md @@ -114,6 +114,7 @@ message GetBackendCapabilitiesResponse { ### Usage Patterns **Client Discovery on Startup**: + ```rust // Client queries capabilities on initialization let caps = client.get_capabilities().await?; @@ -128,6 +129,7 @@ if !caps.features.get("shadow_traffic").unwrap_or(&false) { ``` **Admin CLI Feature Detection**: + ```bash # CLI checks capabilities before presenting commands prism backend list # Only shows available backends @@ -138,6 +140,7 @@ prism shadow enable my-ns ``` **Version Compatibility Check**: + ```python # Python client checks API compatibility caps = await client.get_capabilities() diff --git a/docs-cms/adr/adr-034-sharding-strategy.md b/docs-cms/adr/adr-034-sharding-strategy.md index 3aae30fc9..90bd20f7e 100644 --- a/docs-cms/adr/adr-034-sharding-strategy.md +++ b/docs-cms/adr/adr-034-sharding-strategy.md @@ -77,6 +77,7 @@ Dedicated backend clusters per shard: β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ## Rationale @@ -167,6 +168,7 @@ Dedicated backend clusters per shard: **Shared Namespace Proxy (Small Scale)**: ``` + # Single Prism instance, multiple namespaces services: prism-shared: @@ -176,10 +178,12 @@ services: - user-profiles - session-cache - recommendations + ```text **Product-Sharded Deployment (Medium Scale)**: ``` + # Separate Prism instances per product services: prism-playback: @@ -195,10 +199,12 @@ services: namespaces: - search-index - search-cache + ```text **Feature + SLA Sharded (Large Scale)**: ``` + # Sharded by product, feature, and SLA tier services: prism-playback-live: # Low latency tier @@ -214,12 +220,14 @@ services: sla: p99_50ms backends: - redis-vod-cluster + ```text ### Routing to Shards **Service Mesh Approach** (Recommended): ``` + # Istio VirtualService routes clients to correct shard apiVersion: networking.istio.io/v1beta1 kind: VirtualService @@ -243,10 +251,12 @@ spec: route: - destination: host: prism-search + ```text **Client-Side Routing**: ``` + // Client selects shard based on product let prism_endpoint = match product { Product::Playback => "prism-playback.example.com:50051", @@ -255,6 +265,7 @@ let prism_endpoint = match product { }; let client = PrismClient::connect(prism_endpoint).await?; + ```text ### Configuration Management @@ -262,6 +273,7 @@ let client = PrismClient::connect(prism_endpoint).await?; Use Kubernetes ConfigMaps or CRDs to define shards: ``` + apiVersion: prism.io/v1alpha1 kind: PrismShard metadata: @@ -282,6 +294,7 @@ spec: requests: cpu: "4" memory: "8Gi" + ```text (See ADR-037 for Kubernetes Operator details) diff --git a/docs-cms/adr/adr-035-connection-pooling.md b/docs-cms/adr/adr-035-connection-pooling.md index 01c88828c..ffc1ea073 100644 --- a/docs-cms/adr/adr-035-connection-pooling.md +++ b/docs-cms/adr/adr-035-connection-pooling.md @@ -130,6 +130,7 @@ backends: - NATS client libraries handle this internally **Configuration**: + ```rust // Single NATS connection per namespace let nats_client = nats::connect(&config.connection_string).await?; @@ -144,6 +145,7 @@ let nats_client = nats::connect(&config.connection_string).await?; - Client library's default pooling is usually optimal **Configuration**: + ```rust // HTTP client with built-in connection pool let s3_client = aws_sdk_s3::Client::from_conf( @@ -297,6 +299,7 @@ pub fn record_pool_metrics(pool: &Pool, namespace: &str, backend: &str) { **Start with conservative sizes**: pool_size = max(min_size, expected_p99_rps * p99_query_latency_seconds) + ```text **Example**: diff --git a/docs-cms/adr/adr-036-sqlite-config-storage.md b/docs-cms/adr/adr-036-sqlite-config-storage.md index f5919e61b..5c66dcd94 100644 --- a/docs-cms/adr/adr-036-sqlite-config-storage.md +++ b/docs-cms/adr/adr-036-sqlite-config-storage.md @@ -125,11 +125,13 @@ CREATE TABLE namespace_features ( β”œβ”€β”€ config.db # Primary SQLite database β”œβ”€β”€ config.db-wal # Write-Ahead Log (SQLite WAL mode) └── config.db-shm # Shared memory file + ```text ### Admin API Integration ``` + use rusqlite::{Connection, params}; pub struct ConfigStore { @@ -215,6 +217,7 @@ impl ConfigStore { Ok(namespaces) } } + ```text ## Rationale @@ -322,6 +325,7 @@ impl ConfigStore { ### Initialization on Startup ``` + pub async fn initialize_config_store() -> Result { let db_path = Path::new("/var/lib/prism/config.db"); @@ -340,11 +344,13 @@ pub async fn initialize_config_store() -> Result { Ok(store) } + ```text ### Backup Strategy ``` + # Daily backup via cron #!/bin/bash # /etc/cron.daily/prism-config-backup @@ -360,11 +366,13 @@ cp $DB_PATH $BACKUP_DIR/config-$(date +%Y%m%d).db # Retain last 30 days find $BACKUP_DIR -name "config-*.db" -mtime +30 -delete + ```text ### Read-Heavy Optimization ``` + // Use connection pool for concurrent reads use r2d2_sqlite::SqliteConnectionManager; use r2d2::Pool; @@ -389,6 +397,7 @@ impl ConfigStore { Ok(Self { pool }) } } + ```text ### Multi-Instance Deployment (Future) diff --git a/docs-cms/adr/adr-037-kubernetes-operator.md b/docs-cms/adr/adr-037-kubernetes-operator.md index 68efabb85..fbed8a75f 100644 --- a/docs-cms/adr/adr-037-kubernetes-operator.md +++ b/docs-cms/adr/adr-037-kubernetes-operator.md @@ -193,6 +193,7 @@ status: β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Reconciliation Logic @@ -224,16 +225,19 @@ status: **Without Operator** (raw Kubernetes manifests): ``` + # Must manually define: - Deployment for each shard - StatefulSet for SQLite persistence - Services for each shard - ConfigMaps for namespace configs (must sync manually!) - Plugin sidecar injection (manual, error-prone) + ```text **With Operator**: ``` + # Just define: apiVersion: prism.io/v1alpha1 kind: PrismNamespace @@ -243,6 +247,7 @@ spec: backend: postgres pattern: keyvalue # Operator handles the rest! + ```text ### Compared to Alternatives diff --git a/docs-cms/adr/adr-038-backend-connector-buffers.md b/docs-cms/adr/adr-038-backend-connector-buffers.md index 84f4776a6..c502f4461 100644 --- a/docs-cms/adr/adr-038-backend-connector-buffers.md +++ b/docs-cms/adr/adr-038-backend-connector-buffers.md @@ -87,6 +87,7 @@ From Netflix metrics: β”‚ PostgreSQL β”‚ β”‚ Redis β”‚ β”‚ Kafka β”‚ β”‚ Cluster β”‚ β”‚ Cluster β”‚ β”‚ Cluster β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Connector Buffer Responsibilities @@ -106,6 +107,7 @@ From Netflix metrics: **Before** (plugin does everything): ``` + impl PostgresPlugin { async fn execute(&self, req: ExecuteRequest) -> Result { // Plugin manages: @@ -119,10 +121,12 @@ impl PostgresPlugin { Ok(result) } } + ```text **After** (plugin delegates to connector): ``` + impl PostgresPlugin { async fn execute(&self, req: ExecuteRequest) -> Result { // Plugin just translates request β†’ connector format @@ -134,6 +138,7 @@ impl PostgresPlugin { Ok(self.to_plugin_response(response)) } } + ```text ## Rationale @@ -145,6 +150,7 @@ impl PostgresPlugin { Compute-heavy backends (ClickHouse aggregations, graph queries) need more CPU than connection management: ``` + # Scale plugin for compute (Rust) apiVersion: v1 kind: Deployment @@ -166,6 +172,7 @@ spec: resources: cpu: "1" connections: 500 # Many connections per instance + ```text **2. Language Choice** @@ -187,11 +194,13 @@ Plugin crashes β†’ Connector keeps connections alive β†’ New plugin instance rec Without separation: Plugin crashes β†’ All connections lost β†’ Reconnect storm to database + ```text **4. Global Resource Management** ``` + // Connector enforces global limits across all plugin instances type PostgresConnector struct { globalPool *pgxpool.Pool // Max 500 connections globally @@ -199,6 +208,7 @@ type PostgresConnector struct { // All plugin instances share this pool rateLimiter rate.Limiter // Max 10K QPS globally } + ```text ### Why Go for Connectors? @@ -260,6 +270,7 @@ type PostgresConnector struct { ### Connector gRPC API ``` + syntax = "proto3"; package prism.connector; @@ -298,11 +309,13 @@ message ConnectorStats { int64 queued_requests = 4; double avg_latency_ms = 5; } + ```text ### PostgreSQL Connector Example (Go) ``` + package main import ( @@ -345,12 +358,14 @@ func (c *PostgresConnector) Execute(ctx context.Context, req *pb.ExecuteRequest) c.circuitBreaker.RecordSuccess() return &pb.ExecuteResponse{Success: true, Result: result}, nil } + ```text ### Deployment Topology **Option 1: Sidecar Pattern** (Recommended for Kubernetes): ``` + apiVersion: v1 kind: Pod metadata: @@ -373,6 +388,7 @@ spec: env: - name: CONNECTOR_ENDPOINT value: "localhost:50200" # Talk to sidecar connector + ```text **Option 2: Shared Connector Pool** (For bare metal): diff --git a/docs-cms/adr/adr-039-cli-acceptance-testing.md b/docs-cms/adr/adr-039-cli-acceptance-testing.md index 165ac7383..31c3a063b 100644 --- a/docs-cms/adr/adr-039-cli-acceptance-testing.md +++ b/docs-cms/adr/adr-039-cli-acceptance-testing.md @@ -178,11 +178,13 @@ tools/ β”‚ └── ... β”œβ”€β”€ acceptance_test.go # testscript runner └── go.mod + ```text ### Test Runner ``` + // tools/acceptance_test.go package tools_test @@ -219,11 +221,13 @@ func mainCLI() int { } return 0 } + ```text ### Example Test Script ``` + # testdata/script/namespace_create.txtar # Test: Create a namespace with explicit configuration @@ -253,11 +257,13 @@ stdout 'Deleted namespace "my-app"' # Verify deletion prismctl namespace list ! stdout 'my-app' + ```text ### Advanced Test: Configuration File Discovery ``` + # testdata/script/config_discovery.txtar # Create project config file @@ -278,11 +284,13 @@ stdout 'Backend: postgres' prismctl config show stdout 'namespace: my-project' stdout 'backend:.*postgres' + ```text ### Multi-Step Workflow Test ``` + # testdata/script/shadow_traffic.txtar # Setup: Create source namespace @@ -312,11 +320,13 @@ stdout 'Shadow traffic disabled' # Cleanup prismctl namespace delete prod-app --force prismctl namespace delete prod-app-new --force + ```text ### Error Handling Test ``` + # testdata/script/namespace_errors.txtar # Test: Create namespace with invalid backend @@ -337,11 +347,13 @@ stderr 'error: namespace "duplicate" already exists' # Cleanup prismctl namespace delete duplicate --force + ```text ### JSON Output Test ``` + # testdata/script/json_output.txtar # Create test namespace @@ -359,6 +371,7 @@ stdout '"name":.*"json-test"' # Cleanup prismctl namespace delete json-test --force + ```text ## Testing Strategy diff --git a/docs-cms/adr/adr-040-go-binary-admin-cli.md b/docs-cms/adr/adr-040-go-binary-admin-cli.md index 95679381b..12e203af5 100644 --- a/docs-cms/adr/adr-040-go-binary-admin-cli.md +++ b/docs-cms/adr/adr-040-go-binary-admin-cli.md @@ -28,6 +28,7 @@ Prism needs an admin CLI for operators to manage namespaces, monitor health, and Build the admin CLI as a **Go binary** named **`prismctl`**, following patterns from successful CLIs like `kubectl`, `docker`, and `gh`. **Installation**: + ```bash # Download single binary curl -LO https://github.com/prism/releases/download/v1.0.0/prismctl-$(uname -s)-$(uname -m) @@ -39,6 +40,7 @@ go install github.com/jrepp/prism-data-layer/tools/cmd/prismctl@latest ``` **Usage**: + ```bash prismctl namespace list prismctl namespace create my-app --description "My application" @@ -101,6 +103,7 @@ for i in {1..100}; do prismctl namespace list; done Go has the best CLI ecosystem: **Cobra + Viper** (used by Kubernetes, Docker, GitHub CLI): + ```go var rootCmd = &cobra.Command{ Use: "prismctl", @@ -166,6 +169,7 @@ No need for: #### 6. Easy Distribution **GitHub Releases** (recommended): + ```bash # Automatically upload binaries gh release create v1.0.0 \ @@ -175,6 +179,7 @@ gh release create v1.0.0 \ ``` Users download: + ```bash curl -LO https://github.com/prism/releases/latest/download/prismctl-$(uname -s)-$(uname -m) chmod +x prismctl-* @@ -182,6 +187,7 @@ sudo mv prismctl-* /usr/local/bin/prismctl ``` **Homebrew** (macOS/Linux): + ```ruby class Prismctl < Formula desc "Admin CLI for Prism data gateway" @@ -206,6 +212,7 @@ brew install prismctl #### 7. No Runtime Dependencies Go binary needs **nothing**: + ```bash # Check dependencies (none!) $ ldd prismctl @@ -217,6 +224,7 @@ prismctl version 1.0.0 ``` Compare to Python: + ```bash # Python requires: - Python 3.10+ interpreter @@ -237,25 +245,31 @@ tools/ β”œβ”€β”€ health.go # Health commands β”œβ”€β”€ session.go # Session commands └── config.go # Config management + ```text **Build**: ``` + cd tools go build -o prismctl ./cmd/prismctl ./prismctl --help + ```text **Release build** (optimized): ``` + go build -ldflags="-s -w" -o prismctl ./cmd/prismctl upx prismctl # Optional: compress binary (10MB β†’ 3MB) + ```text ### Configuration **Default config** (`~/.prism/config.yaml`): ``` + admin: endpoint: localhost:8981 @@ -272,6 +286,7 @@ plugins: logging: level: info + ```text **Precedence** (Viper): @@ -285,6 +300,7 @@ logging: **Superseded by ADR-045**: Bootstrap is now handled by `prismctl stack init`. ``` + # Install prismctl (includes bootstrap functionality) go install github.com/jrepp/prism-data-layer/tools/cmd/prismctl@latest @@ -298,6 +314,7 @@ prismctl stack start # Use prismctl for admin operations prismctl namespace list prismctl health + ```text Rationale: @@ -311,6 +328,7 @@ Rationale: **Plugin manifests** in `~/.prism/plugins/`: ``` + # ~/.prism/plugins/postgres.yaml name: postgres image: prism/postgres-plugin:latest @@ -319,10 +337,12 @@ backends: [postgres] capabilities: - keyvalue - transactions + ```text **CLI integration**: ``` + # List available plugins prismctl plugin list @@ -334,10 +354,12 @@ prismctl plugin stop postgres # Plugin health prismctl plugin health postgres + ```text **Go implementation**: ``` + func runPluginStart(cmd *cobra.Command, args []string) error { pluginName := args[0] @@ -365,6 +387,7 @@ func loadPluginManifest(name string) (*PluginManifest, error) { return &manifest, nil } + ```text ## Consequences diff --git a/docs-cms/adr/adr-042-sqs-queue-backend.md b/docs-cms/adr/adr-042-sqs-queue-backend.md index c2c28e551..c23821667 100644 --- a/docs-cms/adr/adr-042-sqs-queue-backend.md +++ b/docs-cms/adr/adr-042-sqs-queue-backend.md @@ -179,6 +179,7 @@ message DeleteMessageResponse { ### Example: Job Processing **Producer** (send job to queue): + ```go import pb "prism/queue/v1" @@ -203,6 +204,7 @@ func submitJob(client pb.QueueServiceClient, jobData string) error { ``` **Consumer** (process jobs from queue): + ```go func processJobs(client pb.QueueServiceClient) { for { diff --git a/docs-cms/adr/adr-043-plugin-capability-discovery.md b/docs-cms/adr/adr-043-plugin-capability-discovery.md index c008314a7..6925afb21 100644 --- a/docs-cms/adr/adr-043-plugin-capability-discovery.md +++ b/docs-cms/adr/adr-043-plugin-capability-discovery.md @@ -380,6 +380,7 @@ message AbstractionCapabilities { ``` **Example: Postgres plugin**: + ```go capabilities := &PluginCapabilities{ PluginName: "postgres", diff --git a/docs-cms/adr/adr-044-tinkerpop-gremlin-plugin.md b/docs-cms/adr/adr-044-tinkerpop-gremlin-plugin.md index 9b8162838..560a07bab 100644 --- a/docs-cms/adr/adr-044-tinkerpop-gremlin-plugin.md +++ b/docs-cms/adr/adr-044-tinkerpop-gremlin-plugin.md @@ -55,6 +55,7 @@ prism-graph-plugin (generic) β”‚ └── tinkergraph/ # In-memory (for testing) └── proto/ └── graph.proto # Unified graph API + ```text ## Generic Gremlin Plugin Architecture @@ -62,6 +63,7 @@ prism-graph-plugin (generic) ### Configuration ``` + # Generic Gremlin Server connection graph_backend: type: gremlin @@ -78,11 +80,13 @@ graph_backend: max_connections: 20 capabilities: auto_detect: true # Query server for capabilities + ```text ### Neptune-Specific Configuration ``` + # Neptune (inherits from gremlin, adds AWS-specific) graph_backend: type: neptune @@ -99,6 +103,7 @@ graph_backend: cloudwatch: metrics_enabled: true log_group: /aws/neptune/my-cluster + ```text ## Capability Detection @@ -106,6 +111,7 @@ graph_backend: Generic plugin **auto-detects** backend capabilities: ``` + // gremlin-core/capabilities.go func (c *GremlinClient) DetectCapabilities() (*PluginCapabilities, error) { caps := &PluginCapabilities{ @@ -168,6 +174,7 @@ func (c *GremlinClient) queryServerFeatures() (*ServerFeatures, error) { return features, nil } + ```text ### Backend-Specific Specialization @@ -175,6 +182,7 @@ func (c *GremlinClient) queryServerFeatures() (*ServerFeatures, error) { Neptune plugin **extends** generic plugin with AWS features: ``` + // plugins/neptune/plugin.go type NeptunePlugin struct { *gremlin.GenericGremlinPlugin // Embed generic plugin @@ -215,6 +223,7 @@ func (p *NeptunePlugin) GetCapabilities() (*PluginCapabilities, error) { return caps, nil } + ```text ## Example: Multi-Backend Support @@ -224,11 +233,13 @@ Application uses **same Gremlin API** across different backends: ### Development: TinkerGraph (in-memory) ``` + namespace: user-graph-dev backend: type: tinkergraph config: auto_detect: true + ```text **Detected Capabilities**: @@ -241,6 +252,7 @@ backend: ### Staging: JanusGraph (self-hosted) ``` + namespace: user-graph-staging backend: type: janusgraph @@ -251,6 +263,7 @@ backend: method: basic username: prism password: ${JANUS_PASSWORD} + ```text **Detected Capabilities**: @@ -263,6 +276,7 @@ backend: ### Production: Neptune (AWS) ``` + namespace: user-graph-prod backend: type: neptune @@ -271,6 +285,7 @@ backend: region: us-east-1 auth: method: iam + ```text **Detected Capabilities**: @@ -286,6 +301,7 @@ backend: Application code is **identical** across all backends: ``` + // Same code works with TinkerGraph, JanusGraph, Neptune client := prism.NewGraphClient(namespace) @@ -309,6 +325,7 @@ client.AddEdge("FOLLOWS", alice.ID, bob.ID, map[string]interface{}{ result, err := client.Gremlin( "g.V('user:alice').out('FOLLOWS').out('FOLLOWS').dedup().limit(10)", ) + ```text **Backend selection** is configuration-driven, not code-driven. @@ -318,6 +335,7 @@ result, err := client.Gremlin( Prism **validates queries** against backend capabilities: ``` + func (p *Proxy) ExecuteGremlinQuery( namespace string, query string, @@ -372,6 +390,7 @@ func validateQueryFeatures(query string, caps *PluginCapabilities) error { return nil } + ```text ## Benefits of Generic Plugin @@ -381,6 +400,7 @@ func validateQueryFeatures(query string, caps *PluginCapabilities) error { Start with in-memory TinkerGraph, move to production Neptune: ``` + # Development (local) prismctl namespace create user-graph-dev --backend tinkergraph @@ -389,6 +409,7 @@ prismctl namespace create user-graph-staging --backend janusgraph # Production (AWS) prismctl namespace create user-graph-prod --backend neptune + ```text ### 2. **Cost Optimization** @@ -396,6 +417,7 @@ prismctl namespace create user-graph-prod --backend neptune Use cheaper backends for non-critical workloads: ``` + # Expensive: Neptune (ACID, replicas, managed) production_graph: backend: neptune @@ -410,6 +432,7 @@ staging_graph: dev_graph: backend: tinkergraph cost: $0 (local) + ```text ### 3. **Vendor Independence** @@ -426,6 +449,7 @@ Not locked into AWS: Integration tests use TinkerGraph (no external dependencies): ``` + func TestGraphTraversal(t *testing.T) { // Fast, deterministic, no setup required plugin := NewTinkerGraphPlugin() @@ -441,6 +465,7 @@ func TestGraphTraversal(t *testing.T) { assert.Len(t, result.Vertices, 1) assert.Equal(t, "B", result.Vertices[0].Id) } + ```text ## Community Ecosystem diff --git a/docs-cms/adr/adr-045-prismctl-stack-management.md b/docs-cms/adr/adr-045-prismctl-stack-management.md index f1a878ff5..8f1327885 100644 --- a/docs-cms/adr/adr-045-prismctl-stack-management.md +++ b/docs-cms/adr/adr-045-prismctl-stack-management.md @@ -31,6 +31,7 @@ Previously, we considered creating a separate `hashistack` tool, but this would Add a **`stack` subcommand** to `prismctl` that manages infrastructure provisioning and lifecycle. **Installation** (single command): + ```bash # Install prismctl with stack management go install github.com/jrepp/prism-data-layer/tools/cmd/prismctl@latest @@ -40,6 +41,7 @@ prismctl stack init ``` **Usage**: + ```bash # Initialize stack configuration prismctl stack init @@ -101,6 +103,7 @@ prismctl β”œβ”€β”€ status # Check stack health β”œβ”€β”€ use # Switch stack provider └── providers # List available providers + ```text #### 3. Shared Configuration @@ -108,6 +111,7 @@ prismctl Stack management shares configuration with other prismctl commands: ``` + # ~/.prism/config.yaml admin: endpoint: localhost:8981 @@ -128,6 +132,7 @@ plugins: postgres: image: prism/postgres-plugin:latest port: 9090 + ```text **Benefits**: @@ -144,6 +149,7 @@ The `stack` subcommand supports multiple infrastructure providers: Uses Consul, Vault, and Nomad for service discovery, secrets, and orchestration: ``` + # Use Hashicorp stack (default) prismctl stack init --provider hashicorp @@ -153,10 +159,12 @@ prismctl stack init --provider hashicorp # - PostgreSQL (via Docker) # - Kafka (via Docker) # - NATS (via Docker) + ```text **Configuration** (`~/.prism/stacks/hashicorp.yaml`): ``` + provider: hashicorp services: @@ -186,10 +194,12 @@ services: enabled: true image: nats:latest ports: [4222, 8222] + ```text **Stack operations**: ``` + type HashicorpStack struct { config *HashicorpConfig } @@ -212,6 +222,7 @@ func (s *HashicorpStack) Start(ctx context.Context) error { return nil } + ```text #### 2. Docker Compose @@ -219,15 +230,18 @@ func (s *HashicorpStack) Start(ctx context.Context) error { Simple Docker-based local development: ``` + # Use Docker Compose stack prismctl stack init --provider docker-compose # Uses docker-compose.yml for all services prismctl stack start + ```text **Configuration** (`~/.prism/stacks/docker-compose.yaml`): ``` + provider: docker-compose compose_file: ~/.prism/docker-compose.yml @@ -237,10 +251,12 @@ services: kafka: {} nats: {} redis: {} + ```text **Stack operations**: ``` + type DockerComposeStack struct { composeFile string } @@ -252,6 +268,7 @@ func (s *DockerComposeStack) Start(ctx context.Context) error { ) return cmd.Run() } + ```text #### 3. AWS @@ -259,6 +276,7 @@ func (s *DockerComposeStack) Start(ctx context.Context) error { Cloud-native using AWS services: ``` + # Use AWS stack prismctl stack init --provider aws @@ -267,10 +285,12 @@ prismctl stack init --provider aws # - MSK (Kafka) cluster # - Secrets Manager for credentials # - VPC, subnets, security groups + ```text **Configuration** (`~/.prism/stacks/aws.yaml`): ``` + provider: aws region: us-west-2 @@ -291,6 +311,7 @@ services: secrets_manager: enabled: true secrets: [postgres-admin, kafka-creds] + ```text #### 4. Kubernetes @@ -298,11 +319,13 @@ services: Deploy to Kubernetes cluster: ``` + # Use Kubernetes stack prismctl stack init --provider kubernetes # Applies Helm charts or manifests prismctl stack start + ```text ### Stack Provider Interface @@ -310,6 +333,7 @@ prismctl stack start All providers implement a common interface: ``` + type StackProvider interface { // Initialize creates configuration files Init(ctx context.Context, opts *InitOptions) error @@ -345,12 +369,14 @@ type Endpoints struct { Kafka []string NATS string } + ```text ### Implementation Example **Stack initialization**: ``` + // cmd/prismctl/stack.go var stackInitCmd = &cobra.Command{ Use: "init", @@ -387,10 +413,12 @@ func createStackProvider(name string) (StackProvider, error) { return nil, fmt.Errorf("unknown provider: %s", name) } } + ```text **Stack start**: ``` + var stackStartCmd = &cobra.Command{ Use: "start", Short: "Start infrastructure stack", @@ -424,6 +452,7 @@ func runStackStart(cmd *cobra.Command, args []string) error { return nil } + ```text ## Bootstrap Workflow @@ -431,6 +460,7 @@ func runStackStart(cmd *cobra.Command, args []string) error { ### Installation and Setup ``` + # 1. Install prismctl go install github.com/jrepp/prism-data-layer/tools/cmd/prismctl@latest @@ -463,6 +493,7 @@ prismctl stack start # 4. Use prismctl for admin operations prismctl health prismctl namespace create my-app + ```text ### One-Command Bootstrap @@ -470,12 +501,14 @@ prismctl namespace create my-app Combine init + start: ``` + # Bootstrap everything in one command prismctl stack bootstrap # Equivalent to: # prismctl stack init # prismctl stack start + ```text ## Configuration Files diff --git a/docs-cms/adr/adr-047-opentelemetry-tracing-integration.md b/docs-cms/adr/adr-047-opentelemetry-tracing-integration.md index b2930cf1b..71a4e8470 100644 --- a/docs-cms/adr/adr-047-opentelemetry-tracing-integration.md +++ b/docs-cms/adr/adr-047-opentelemetry-tracing-integration.md @@ -56,11 +56,13 @@ prism.handle_request [150ms total] β”‚ └─ postgres.query [137ms] β”‚ └─ SQL: SELECT * FROM... [135ms] └─ prism.response.serialize [3ms] + ```text ### Architecture ``` + sequenceDiagram participant Client participant Proxy as Rust Proxy @@ -83,6 +85,7 @@ sequenceDiagram Proxy-->>Client: gRPC Response Note over Client,Backend: All spans linked by
trace_id: abc123 + ```text ### Trace Context Propagation @@ -774,6 +777,7 @@ prism-proxy: handle_data_request [142ms] β”‚ β”œβ”€ prism-plugin-postgres: pool.acquire [2ms] β”‚ └─ prism-plugin-postgres: postgres.query [134ms] β”‚ └─ postgresql: SELECT * FROM users WHERE id = $1 [132ms] + ```text ## References diff --git a/docs-cms/adr/adr-048-local-signoz-observability.md b/docs-cms/adr/adr-048-local-signoz-observability.md index f65d4a47d..93355c59a 100644 --- a/docs-cms/adr/adr-048-local-signoz-observability.md +++ b/docs-cms/adr/adr-048-local-signoz-observability.md @@ -80,6 +80,7 @@ Signoz Components: β”œβ”€β”€ Query Service (API + UI :3301) β”œβ”€β”€ ClickHouse (storage :9000) └── AlertManager (optional) + ```text ## Decision @@ -122,6 +123,7 @@ Signoz Components: Location: `local-dev/signoz/docker-compose.signoz.yml` ``` + version: '3.8' services: @@ -220,6 +222,7 @@ networks: driver: bridge prism: external: true # Connect to Prism components + ```text ### OpenTelemetry Collector Configuration @@ -227,6 +230,7 @@ networks: Location: `local-dev/signoz/otel-collector-config.yaml` ``` + receivers: otlp: protocols: @@ -287,6 +291,7 @@ service: receivers: [otlp] processors: [memory_limiter, resource, batch] exporters: [clickhouse] + ```text ### Prism Proxy Integration @@ -294,6 +299,7 @@ service: The proxy automatically detects and uses Signoz when available: ``` + // proxy/src/observability/tracer.rs pub fn init_tracer(config: &ObservabilityConfig) -> Result { @@ -320,6 +326,7 @@ pub fn init_tracer(config: &ObservabilityConfig) -> Result { Ok(tracer) } + ```text ### Plugin Integration @@ -327,6 +334,7 @@ pub fn init_tracer(config: &ObservabilityConfig) -> Result { Plugins receive OTLP configuration via environment variables: ``` + // plugins/core/observability/tracer.go func InitTracer(serviceName string) error { @@ -352,6 +360,7 @@ func InitTracer(serviceName string) error { otel.SetTracerProvider(tp) return nil } + ```text ## Usage @@ -359,6 +368,7 @@ func InitTracer(serviceName string) error { ### Starting Signoz ``` + # Start Signoz stack cd local-dev/signoz docker-compose -f docker-compose.signoz.yml up -d @@ -368,11 +378,13 @@ docker-compose -f docker-compose.signoz.yml ps # Access UI open http://localhost:3301 + ```text ### Starting Prism with Signoz ``` + # Set OTLP endpoint environment variable export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 @@ -384,6 +396,7 @@ cargo run --release cd plugins/postgres OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ go run ./cmd/server + ```text ### Viewing Traces @@ -396,11 +409,13 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ ### Resetting Data ``` + # Stop and remove volumes docker-compose -f docker-compose.signoz.yml down -v # Restart fresh docker-compose -f docker-compose.signoz.yml up -d + ```text ## Consequences diff --git a/docs-cms/adr/adr-049-podman-container-optimization.md b/docs-cms/adr/adr-049-podman-container-optimization.md index f23580307..2c13ad967 100644 --- a/docs-cms/adr/adr-049-podman-container-optimization.md +++ b/docs-cms/adr/adr-049-podman-container-optimization.md @@ -41,6 +41,7 @@ Developers need the **fastest possible build-test cycle** for backend plugin dev - OCI compliance questions **Testing Workflow:** + ```bash # Current slow path (Docker) docker-compose up -d postgres # 5-10 seconds @@ -50,6 +51,7 @@ docker-compose down # 2-3 seconds ``` **Desired Workflow:** + ```bash # Instant testing (goal) go test ./... # <1 second total @@ -124,6 +126,7 @@ For backends requiring real services, use **Podman**: - βœ… **Smaller footprint**: No daemon overhead **Podman on Mac:** + ```bash # Install brew install podman @@ -249,6 +252,7 @@ func TestPostgresPlugin_RealBackend(t *testing.T) { ``` **Run modes:** + ```bash # Fast: In-process only (TDD workflow) go test -short ./... # <1 second @@ -279,6 +283,7 @@ podman machine start prism-dev ``` **Makefile updates:** + ```makefile # Old DOCKER := docker @@ -418,6 +423,7 @@ network_backend = "netavark" # Faster than CNI ``` **VM resource allocation:** + ```bash # For 16GB Mac (adjust proportionally) podman machine init prism-dev \ @@ -451,6 +457,7 @@ ENTRYPOINT ["/plugin"] ``` **Build cache usage:** + ```bash # First build: ~60 seconds (downloads deps) podman build -t plugin-postgres:latest . @@ -492,6 +499,7 @@ func newRealBackend(t *testing.T) Store { ``` **Environment-based control:** + ```bash # Development: Instant tests only go test -short ./... diff --git a/docs-cms/adr/adr-051-minio-claim-check-testing.md b/docs-cms/adr/adr-051-minio-claim-check-testing.md index ce51d3c40..4b4d43a6c 100644 --- a/docs-cms/adr/adr-051-minio-claim-check-testing.md +++ b/docs-cms/adr/adr-051-minio-claim-check-testing.md @@ -206,6 +206,7 @@ func createTestBucket(t *testing.T, driver ObjectStoreInterface) string { ## Implementation Plan ### Phase 1: MinIO Driver (Week 1) + ```text pkg/drivers/minio/ β”œβ”€β”€ driver.go # ObjectStoreInterface implementation @@ -216,6 +217,7 @@ pkg/drivers/minio/ ``` ### Phase 2: Test Framework Integration (Week 1) + ```text tests/acceptance/backends/ └── minio.go # Backend registration @@ -225,6 +227,7 @@ tests/acceptance/framework/ ``` ### Phase 3: Claim Check Tests (Week 2) + ```text tests/acceptance/patterns/claimcheck/ β”œβ”€β”€ claimcheck_test.go # Multi-pattern tests @@ -301,6 +304,7 @@ func TestMinIOSetup(t *testing.T) { ## Monitoring and Debugging ### Container Logs + ```bash # View MinIO logs during test failures docker logs @@ -311,6 +315,7 @@ t.Logf("MinIO logs: %s", minioContainer.Logs(ctx)) ### MinIO Console Access web UI for debugging: + ```bash # Get console URL echo "http://$(docker port 9001)" @@ -320,6 +325,7 @@ echo "http://$(docker port 9001)" ``` ### Performance Metrics + ```go // Track MinIO operation latency func (d *MinioDriver) Put(ctx context.Context, bucket, key string, data []byte) error { diff --git a/docs-cms/adr/adr-052-object-store-interface.md b/docs-cms/adr/adr-052-object-store-interface.md index 31528fbac..ade4712ca 100644 --- a/docs-cms/adr/adr-052-object-store-interface.md +++ b/docs-cms/adr/adr-052-object-store-interface.md @@ -328,6 +328,7 @@ func (d *MinioDriver) BucketExists(ctx context.Context, bucket string) (bool, er - Add `PatternObjectStore` constant to framework ### Phase 2: MinIO Driver (3 days) + ```text pkg/drivers/minio/ β”œβ”€β”€ driver.go # Main implementation @@ -338,6 +339,7 @@ pkg/drivers/minio/ ``` ### Phase 3: S3 Driver (3 days) + ```text pkg/drivers/s3/ β”œβ”€β”€ driver.go # AWS SDK v2 implementation @@ -347,6 +349,7 @@ pkg/drivers/s3/ ``` ### Phase 4: Mock Implementation (1 day) + ```text pkg/drivers/mock/ └── objectstore.go # In-memory implementation for unit tests @@ -355,6 +358,7 @@ pkg/drivers/mock/ ## Testing Strategy ### Unit Tests (No External Dependencies) + ```go func TestObjectStoreInterface(t *testing.T) { // Use in-memory mock @@ -374,6 +378,7 @@ func TestObjectStoreInterface(t *testing.T) { ``` ### Integration Tests (MinIO via testcontainers) + ```go func TestMinIODriver(t *testing.T) { driver, cleanup := setupMinIO(t) @@ -394,6 +399,7 @@ func runObjectStoreTests(t *testing.T, store ObjectStoreInterface) { ``` ### Contract Tests (Verify Backend Compatibility) + ```go // tests/interface-suites/objectstore/ func TestObjectStoreContract(t *testing.T) { @@ -419,6 +425,7 @@ func TestObjectStoreContract(t *testing.T) { ## Error Handling ### Error Types + ```go // pkg/plugin/errors.go @@ -469,6 +476,7 @@ func (d *MinioDriver) translateError(err error) error { ### 1. Access Control Interface doesn't include ACL operations - manage via backend configuration: + ```yaml minio: access_key: ${MINIO_ACCESS_KEY} @@ -480,6 +488,7 @@ s3: ### 2. Encryption Backend-specific encryption handled via driver configuration: + ```yaml s3: server_side_encryption: AES256 @@ -488,6 +497,7 @@ s3: ### 3. Network Security TLS configuration per backend: + ```yaml minio: use_ssl: true @@ -496,6 +506,7 @@ minio: ### 4. Audit Logging All operations logged via driver observability hooks: + ```go func (d *MinioDriver) Put(ctx context.Context, bucket, key string, data []byte) error { start := time.Now() @@ -515,6 +526,7 @@ func (d *MinioDriver) Put(ctx context.Context, bucket, key string, data []byte) ## Performance Considerations ### 1. Connection Pooling + ```go type MinioDriver struct { client *minio.Client // Internally connection-pooled @@ -527,6 +539,7 @@ type MinioDriver struct { ``` ### 2. Retry Strategy + ```go type Config struct { MaxRetries int `json:"max_retries"` @@ -536,6 +549,7 @@ type Config struct { ``` ### 3. Streaming Thresholds + ```go const ( // Use PutStream for payloads > 10MB @@ -552,6 +566,7 @@ func (p *Producer) uploadClaim(ctx context.Context, payload []byte) error { ``` ### 4. Metadata Caching + ```go // Cache frequently accessed metadata type MetadataCache struct { diff --git a/docs-cms/adr/adr-053-claim-check-ttl-garbage-collection.md b/docs-cms/adr/adr-053-claim-check-ttl-garbage-collection.md index a2f881b40..c31261425 100644 --- a/docs-cms/adr/adr-053-claim-check-ttl-garbage-collection.md +++ b/docs-cms/adr/adr-053-claim-check-ttl-garbage-collection.md @@ -27,24 +27,28 @@ The claim check pattern (RFC-033) stores large payloads in object storage. Witho ### Problem Statement **Scenario 1: Happy Path** + ```text Producer β†’ Upload claim β†’ Consumer retrieves β†’ Claim should be deleted (claim valid) (immediate cleanup) ``` **Scenario 2: Consumer Crash** + ```text Producer β†’ Upload claim β†’ Consumer crashes β†’ Claim orphaned (claim valid) (never retrieved) (needs TTL cleanup) ``` **Scenario 3: Slow Consumer** + ```text Producer β†’ Upload claim β†’ Long processing β†’ Consumer retrieves β†’ Claim deleted (claim valid) (still valid) (delayed cleanup) ``` **Scenario 4: Replay/Redelivery** + ```text Producer β†’ Upload claim β†’ Consumer retrieves β†’ Message redelivered β†’ Claim missing (claim valid) (claim deleted) (ERROR!) @@ -297,6 +301,7 @@ func (p *Proxy) validateClaimCheckTTL(producerTTL, consumerTTL ClaimCheckTTL) er ### TTL Configuration Examples #### Example 1: Aggressive Cleanup (Minimize Storage Cost) + ```yaml claim_check: ttl: @@ -309,6 +314,7 @@ claim_check: **Use Case**: High-throughput, reliable consumers, no message redelivery. #### Example 2: Conservative (Handle Slow Consumers) + ```yaml claim_check: ttl: @@ -321,6 +327,7 @@ claim_check: **Use Case**: Long-running ML processing, batch jobs, debugging. #### Example 3: Redelivery Protection (Handle Message Broker Retries) + ```yaml claim_check: ttl: @@ -363,6 +370,7 @@ claim_check: ### S3/MinIO Lifecycle Behavior **Lifecycle Rules**: + ```xml diff --git a/docs-cms/adr/adr-055-proxy-admin-control-plane.md b/docs-cms/adr/adr-055-proxy-admin-control-plane.md index 3a5a1863d..53877f629 100644 --- a/docs-cms/adr/adr-055-proxy-admin-control-plane.md +++ b/docs-cms/adr/adr-055-proxy-admin-control-plane.md @@ -36,6 +36,7 @@ We need a control plane protocol that enables: Implement bidirectional gRPC control plane protocol between prism-proxy and prism-admin: **Proxy Startup**: + ```bash prism-proxy --admin-endpoint admin.prism.local:8981 --proxy-id proxy-01 --region us-west-2 ``` @@ -73,6 +74,7 @@ Namespaces include partition identifier for horizontal scaling: - **Rebalancing**: Admin redistributes partitions when proxies join/leave Example partition distribution: + ```text proxy-01: partitions [0-63] β†’ namespaces: ns-a (hash=12), ns-d (hash=55) proxy-02: partitions [64-127] β†’ namespaces: ns-b (hash=88), ns-e (hash=100) diff --git a/docs-cms/adr/adr-056-launcher-admin-control-plane.md b/docs-cms/adr/adr-056-launcher-admin-control-plane.md index 19c0d25dd..93e87ce9e 100644 --- a/docs-cms/adr/adr-056-launcher-admin-control-plane.md +++ b/docs-cms/adr/adr-056-launcher-admin-control-plane.md @@ -38,6 +38,7 @@ Per ADR-055, prism-proxy now connects to prism-admin via control plane protocol. Extend the ControlPlane gRPC service (from ADR-055) to support launcher registration and pattern lifecycle management: **Launcher Startup**: + ```bash pattern-launcher --admin-endpoint admin.prism.local:8981 --launcher-id launcher-01 --listen :7070 ``` diff --git a/docs-cms/adr/adr-057-prism-launcher-refactoring.md b/docs-cms/adr/adr-057-prism-launcher-refactoring.md index 000ea185d..d8f0781e2 100644 --- a/docs-cms/adr/adr-057-prism-launcher-refactoring.md +++ b/docs-cms/adr/adr-057-prism-launcher-refactoring.md @@ -50,6 +50,7 @@ Refactor `pattern-launcher` to `prism-launcher` as a general-purpose control pla **Architecture Changes**: 1. **Process Type Abstraction**: + ```go type ProcessType string @@ -74,6 +75,7 @@ type ManagedProcess struct { ``` 2. **Unified Control Plane Registration**: + ```go // Register launcher with admin (not just pattern-launcher) type LauncherRegistration struct { @@ -87,6 +89,7 @@ type LauncherRegistration struct { ``` 3. **Process Assignment Protocol**: + ```go // Admin assigns any process type, not just patterns type ProcessAssignment struct { @@ -113,6 +116,7 @@ type ProcessConfig struct { ``` 4. **Process Manager Generalization**: + ```go // pkg/procmgr stays mostly the same but concepts generalize type ProcessManager struct { @@ -128,6 +132,7 @@ func (pm *ProcessManager) Launch(proc *ManagedProcess) error { ``` 5. **Launcher Command Structure**: + ```bash prism-launcher \ --admin-endpoint admin.prism.local:8981 \ @@ -238,6 +243,7 @@ prism-launcher \ ### Phase 1: Rename and Generalize Types (Week 1) 1. Rename binary: + ```bash # Makefile build/binaries/prism-launcher: pkg/launcher/*.go cmd/prism-launcher/*.go @@ -245,6 +251,7 @@ build/binaries/prism-launcher: pkg/launcher/*.go cmd/prism-launcher/*.go ``` 2. Introduce ProcessType enum: + ```go // pkg/launcher/types.go type ProcessType string @@ -258,6 +265,7 @@ const ( ``` 3. Rename Process β†’ ManagedProcess: + ```go // pkg/launcher/process.go type ManagedProcess struct { diff --git a/docs-cms/analysis-summary.md b/docs-cms/analysis-summary.md index 38e8c06cf..3028fdb07 100644 --- a/docs-cms/analysis-summary.md +++ b/docs-cms/analysis-summary.md @@ -129,6 +129,7 @@ prism/ **Rationale**: Applications know their access patterns best; automate capacity planning **Example**: + ```protobuf message UserEvents { option (prism.access_pattern) = "append_heavy"; @@ -147,6 +148,7 @@ message UserEvents { **Rationale**: One definition generates Rust, Python, TypeScript, SQL schemas, deployment configs **Example**: + ```protobuf message UserProfile { string email = 2 [(prism.pii) = "email", (prism.encrypt_at_rest) = true]; @@ -163,6 +165,7 @@ message UserProfile { **Rationale**: Real backends catch real bugs; mocks give false confidence **Example**: + ```bash python -m tooling.test.local-stack up # Starts Postgres, Kafka, NATS cargo test --workspace # Tests against real databases diff --git a/docs-cms/memos/memo-002-admin-protocol-review.md b/docs-cms/memos/memo-002-admin-protocol-review.md index b436ca23b..ed94c635d 100644 --- a/docs-cms/memos/memo-002-admin-protocol-review.md +++ b/docs-cms/memos/memo-002-admin-protocol-review.md @@ -77,6 +77,7 @@ This memo now serves as a **historical record** of the security review process ( **Issues**: None critical **Recommendations**: + ```diff + Add JWT revocation checking (check against revocation list) + Add token binding to prevent token theft @@ -84,6 +85,7 @@ This memo now serves as a **historical record** of the security review process ( ``` **Improvement**: + ```rust pub struct JwtValidator { issuer: String, @@ -125,6 +127,7 @@ impl JwtValidator { 3. **Coarse-grained permissions**: Can't delegate specific operations **Improvement**: + ```protobuf // Add resource-level authorization to requests message CreateNamespaceRequest { @@ -144,6 +147,7 @@ message CreateNamespaceRequest { ``` **RBAC Policy Enhancement**: + ```yaml roles: namespace-admin: @@ -193,6 +197,7 @@ policies: 4. **No retention policy**: Logs could grow unbounded **Improvement**: + ```rust #[derive(Debug, Serialize)] pub struct AuditLogEntry { @@ -252,6 +257,7 @@ impl AuditLogger { ``` **Storage**: + ```sql CREATE TABLE admin_audit_log ( id UUID PRIMARY KEY, @@ -300,6 +306,7 @@ EXECUTE FUNCTION prevent_modification(); 3. **No per-operation limits**: Can spam expensive operations **Improvement**: + ```rust pub struct AdaptiveRateLimiter { // Different quotas for different operation types @@ -351,6 +358,7 @@ impl AdaptiveRateLimiter { 3. **No sanitization**: Potential for injection attacks in metadata **Improvement**: + ```protobuf message CreateNamespaceRequest { string name = 1 [ @@ -407,6 +415,7 @@ message CreateNamespaceRequest { ``` **Validation middleware**: + ```rust use validator::Validate; @@ -435,6 +444,7 @@ impl ValidationInterceptor { 3. **Feature flags**: No way to opt-in to new features **Improvement**: + ```protobuf // Package with explicit version package prism.admin.v2; @@ -473,6 +483,7 @@ service AdminService { 3. **Man-in-the-middle**: TLS protects transport, but not request integrity **Improvement**: + ```protobuf message RequestMetadata { string timestamp = 1; // ISO 8601 timestamp @@ -491,6 +502,7 @@ message CreateNamespaceRequest { ``` **Signature verification**: + ```rust pub struct SignatureVerifier { nonce_cache: Arc, // Redis-based cache @@ -539,6 +551,7 @@ impl SignatureVerifier { **Current**: Separate GetSession, DescribeSession, ListSessions **Simplified**: + ```protobuf message GetSessionsRequest { // Filters (all optional) @@ -571,6 +584,7 @@ service AdminService { **Current**: ListConfigs, GetConfig, CreateConfig, UpdateConfig, DeleteConfig **Simplified**: + ```protobuf service AdminService { // Read configs (supports filtering, pagination) @@ -589,6 +603,7 @@ service AdminService { **Current**: Inconsistent pagination across endpoints **Improved**: + ```protobuf // Standard pagination pattern for all list operations message PaginationRequest { @@ -623,6 +638,7 @@ message ListNamespacesResponse { ### 1. Batch Operations For automation and efficiency: + ```protobuf message BatchCreateNamespacesRequest { repeated CreateNamespaceRequest requests = 1 [ @@ -645,6 +661,7 @@ message BatchCreateNamespacesResponse { ### 2. Watch/Subscribe for Real-Time Updates For UI and automation: + ```protobuf message WatchNamespacesRequest { // Filters @@ -676,6 +693,7 @@ service AdminService { ### 3. Query Language for Complex Filters For advanced filtering: + ```protobuf message QueryRequest { // SQL-like or JMESPath query diff --git a/docs-cms/memos/memo-003-documentation-first-development.md b/docs-cms/memos/memo-003-documentation-first-development.md index c22388086..1fefe1449 100644 --- a/docs-cms/memos/memo-003-documentation-first-development.md +++ b/docs-cms/memos/memo-003-documentation-first-development.md @@ -45,6 +45,7 @@ docs/ β”œβ”€β”€ README.md # Wall of text β”œβ”€β”€ ARCHITECTURE.md # ASCII diagrams └── API.md # Code comments extracted + ```text **After** (micro-CMS with Docusaurus + GitHub Pages): @@ -146,6 +147,7 @@ async fn main() -> Result<()> { use prism_admin_client::AdminClient; // No syntax highlighting // No copy button // Hard to read + ```text vs @@ -168,6 +170,7 @@ Search returns: 3. ADR-012: Object Storage Integration Time to answer: 5 seconds (vs 5 minutes grepping) + ```text **Real Example**: When reviewing RFC-015 (Plugin Acceptance Tests), search for "authentication" immediately shows: @@ -258,6 +261,7 @@ User sees: - No mermaid rendering (just code blocks) - Hard to navigate (no sidebar) - Ugly monospace font + ```text **After** (GitHub Pages): @@ -331,6 +335,7 @@ We use three documentation types, each serving a distinct purpose: - Establishing patterns (OIDC authentication, client-originated config) **Format**: + ```markdown --- title: "ADR-XXX: Descriptive Title" @@ -361,6 +366,7 @@ tags: [category1, category2] - Specifying complex patterns (Layered Data Access, Distributed Reliability) **Format**: + ```markdown --- title: "RFC-XXX: Feature Name" @@ -399,6 +405,7 @@ tags: [feature-area, components] - Cross-cutting concerns that don't fit ADR/RFC format **Format**: + ```markdown --- title: "MEMO-XXX: Topic" @@ -425,6 +432,7 @@ tags: [category1, category2] ### Traditional Code-First Workflow (Previous Approach) Problem β†’ Prototype Code β†’ Review Code β†’ Fix Issues β†’ Document (Maybe) + ```text **Problems**: @@ -591,6 +599,7 @@ Problem β†’ Write RFC/ADR β†’ Review Design β†’ Implement β†’ Validate Against D - Builds documentation muscle memory **Example from CLAUDE.md**: + ```bash # THIS IS A BLOCKING REQUIREMENT - NEVER SKIP python3 tooling/validate_docs.py @@ -625,6 +634,7 @@ python3 tooling/validate_docs.py - Related decisions linked **Example**: RFC-010 revision history shows feedback integration: + ```markdown ## Revision History @@ -667,6 +677,7 @@ python3 tooling/validate_docs.py - Continuous improvement **Example Workflow**: + ```bash # Fast iteration cycle cd docusaurus && npm run start # Live preview @@ -724,6 +735,7 @@ git add . && git commit - All pages render **Usage**: + ```bash # Full validation (run before commit) python3 tooling/validate_docs.py diff --git a/docs-cms/memos/memo-004-backend-plugin-implementation-guide.md b/docs-cms/memos/memo-004-backend-plugin-implementation-guide.md index 2fce54c46..2fb1ca3b7 100644 --- a/docs-cms/memos/memo-004-backend-plugin-implementation-guide.md +++ b/docs-cms/memos/memo-004-backend-plugin-implementation-guide.md @@ -58,6 +58,7 @@ Comprehensive comparison of all backends discussed for Prism, prioritized by int - **Reference implementation**: Demonstrates plugin interface patterns **Go Implementation:** + ```go // plugins/memstore/store.go package memstore @@ -119,6 +120,7 @@ func (m *MemStore) Delete(ctx context.Context, key string) error { - PubSub (can add channels with Go channels) **Testing Strategy:** + ```go func TestMemStore(t *testing.T) { store := NewMemStore() @@ -175,6 +177,7 @@ func TestMemStore(t *testing.T) { - **Excellent Go SDK**: `go-redis/redis` is mature, well-documented, idiomatic Go **Go SDK:** + ```go import "github.com/redis/go-redis/v9" @@ -190,6 +193,7 @@ client := redis.NewClient(&redis.Options{ - Lists, Sets, Sorted Sets **Testing Strategy:** + ```go // testcontainers integration func NewRedisInstance(t *testing.T) *RedisInstance { @@ -224,6 +228,7 @@ func NewRedisInstance(t *testing.T) *RedisInstance { - **Complex data models**: Supports JSON, arrays, full-text search **Go SDK:** + ```go import "github.com/jackc/pgx/v5" @@ -237,6 +242,7 @@ conn, _ := pgx.Connect(context.Background(), "postgres://user:pass@localhost:543 - Time-series (with extensions like TimescaleDB) **Testing Strategy:** + ```go func NewPostgresInstance(t *testing.T) *PostgresInstance { req := testcontainers.ContainerRequest{ @@ -275,6 +281,7 @@ func NewPostgresInstance(t *testing.T) *PostgresInstance { - **Same SQL as Postgres**: Easy to understand **Go SDK:** + ```go import "github.com/mattn/go-sqlite3" @@ -287,6 +294,7 @@ db, _ := sql.Open("sqlite3", ":memory:") // In-memory DB - Full-text search (FTS5) **Testing Strategy:** + ```go func NewSQLiteInstance(t *testing.T) *SQLiteInstance { // No container needed! @@ -328,6 +336,7 @@ func NewSQLiteInstance(t *testing.T) *SQLiteInstance { - **Simple protocol**: Text-based, easy to debug **Go SDK:** + ```go import "github.com/nats-io/nats.go" @@ -341,6 +350,7 @@ nc, _ := nats.Connect("nats://localhost:4222") - Key-Value store (JetStream KV) **Testing Strategy:** + ```go func NewNATSInstance(t *testing.T) *NATSInstance { // Option 1: Embedded NATS server (no container!) @@ -381,6 +391,7 @@ func NewNATSInstance(t *testing.T) *NATSInstance { - **Testable**: testcontainers support, but slow startup **Go SDK:** + ```go // Option 1: segmentio/kafka-go (pure Go) import "github.com/segmentio/kafka-go" @@ -405,6 +416,7 @@ producer, _ := kafka.NewProducer(&kafka.ConfigMap{ - Stream processing (Kafka Streams) **Testing Strategy:** + ```go func NewKafkaInstance(t *testing.T) *KafkaInstance { req := testcontainers.ContainerRequest{ @@ -450,6 +462,7 @@ func NewKafkaInstance(t *testing.T) *KafkaInstance { - **Excellent SDKs**: AWS SDK v2 and MinIO Go client **Go SDK:** + ```go // AWS S3 import "github.com/aws/aws-sdk-go-v2/service/s3" @@ -471,6 +484,7 @@ minioClient, _ := minio.New("localhost:9000", &minio.Options{ - Lifecycle policies (auto-archival) **Testing Strategy:** + ```go func NewMinIOInstance(t *testing.T) *MinIOInstance { req := testcontainers.ContainerRequest{ @@ -515,6 +529,7 @@ func NewMinIOInstance(t *testing.T) *MinIOInstance { - **Testable**: testcontainers support **Go SDK:** + ```go import "github.com/ClickHouse/clickhouse-go/v2" @@ -534,6 +549,7 @@ conn, _ := clickhouse.Open(&clickhouse.Options{ - Event logs (append-only) **Testing Strategy:** + ```go func NewClickHouseInstance(t *testing.T) *ClickHouseInstance { req := testcontainers.ContainerRequest{ @@ -572,6 +588,7 @@ func NewClickHouseInstance(t *testing.T) *ClickHouseInstance { - **Expensive to test**: AWS charges, no free tier for Neptune **Go SDK:** + ```go import "github.com/apache/tinkerpop/gremlin-go/v3/driver" @@ -585,6 +602,7 @@ g := gremlingo.Traversal_().WithRemote(remote) - RDF triples (SPARQL) **Testing Strategy:** + ```go // Problem: No good local testing option // Option 1: Mock Gremlin responses (not ideal) @@ -724,6 +742,7 @@ backend: ``` **Client Code:** + ```go // Demo: Instant GET/SET operations client.Set("session:abc123", sessionData, 5*time.Minute) @@ -767,6 +786,7 @@ backend: ``` **Client Code:** + ```go // Demo: GET/SET operations client.Set("user:123", userData, 300*time.Second) // 5 min TTL @@ -799,6 +819,7 @@ backend: ``` **Client Code:** + ```go // Demo: Outbox pattern tx := client.BeginTx() @@ -835,6 +856,7 @@ backend: ``` **Client Code:** + ```go // Demo: Producer for i := 0; i < 10000; i++ { @@ -876,6 +898,7 @@ backend: ``` **Client Code:** + ```go // Demo: Transparent large payload handling video := loadVideo("movie.mp4") // 2GB @@ -915,6 +938,7 @@ backends: ``` **Client Code:** + ```go // Demo: All patterns composed automatically with client.transaction() as tx: @@ -999,6 +1023,7 @@ services: ``` **Usage:** + ```bash # Start all backends docker-compose -f docker-compose.test.yml up -d @@ -1149,6 +1174,7 @@ Applications (client code) **Implemented**: `patterns/core/interfaces.go` - Declarative metadata for pattern slot matching **45 Backend Interface Constants**: + ```go // KeyValue interfaces (6) InterfaceKeyValueBasic // Set, Get, Delete, Exists @@ -1176,6 +1202,7 @@ InterfaceStreamPartitioning // GetPartitions, AppendToPartition ``` **Driver Metadata Example** (MemStore): + ```go func (m *MemStore) Metadata() *core.BackendInterfaceMetadata { return &core.BackendInterfaceMetadata{ @@ -1192,6 +1219,7 @@ func (m *MemStore) Metadata() *core.BackendInterfaceMetadata { ``` **Driver Metadata Example** (Redis - 24 interfaces!): + ```go func (r *RedisPattern) Metadata() *core.BackendInterfaceMetadata { return &core.BackendInterfaceMetadata{ @@ -1230,6 +1258,7 @@ func (r *RedisPattern) Metadata() *core.BackendInterfaceMetadata { ``` **Usage**: At configuration time, patterns can query driver metadata to match requirements: + ```go // Pattern requires these interfaces for a slot required := []core.BackendInterface{ @@ -1250,6 +1279,7 @@ if metadata.ImplementsAll(required) { **Implemented**: `patterns/core/serve.go` - `ServeBackendDriver()` function **Before** (65 lines of boilerplate in every driver): + ```go func main() { configPath := flag.String("config", "config.yaml", ...) @@ -1277,6 +1307,7 @@ func main() { ``` **After** (25 lines with SDK pattern): + ```go func main() { core.ServeBackendDriver(func() core.Plugin { @@ -1305,6 +1336,7 @@ func main() { **Implemented**: `tests/acceptance/interfaces/` - Test interfaces across multiple backends **Structure**: + ```go // tests/acceptance/interfaces/keyvalue_basic_test.go diff --git a/docs-cms/memos/memo-005-client-protocol-design-philosophy.md b/docs-cms/memos/memo-005-client-protocol-design-philosophy.md index 8f203bc53..03be10f22 100644 --- a/docs-cms/memos/memo-005-client-protocol-design-philosophy.md +++ b/docs-cms/memos/memo-005-client-protocol-design-philosophy.md @@ -29,6 +29,7 @@ Resolve the architectural tension between: ### RFC-014: Layered Data Access Patterns (Composable Approach) **Defines 6 generic patterns**: + ```protobuf service KeyValueService { rpc Set(SetRequest) returns (SetResponse); @@ -44,6 +45,7 @@ service PubSubService { ``` **Application must compose**: + ```python # IoT device management - application composes primitives await client.keyvalue.set(f"device:{id}", metadata) # Registry @@ -66,6 +68,7 @@ await client.pubsub.publish("commands", message) # Broadcast ### RFC-017: Multicast Registry (Use-Case-Specific Approach) **Defines purpose-built API**: + ```protobuf service MulticastRegistryService { rpc Register(RegisterRequest) returns (RegisterResponse); @@ -75,6 +78,7 @@ service MulticastRegistryService { ``` **Application uses clear semantics**: + ```python # IoT device management - clear intent await client.registry.register( @@ -117,6 +121,7 @@ result = await client.registry.multicast( - ❌ Avoid forcing developers to compose primitives manually **Example**: + ```python # BAD: Application must coordinate registry + pub/sub devices = await client.keyvalue.scan("device:*") @@ -140,6 +145,7 @@ await client.registry.multicast(filter={}, message=message) - ❌ Avoid generic terms requiring mental mapping (e.g., "put into keyvalue to register") **Example**: + ```protobuf // CLEAR: Purpose obvious from name rpc Register(RegisterRequest) returns (RegisterResponse); @@ -159,6 +165,7 @@ rpc Set(SetRequest) returns (SetResponse); - ❌ Changing generic primitive affects many use cases **Example**: + ```protobuf // Adding feature to MulticastRegistry: localized impact message RegisterRequest { @@ -188,6 +195,7 @@ message SetRequest { **Example**: Proxy with 6 generic services: ~10k LOC, 50MB binary Proxy with 20 use-case services: ~40k LOC, 150MB binary + ```text ## Proposed Solution: Layered API Architecture @@ -198,12 +206,14 @@ Proxy with 20 use-case services: ~40k LOC, 150MB binary **Six core primitives** (RFC-014): ``` + service KeyValueService { ... } service PubSubService { ... } service QueueService { ... } service TimeSeriesService { ... } service GraphService { ... } service TransactionalService { ... } + ```text **Characteristics**: @@ -221,11 +231,13 @@ service TransactionalService { ... } **Purpose-built patterns** (RFC-017, plus more): ``` + service MulticastRegistryService { ... } // IoT, presence, service discovery service SagaService { ... } // Distributed transactions service EventSourcingService { ... } // Audit trails, event log service WorkQueueService { ... } // Background jobs service CacheAsideService { ... } // Read-through cache + ```text **Characteristics**: @@ -280,6 +292,7 @@ service CacheAsideService { ... } // Read-through cache - βœ… Community can contribute patterns (not just core team) **Configuration**: + ```yaml namespaces: - name: iot-devices @@ -444,6 +457,7 @@ SagaService (Layer 2) KeyValueService (Layer 1) ↑ used by MulticastRegistryService (Layer 2) + ```text **Rationale**: Keeps patterns loosely coupled, evolution independent. @@ -453,7 +467,9 @@ MulticastRegistryService (Layer 2) **Proposal**: Pattern coordinators are plugins with semantic versioning: ``` + coordinator_plugin: prism-multicast-registry:v1.2.0 + ```text **Migration path**: diff --git a/docs-cms/memos/memo-006-backend-interface-decomposition-schema-registry.md b/docs-cms/memos/memo-006-backend-interface-decomposition-schema-registry.md index f310025d9..e0c7327a6 100644 --- a/docs-cms/memos/memo-006-backend-interface-decomposition-schema-registry.md +++ b/docs-cms/memos/memo-006-backend-interface-decomposition-schema-registry.md @@ -529,6 +529,7 @@ implementation: **Example Configuration** (using the pattern): ``` + namespaces: - name: iot-devices pattern: multicast-registry @@ -651,6 +652,7 @@ proto/ ### 1. **Explicit Capability Mapping** ``` + # Before (ambiguous) backend: redis @@ -696,6 +698,7 @@ config = generate_namespace_config( ### 3. **Backend Substitutability** ``` + # Development (fast local testing) slots: registry: @@ -762,6 +765,7 @@ slots: {registry: dynamodb, messaging: sns} ## Example: Full Configuration Generation Flow ``` + # 1. List available patterns $ prismctl registry patterns list multicast-registry v1 Register identities and multicast to subsets @@ -835,6 +839,7 @@ namespaces: 4. Connection strings must match backend's expected format **Example Validation**: + ```bash $ prismctl validate namespace-config.yaml diff --git a/docs-cms/memos/memo-007-podman-scratch-container-demo.md b/docs-cms/memos/memo-007-podman-scratch-container-demo.md index dfe1f7205..f772ff730 100644 --- a/docs-cms/memos/memo-007-podman-scratch-container-demo.md +++ b/docs-cms/memos/memo-007-podman-scratch-container-demo.md @@ -204,6 +204,7 @@ CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] ### Install Podman **Linux**: + ```bash # Debian/Ubuntu sudo apt-get install podman @@ -213,6 +214,7 @@ sudo dnf install podman ``` **macOS**: + ```bash # Install via Homebrew brew install podman @@ -223,6 +225,7 @@ podman machine start ``` **Windows**: + ```bash # Install via winget winget install RedHat.Podman @@ -311,6 +314,7 @@ spec: ``` **Run pod**: + ```bash # Create pod from YAML podman play kube prism-pod.yaml @@ -355,6 +359,7 @@ echo "=== Reduction: $(echo "scale=1; ($(podman inspect prism-proxy:regular --fo ``` **Expected output**: + ```text Regular image: 127MB Scratch image: 6MB @@ -437,6 +442,7 @@ podman build --squash -t prism-proxy:dev -f proxy/Containerfile . \ ### Fast Iteration Tips 1. **Use Layer Caching**: + ```dockerfile # Copy dependencies first (changes less often) COPY Cargo.toml Cargo.lock ./ @@ -448,6 +454,7 @@ podman build --squash -t prism-proxy:dev -f proxy/Containerfile . \ ``` 2. **Build in Parallel**: + ```bash # Build multiple images concurrently podman build -t prism-proxy:scratch -f proxy/Containerfile . & @@ -456,6 +463,7 @@ podman build --squash -t prism-proxy:dev -f proxy/Containerfile . \ ``` 3. **Skip Tests During Dev Build**: + ```bash # Fast build (no tests) podman build --build-arg SKIP_TESTS=true -t prism-proxy:dev . @@ -521,6 +529,7 @@ ps aux | grep prism-proxy - No VM overhead per-container (all share same VM) **Optimization**: + ```bash # Pre-start Podman machine podman machine start @@ -601,6 +610,7 @@ kubectl apply -f prism-k8s.yaml - ❌ **Static linking required**: Must compile with musl (Rust) or CGO_ENABLED=0 (Go) **Mitigation**: + ```bash # Use debug variant for troubleshooting FROM scratch AS release diff --git a/docs-cms/memos/memo-008-vault-token-exchange-flow.md b/docs-cms/memos/memo-008-vault-token-exchange-flow.md index e68f0b571..ff3fb0067 100644 --- a/docs-cms/memos/memo-008-vault-token-exchange-flow.md +++ b/docs-cms/memos/memo-008-vault-token-exchange-flow.md @@ -1187,6 +1187,7 @@ path "database/creds/redis-role" { ``` Apply policy: + ```bash vault policy write prism-redis prism-redis.hcl ``` diff --git a/docs-cms/memos/memo-009-topaz-local-authorizer-configuration.md b/docs-cms/memos/memo-009-topaz-local-authorizer-configuration.md index 69d66efd8..e6a793c53 100644 --- a/docs-cms/memos/memo-009-topaz-local-authorizer-configuration.md +++ b/docs-cms/memos/memo-009-topaz-local-authorizer-configuration.md @@ -762,6 +762,7 @@ authorizer: **Symptom**: `docker compose up` fails with connection refused **Diagnosis**: + ```bash # Check Topaz logs docker compose logs topaz @@ -773,6 +774,7 @@ docker compose logs topaz ``` **Solution**: + ```bash # Check port availability lsof -i :8282 @@ -791,6 +793,7 @@ docker run --rm -v $(pwd)/topaz/policies:/policies \ **Symptom**: `bootstrap.sh` exits with "Topaz not ready" **Diagnosis**: + ```bash # Check if Topaz is listening curl -v http://localhost:8383/health @@ -800,6 +803,7 @@ docker compose logs topaz | grep -i error ``` **Solution**: + ```bash # Increase wait time in bootstrap script until curl -s "$TOPAZ_REST/health" > /dev/null; do @@ -816,6 +820,7 @@ curl -f http://localhost:8383/api/v2/directory/objects || exit 1 **Symptom**: All authorization checks return `allowed: false` **Diagnosis**: + ```bash # Check directory state curl http://localhost:8383/api/v2/directory/objects | jq . @@ -830,6 +835,7 @@ curl -X POST http://localhost:8282/api/v2/authz/is \ ``` **Solution**: + ```bash # Re-run bootstrap bash topaz/seed/bootstrap.sh @@ -852,6 +858,7 @@ docker run --rm -v $(pwd)/topaz/policies:/policies \ **Symptom**: Modified policies don't take effect **Solution**: + ```bash # Topaz should auto-reload, but force reload: docker compose restart topaz diff --git a/docs-cms/memos/memo-010-poc1-edge-case-analysis.md b/docs-cms/memos/memo-010-poc1-edge-case-analysis.md index b4993031b..770510e74 100644 --- a/docs-cms/memos/memo-010-poc1-edge-case-analysis.md +++ b/docs-cms/memos/memo-010-poc1-edge-case-analysis.md @@ -55,6 +55,7 @@ Without thorough edge case testing, these scenarios could cause cascading failur **Test**: `test_pattern_spawn_failure_updates_status` **Implementation**: + ```rust // Pattern status transitions to Failed on spawn error pattern.status = PatternStatus::Failed(format!("Spawn failed: {}", e)); @@ -68,6 +69,7 @@ pattern.status = PatternStatus::Failed(format!("Spawn failed: {}", e)); **Test**: `test_health_check_on_uninitialized_pattern` **Implementation**: + ```rust // Return current status without gRPC call if not running if !pattern.is_running() { @@ -83,6 +85,7 @@ if !pattern.is_running() { **Test**: `test_stop_pattern_that_never_started` **Implementation**: + ```rust // Graceful stop even if no process running if let Some(mut process) = self.process.take() { @@ -98,6 +101,7 @@ if let Some(mut process) = self.process.take() { **Test**: `test_multiple_start_attempts` **Implementation**: + ```rust // Each attempt updates status independently pattern.status = PatternStatus::Failed(...); @@ -111,6 +115,7 @@ pattern.status = PatternStatus::Failed(...); **Scenario**: gRPC server not immediately ready after process spawn **Implementation**: + ```rust // 5 attempts with exponential backoff: 100ms, 200ms, 400ms, 800ms, 1600ms let max_attempts = 5; @@ -149,6 +154,7 @@ loop { **Test**: `test_health_check_timeout_handling` **Implementation**: + ```rust use tokio::time::timeout; @@ -168,6 +174,7 @@ let result = timeout( **Test**: `test_concurrent_pattern_registration` **Implementation**: + ```rust // RwLock allows safe concurrent writes patterns: Arc>> @@ -238,6 +245,7 @@ patterns: Arc>> **Test**: `test_pattern_manager_is_send_and_sync` **Implementation**: + ```rust fn assert_send() {} fn assert_sync() {} @@ -292,6 +300,7 @@ The following tests are marked as `#[ignore]` and require actual pattern binarie ### 1. Connection Retry with Exponential Backoff **Before**: + ```rust // Fixed 1.5s sleep, no retry sleep(Duration::from_millis(1500)).await; @@ -299,6 +308,7 @@ let client = PatternClient::connect(endpoint).await?; ``` **After**: + ```rust // Exponential backoff: 100ms β†’ 200ms β†’ 400ms β†’ 800ms β†’ 1600ms let mut delay = Duration::from_millis(100); @@ -339,6 +349,7 @@ for attempt in 1..=5 { - Connection failure reasons **Example**: + ```text WARN pattern=redis attempt=2 next_delay_ms=200 error="connection refused" gRPC connection attempt failed, retrying INFO pattern=redis attempts=3 gRPC client connected successfully diff --git a/docs-cms/memos/memo-011-error-handling-best-practices.md b/docs-cms/memos/memo-011-error-handling-best-practices.md index 7c3324b99..81bc1f9b1 100644 --- a/docs-cms/memos/memo-011-error-handling-best-practices.md +++ b/docs-cms/memos/memo-011-error-handling-best-practices.md @@ -44,6 +44,7 @@ Prism's `Error` message captures distributed systems best practices from: ### 1. Simple by Default, Rich When Needed **Basic error** (minimal fields): + ```protobuf Error { code: ERROR_CODE_NOT_FOUND @@ -53,6 +54,7 @@ Error { ``` **Rich error** (with full context): + ```protobuf Error { code: ERROR_CODE_BACKEND_ERROR @@ -152,6 +154,7 @@ RetryPolicy { ``` **Non-retryable errors**: + ```protobuf RetryPolicy { retryable: false @@ -165,6 +168,7 @@ RetryPolicy { Use `ErrorDetail` oneof for type-safe error information: #### FieldViolation (validation errors) + ```protobuf field_violation: { field: "ttl_seconds" @@ -175,6 +179,7 @@ field_violation: { ``` #### BackendError (backend-specific context) + ```protobuf backend_error: { backend_type: "kafka" @@ -186,6 +191,7 @@ backend_error: { ``` #### PatternError (pattern-level semantics) + ```protobuf pattern_error: { pattern_type: "keyvalue" @@ -196,6 +202,7 @@ pattern_error: { ``` #### QuotaViolation (rate limiting) + ```protobuf quota_violation: { dimension: "requests_per_second" @@ -206,6 +213,7 @@ quota_violation: { ``` #### PreconditionFailure (CAS, version conflicts) + ```protobuf precondition_failure: { type: "ETAG_MISMATCH" @@ -231,6 +239,7 @@ precondition_failure: { - `CONCURRENCY_ERROR` - Concurrent modification conflict **Prometheus metrics**: + ```text prism_errors_total{category="backend_error", backend="redis", code="503"} 42 prism_errors_total{category="timeout_error", pattern="keyvalue", code="504"} 12 @@ -247,21 +256,25 @@ prism_errors_total{category="rate_limit_error", namespace="prod", code="429"} 15 ### 7. Traceability and Correlation **Request ID**: Correlate errors across services + ```protobuf request_id: "req-abc123-def456" ``` **Source**: Which service generated the error + ```protobuf source: "prism-proxy-pod-3" ``` **Timestamp**: When the error occurred + ```protobuf timestamp: { seconds: 1696867200 } ``` **Debug Info** (development only): + ```protobuf debug_info: { trace_id: "abc123def456" diff --git a/docs-cms/memos/memo-012-developer-experience.md b/docs-cms/memos/memo-012-developer-experience.md index 2df97a736..a94b08503 100644 --- a/docs-cms/memos/memo-012-developer-experience.md +++ b/docs-cms/memos/memo-012-developer-experience.md @@ -256,6 +256,7 @@ git commit -m "Add MEMO-XXX documenting " ### Frontmatter Templates **ADR**: + ```yaml --- title: "ADR-XXX: Title" @@ -268,6 +269,7 @@ id: adr-xxx ``` **RFC**: + ```yaml --- title: "RFC-XXX: Title" @@ -281,6 +283,7 @@ id: rfc-xxx ``` **MEMO**: + ```yaml --- title: "MEMO-XXX: Title" diff --git a/docs-cms/memos/memo-013-poc1-infrastructure-analysis.md b/docs-cms/memos/memo-013-poc1-infrastructure-analysis.md index a87156876..9d6072e08 100644 --- a/docs-cms/memos/memo-013-poc1-infrastructure-analysis.md +++ b/docs-cms/memos/memo-013-poc1-infrastructure-analysis.md @@ -121,18 +121,22 @@ Current `prism-loadtest` tool: Prism requires **two distinct testing levels**: **Pattern-Level Testing** (Unit Load Testing): + ```text prism-loadtest β†’ Coordinator (direct) β†’ Redis/NATS ``` + - **Purpose**: Test pattern logic in isolation - **Speed**: Fastest (no gRPC overhead) - **Metrics**: Custom (multicast delivery) - **Use Case**: Development, optimization **Integration-Level Testing** (End-to-End): + ```text ghz β†’ Rust Proxy (gRPC) β†’ Pattern (gRPC) β†’ Redis/NATS ``` + - **Purpose**: Test production path - **Speed**: Realistic (includes gRPC) - **Metrics**: Standard (gRPC) @@ -306,6 +310,7 @@ Refactor existing plugins to use new SDK packages: ### Developer Experience **Before** (current state): + ```go // Redis plugin: ~700 LOC // - 150 LOC connection pooling @@ -315,6 +320,7 @@ Refactor existing plugins to use new SDK packages: ``` **After** (with SDK): + ```go // Redis plugin: ~450 LOC // - 10 LOC pool setup (use sdk.Pool) diff --git a/docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md b/docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md index 7b354f89a..8bbe0d71a 100644 --- a/docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md +++ b/docs-cms/memos/memo-014-pattern-sdk-shared-complexity.md @@ -205,6 +205,7 @@ func (p *Pool) checkHealth() { ``` **Usage in Redis Plugin**: + ```go // plugins/redis/client/pool.go package client @@ -431,6 +432,7 @@ func (m *Manager) processExpiries() { ``` **Usage in MemStore Plugin**: + ```go // plugins/memstore/storage/keyvalue.go package storage @@ -671,6 +673,7 @@ func (c *Checker) runChecks() { ``` **Usage in Redis Plugin**: + ```go // plugins/redis/main.go package main @@ -866,6 +869,7 @@ func (l *Loader) Required(key string) string { ``` **Usage**: + ```go // plugins/redis/main.go func loadConfig() RedisConfig { @@ -1171,6 +1175,7 @@ coverage-sdk-enforce: ### Developer Experience **Before** (without SDK enhancements): + ```go // plugins/redis/client/pool.go - ~150 lines of custom pooling // plugins/redis/client/health.go - ~50 lines of custom health checks @@ -1179,6 +1184,7 @@ coverage-sdk-enforce: ``` **After** (with SDK enhancements): + ```go // plugins/redis/main.go - ~30 lines using SDK packages pool := pool.NewPool(factory, poolConfig) diff --git a/docs-cms/memos/memo-016-observability-lifecycle-implementation.md b/docs-cms/memos/memo-016-observability-lifecycle-implementation.md index aa4bde999..e3dcb1e24 100644 --- a/docs-cms/memos/memo-016-observability-lifecycle-implementation.md +++ b/docs-cms/memos/memo-016-observability-lifecycle-implementation.md @@ -47,6 +47,7 @@ Comprehensive observability manager implementing: - Metrics endpoint: `GET /metrics` β†’ Prometheus text format **Stub Metrics Exposed:** + ```prometheus # Backend driver information backend_driver_info{name="memstore",version="0.1.0"} 1 @@ -62,6 +63,7 @@ backend_driver_uptime_seconds 123.45 - `backend_driver_connections_active` - Active connection gauge **Configuration:** + ```go type ObservabilityConfig struct { ServiceName string // e.g., "memstore", "redis" @@ -73,6 +75,7 @@ type ObservabilityConfig struct { ``` **Lifecycle Management:** + ```go // Initialize observability components observability := NewObservabilityManager(config) @@ -89,6 +92,7 @@ observability.Shutdown(ctx) #### `patterns/core/serve.go` (Enhanced) **New Command-Line Flags:** + ```bash --metrics-port # Prometheus metrics port (0 to disable) --enable-tracing # Enable OpenTelemetry tracing @@ -96,6 +100,7 @@ observability.Shutdown(ctx) ``` **Enhanced ServeOptions:** + ```go type ServeOptions struct { DefaultName string @@ -109,6 +114,7 @@ type ServeOptions struct { ``` **Automatic Initialization:** + ```go // Observability is automatically initialized in ServeBackendDriver // Before plugin lifecycle starts: @@ -126,6 +132,7 @@ slog.Info("bootstrapping backend driver", #### `patterns/core/go.mod` (Updated) **New Dependencies:** + ```go require ( go.opentelemetry.io/otel v1.24.0 @@ -140,6 +147,7 @@ require ( **Location**: `patterns/core/plugin.go:BootstrapWithConfig()` **Existing Implementation:** + ```go // Wait for shutdown signal sigChan := make(chan os.Signal, 1) @@ -173,6 +181,7 @@ controlPlane.Stop(ctx) // Stop control plane ### Usage Example **Backend Driver Main (e.g., `drivers/memstore/cmd/memstore/main.go`):** + ```go func main() { core.ServeBackendDriver(func() core.Plugin { @@ -190,6 +199,7 @@ func main() { ``` **Running with Observability:** + ```bash # Development mode (stdout tracing, metrics on port 9091) ./memstore --debug --metrics-port 9091 --enable-tracing @@ -202,6 +212,7 @@ func main() { ``` **Accessing Metrics:** + ```bash # Health check curl http://localhost:9091/health @@ -232,6 +243,7 @@ Comprehensive integration tests validating proxy-to-pattern communication. **Test**: `TestProxyPatternLifecycle` **Flow**: + ```text Step 1: Start backend driver (memstore) with control plane ↓ @@ -261,6 +273,7 @@ Step 9: Verify graceful shutdown - βœ… Graceful shutdown completes **Code Excerpt:** + ```go // Proxy sends Initialize initResp, err := client.Initialize(ctx, &pb.InitializeRequest{ @@ -295,6 +308,7 @@ assert.Equal(t, pb.HealthStatus_HEALTH_STATUS_HEALTHY, healthResp.Status) 4. Proxy validates debug info received **Debug Info Structure:** + ```go healthResp := &pb.HealthCheckResponse{ Status: pb.HealthStatus_HEALTH_STATUS_HEALTHY, @@ -331,6 +345,7 @@ healthResp := &pb.HealthCheckResponse{ **Purpose**: Get dynamically allocated port after control plane starts. **Usage:** + ```go controlPlane := core.NewControlPlaneServer(driver, 0) // 0 = dynamic port controlPlane.Start(ctx) @@ -340,6 +355,7 @@ fmt.Printf("Control plane listening on port: %d\n", port) ``` **Implementation:** + ```go func (s *ControlPlaneServer) Port() int { if s.listener != nil { @@ -360,6 +376,7 @@ func (s *ControlPlaneServer) Port() int { Go module for integration tests with proper replace directives. **Content:** + ```go module github.com/jrepp/prism-data-layer/tests/integration @@ -392,6 +409,7 @@ go test -timeout 30s -v ./... ``` **Expected Output:** + ```text === RUN TestProxyPatternLifecycle lifecycle_test.go:33: Step 1: Starting backend driver (memstore) @@ -428,6 +446,7 @@ go test -timeout 30s -v ./... ### 2. Zero-Boilerplate Backend Drivers **Before** (drivers/memstore/cmd/memstore/main.go - 65 lines): + ```go func main() { configPath := flag.String("config", "config.yaml", ...) @@ -438,6 +457,7 @@ func main() { ``` **After** (drivers/memstore/cmd/memstore/main.go - 25 lines): + ```go func main() { core.ServeBackendDriver(func() core.Plugin { @@ -472,6 +492,7 @@ func main() { ### 4. Production-Ready Deployment **Kubernetes Deployment Example:** + ```yaml apiVersion: v1 kind: Service @@ -529,6 +550,7 @@ spec: ### Compile-Time Validation **Observability Module:** + ```bash cd patterns/core go build -o /dev/null observability.go serve.go plugin.go config.go controlplane.go lifecycle_service.go @@ -536,6 +558,7 @@ go build -o /dev/null observability.go serve.go plugin.go config.go controlplane ``` **Integration Tests:** + ```bash cd tests/integration go test -c @@ -545,6 +568,7 @@ go test -c ### Runtime Validation (Manual) **Test Observability Endpoints:** + ```bash # Terminal 1: Start memstore with observability cd drivers/memstore/cmd/memstore @@ -562,6 +586,7 @@ curl http://localhost:9091/metrics ``` **Test Integration:** + ```bash cd tests/integration go test -v -run TestProxyPatternLifecycle @@ -575,10 +600,12 @@ go test -v -run TestProxyPatternLifecycle ### Immediate (Optional) 1. **Run Integration Tests End-to-End** + ```bash cd tests/integration go test -v ./... ``` + - May require fixing proto dependency issues - Tests should pass with proper module setup diff --git a/docs-cms/memos/memo-017-message-schema-configuration.md b/docs-cms/memos/memo-017-message-schema-configuration.md index 96a583027..1a64fd09f 100644 --- a/docs-cms/memos/memo-017-message-schema-configuration.md +++ b/docs-cms/memos/memo-017-message-schema-configuration.md @@ -154,6 +154,7 @@ message_schema: Consumers can discover message schemas via: #### 1. **Admin API** + ```bash GET /api/patterns/device-notifications/schema @@ -178,6 +179,7 @@ Response: ### Example: End-to-End Flow **1. Operator configures pattern with schema:** + ```yaml pattern: multicast-registry name: iot-telemetry @@ -192,6 +194,7 @@ slots: ``` **2. Consumer queries schema:** + ```bash $ prism-cli pattern schema iot-telemetry @@ -208,6 +211,7 @@ message TelemetryEvent { ``` **3. Consumer generates client code:** + ```bash $ buf generate https://schemas.prism.internal/iot/v1/telemetry.proto @@ -215,6 +219,7 @@ Generated: iot/v1/telemetry_pb2.py ``` **4. Consumer subscribes with typed handler:** + ```python from iot.v1 import telemetry_pb2 @@ -225,6 +230,7 @@ client.subscribe("iot-telemetry", handle_telemetry) ``` **5. Publisher sends validated message:** + ```python event = telemetry_pb2.TelemetryEvent( device_id="sensor-42", diff --git a/docs-cms/memos/memo-019-loadtest-results-100rps.md b/docs-cms/memos/memo-019-loadtest-results-100rps.md index 5acf30b27..ec8811297 100644 --- a/docs-cms/memos/memo-019-loadtest-results-100rps.md +++ b/docs-cms/memos/memo-019-loadtest-results-100rps.md @@ -210,9 +210,11 @@ Goroutine fan-out with 1,463 targets creates: - **Expected improvement**: 10-50x latency reduction 2. **Add semaphore-based concurrency limit**: + ```go sem := make(chan struct{}, 100) // Max 100 concurrent publishes ``` + - Prevents goroutine explosion - Smooths network traffic - **Expected improvement**: 50% latency reduction, 99% delivery rate diff --git a/docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md b/docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md index 20d3aa140..0d3090feb 100644 --- a/docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md +++ b/docs-cms/memos/memo-020-parallel-testing-and-build-hygiene.md @@ -69,21 +69,25 @@ class ParallelTestRunner: **Key Features:** 1. **Dependency Management** + ```python # Wait for dependencies using asyncio.Event for dep in suite.depends_on: await self.completion_events[dep].wait() ``` + - Integration tests wait for memstore-unit to complete - Ensures test ordering correctness 2. **Parallel Groups** + ```python # Serialize tests that conflict on resources if suite.parallel_group == "acceptance": async with self.parallel_groups[suite.parallel_group]: await self._execute_suite(suite) ``` + - Acceptance tests use Docker containers with conflicting ports - Tests within group run serially, but parallel to other groups @@ -196,6 +200,7 @@ coverage-memstore: **Benefits:** 1. **Single Cleanup Command** + ```bash make clean-build # Removes entire ./build directory ``` @@ -286,6 +291,7 @@ coverage-memstore: ### Test Execution Performance **Before:** + ```text Sequential Execution: Unit: 60s (5 test suites) @@ -297,6 +303,7 @@ Sequential Execution: ``` **After:** + ```text Parallel Execution (max_parallel=8): Unit: 2s (all 5 in parallel) @@ -310,6 +317,7 @@ Parallel Execution (max_parallel=8): ``` **Validation:** + ```bash $ make test-parallel πŸš€ Prism Parallel Test Runner @@ -333,6 +341,7 @@ $ make test-parallel ### Build Hygiene Impact **Before:** + ```bash $ find . -name "coverage.out" -o -name "coverage.html" | wc -l 16 # Scattered across patterns/ and tests/ @@ -342,6 +351,7 @@ $ du -sh proxy/target/ ``` **After:** + ```bash $ tree -L 3 build/ build/ diff --git a/docs-cms/memos/memo-022-prismctl-integration-testing.md b/docs-cms/memos/memo-022-prismctl-integration-testing.md index c6ed2ab72..0d493081a 100644 --- a/docs-cms/memos/memo-022-prismctl-integration-testing.md +++ b/docs-cms/memos/memo-022-prismctl-integration-testing.md @@ -28,6 +28,7 @@ Prismctl's authentication system (`cli/prismctl/auth.py`) has **40% code coverag - OIDC endpoint discovery **Current State**: + ```text βœ… Unit tests: 6/6 passing βœ… Token storage: Secure (600 permissions) @@ -47,6 +48,7 @@ Prismctl's authentication system (`cli/prismctl/auth.py`) has **40% code coverag ### Test Infrastructure **Local Dex Server** (from [RFC-016](/rfc/rfc-016)): + ```yaml # tests/integration/docker-compose.dex.yml services: @@ -65,6 +67,7 @@ services: ``` **Dex Test Configuration**: + ```yaml # tests/integration/dex-config.yaml issuer: http://localhost:5556/dex @@ -104,6 +107,7 @@ staticPasswords: #### 1. **Device Code Flow** (Priority: HIGH) **Test**: `test_device_code_flow_success` + ```python def test_device_code_flow_success(): """Test successful device code authentication.""" @@ -132,6 +136,7 @@ def test_device_code_flow_success(): ``` **Test**: `test_device_code_flow_timeout` + ```python def test_device_code_flow_timeout(): """Test device code flow timeout without approval.""" @@ -144,6 +149,7 @@ def test_device_code_flow_timeout(): ``` **Test**: `test_device_code_flow_denied` + ```python def test_device_code_flow_denied(): """Test device code flow when user denies.""" @@ -156,6 +162,7 @@ def test_device_code_flow_denied(): #### 2. **Password Flow** (Priority: MEDIUM) **Test**: `test_password_flow_success` + ```python def test_password_flow_success(): """Test successful password authentication.""" @@ -172,6 +179,7 @@ def test_password_flow_success(): ``` **Test**: `test_password_flow_invalid_credentials` + ```python def test_password_flow_invalid_credentials(): """Test password flow with wrong credentials.""" @@ -188,6 +196,7 @@ def test_password_flow_invalid_credentials(): #### 3. **Token Refresh** (Priority: HIGH) **Test**: `test_token_refresh_success` + ```python def test_token_refresh_success(): """Test successful token refresh.""" @@ -209,6 +218,7 @@ def test_token_refresh_success(): ``` **Test**: `test_token_refresh_without_refresh_token` + ```python def test_token_refresh_without_refresh_token(): """Test refresh fails when no refresh_token available.""" @@ -226,6 +236,7 @@ def test_token_refresh_without_refresh_token(): #### 4. **Userinfo Endpoint** (Priority: MEDIUM) **Test**: `test_get_userinfo_success` + ```python def test_get_userinfo_success(): """Test retrieving user information.""" @@ -241,6 +252,7 @@ def test_get_userinfo_success(): ``` **Test**: `test_get_userinfo_expired_token` + ```python def test_get_userinfo_expired_token(): """Test userinfo fails with expired token.""" @@ -258,6 +270,7 @@ def test_get_userinfo_expired_token(): #### 5. **CLI End-to-End** (Priority: HIGH) **Test**: `test_cli_login_logout_cycle` + ```python def test_cli_login_logout_cycle(): """Test full login/logout cycle via CLI.""" @@ -300,6 +313,7 @@ def test_cli_login_logout_cycle(): ### Test Utilities **DexTestServer Context Manager**: + ```python # tests/integration/dex_server.py import subprocess @@ -418,6 +432,7 @@ token_path: {tmpdir}/token ## CI/CD Integration **GitHub Actions Workflow**: + ```yaml # .github/workflows/ci.yml (add new job) @@ -458,6 +473,7 @@ test-prismctl-integration: ``` **Makefile Target**: + ```makefile # Makefile (add to testing section) diff --git a/docs-cms/memos/memo-023-tenancy-models.md b/docs-cms/memos/memo-023-tenancy-models.md index 4e7f69499..2feed433b 100644 --- a/docs-cms/memos/memo-023-tenancy-models.md +++ b/docs-cms/memos/memo-023-tenancy-models.md @@ -67,6 +67,7 @@ This memo explores three primary tenancy models and three isolation levels, prov - **Higher cost**: Full infrastructure stack per tenant **Configuration Example** (`single-tenant-config.yaml`): + ```yaml deployment: model: single_tenant @@ -108,6 +109,7 @@ observability: **Deployment Patterns**: 1. **Kubernetes Namespace per Tenant**: + ```bash kubectl create namespace tenant-a helm install prism-proxy prism/proxy \ @@ -116,6 +118,7 @@ observability: ``` 2. **Bare Metal / VM Deployment**: + ```bash # Each tenant gets dedicated servers ansible-playbook deploy-prism.yml \ @@ -123,6 +126,7 @@ observability: ``` 3. **Cloud Provider (AWS)**: + ```terraform module "prism_tenant_a" { source = "./modules/prism-single-tenant" @@ -199,6 +203,7 @@ observability: - **Noisy neighbor risk**: One tenant's traffic can impact others **Configuration Example** (`multi-tenant-config.yaml`): + ```yaml deployment: model: multi_tenant @@ -319,6 +324,7 @@ curl -X POST http://prism-bridge:8980/api/v1/loadbalancer/reload **Orchestrator Integration**: 1. **Kubernetes (Controller Pattern)**: + ```yaml apiVersion: prism.io/v1alpha1 kind: PrismTenant @@ -335,6 +341,7 @@ curl -X POST http://prism-bridge:8980/api/v1/loadbalancer/reload prism-bridge watches for `PrismTenant` CRDs and updates routing accordingly. 2. **Nomad (Service Discovery)**: + ```hcl job "prism-bridge" { group "control-plane" { @@ -354,6 +361,7 @@ curl -X POST http://prism-bridge:8980/api/v1/loadbalancer/reload ``` 3. **Bare Metal (Address Lists)**: + ```bash # prism-bridge maintains address list file prism-bridge get-addresses --pool standard > /etc/haproxy/backends-standard.lst @@ -394,6 +402,7 @@ curl -X POST http://prism-bridge:8980/api/v1/loadbalancer/reload - **Performance SLAs**: Different SLAs for different customer tiers **Configuration Example**: + ```yaml deployment: model: hybrid @@ -452,6 +461,7 @@ Isolation levels control how tenant workloads are separated **within** a shared - **Trusted tenants** - all tenants are internal teams within same organization **Configuration**: + ```yaml proxy: isolation_level: none @@ -511,6 +521,7 @@ proxy: - **Compliance requirements** needing logical separation (but not physical) **Configuration**: + ```yaml proxy: isolation_level: namespace @@ -536,6 +547,7 @@ proxy: **Implementation Details**: 1. **Connection Pool Isolation**: + ```go type NamespaceConnectionPool struct { namespace string @@ -557,6 +569,7 @@ proxy: - Idle pools can be garbage collected after timeout 3. **Resource Accounting**: + ```bash # Query per-namespace resource usage prismctl metrics namespace tenant-a-prod @@ -612,6 +625,7 @@ proxy: - **Short-lived sessions** where setup cost is acceptable **Configuration**: + ```yaml proxy: isolation_level: session @@ -640,6 +654,7 @@ proxy: **Implementation Details**: 1. **Session Identification**: + ```go type Session struct { ID string // UUID @@ -652,6 +667,7 @@ proxy: ``` 2. **Connection Lifecycle**: + ```text Client connects β†’ Session created β†’ Connections established ↓ @@ -667,6 +683,7 @@ proxy: - **Kafka**: ~10-50ms per connection setup (TCP + SASL + metadata fetch) 4. **Audit Trail**: + ```json { "event": "session_created", @@ -950,6 +967,7 @@ tenant_routing: ### Audit and Compliance 1. **Audit Logging**: + ```json { "timestamp": "2025-10-13T22:45:00Z", @@ -976,6 +994,7 @@ tenant_routing: ### Monitoring Per-tenant metrics to track: + ```promql # Request rate per tenant rate(prism_requests_total{tenant_id="tenant-a"}[5m]) @@ -998,6 +1017,7 @@ prism_namespace_memory_usage{namespace="tenant-a-prod"} ### Alerting Critical alerts: + ```yaml alerts: - name: TenantConnectionPoolExhausted diff --git a/docs-cms/memos/memo-030-pattern-based-acceptance-testing.md b/docs-cms/memos/memo-030-pattern-based-acceptance-testing.md index 8b5e7939b..49df02822 100644 --- a/docs-cms/memos/memo-030-pattern-based-acceptance-testing.md +++ b/docs-cms/memos/memo-030-pattern-based-acceptance-testing.md @@ -25,6 +25,7 @@ We've transitioned from backend-specific acceptance tests to a **pattern-based a ### Problems with Backend-Specific Tests **Before** (MEMO-015 approach): + ```text tests/acceptance/ β”œβ”€β”€ interfaces/ @@ -47,6 +48,7 @@ Issues: ### Pattern-Based Solution **After** (current approach): + ```text tests/acceptance/ β”œβ”€β”€ framework/ @@ -81,6 +83,7 @@ Benefits: Backends register themselves with the framework at package init time: **File**: `tests/acceptance/backends/memstore.go` + ```go func init() { framework.MustRegisterBackend(framework.Backend{ @@ -104,6 +107,7 @@ func init() { ``` **File**: `tests/acceptance/backends/redis.go` + ```go func init() { framework.MustRegisterBackend(framework.Backend{ @@ -132,6 +136,7 @@ func init() { Pattern tests are written once and run against all compatible backends: **File**: `tests/acceptance/patterns/keyvalue/basic_test.go` + ```go func TestKeyValueBasicPattern(t *testing.T) { tests := []framework.PatternTest{ @@ -175,6 +180,7 @@ func testSetAndGet(t *testing.T, driver interface{}, caps framework.Capabilities The framework discovers backends and runs tests automatically: **File**: `tests/acceptance/framework/pattern_runner.go` + ```go func RunPatternTests(t *testing.T, pattern Pattern, tests []PatternTest) { // Find all backends that support this pattern @@ -265,6 +271,7 @@ graph TD ### GitHub Actions Workflow **File**: `.github/workflows/pattern-acceptance-tests.yml` + ```yaml name: Pattern Acceptance Tests @@ -317,6 +324,7 @@ test-acceptance-consumer: ## Run Consumer pattern tests only ### 1. Create Pattern Test File **File**: `tests/acceptance/patterns/timeseries/basic_test.go` + ```go package timeseries_test @@ -350,6 +358,7 @@ func testWritePoints(t *testing.T, driver interface{}, caps framework.Capabiliti ### 2. Register Pattern Constant **File**: `tests/acceptance/framework/types.go` + ```go type Pattern string @@ -385,6 +394,7 @@ func init() { ### 1. Implement Backend Setup **File**: `tests/acceptance/backends/influxdb.go` + ```go package backends @@ -456,6 +466,7 @@ func setupInfluxDB(t *testing.T, ctx context.Context) (interface{}, func()) { ### 2. Import in Tests **File**: `tests/acceptance/patterns/timeseries/basic_test.go` + ```go import ( _ "github.com/jrepp/prism-data-layer/tests/acceptance/backends" @@ -469,6 +480,7 @@ import ( ### 1. Zero Test Duplication Write test logic once: + ```go func testSetAndGet(t *testing.T, driver interface{}, caps framework.Capabilities) { // Single implementation @@ -518,6 +530,7 @@ No manual skip logic in test code. ### 5. Parallel Execution Tests run in parallel by backend: + ```go t.Run(backend.Name, func(t *testing.T) { t.Parallel() // Backends test concurrently diff --git a/docs-cms/memos/memo-032-driver-test-consolidation.md b/docs-cms/memos/memo-032-driver-test-consolidation.md index 7489ed9c0..233410d55 100644 --- a/docs-cms/memos/memo-032-driver-test-consolidation.md +++ b/docs-cms/memos/memo-032-driver-test-consolidation.md @@ -137,6 +137,7 @@ tests/unit/backends/ ### Phase 2: Remove Redundant Tests from pkg/drivers **Delete redundant tests**: + ```bash # Remove functional tests from driver packages git rm pkg/drivers/memstore/memstore_test.go @@ -158,6 +159,7 @@ git rm pkg/drivers/nats/nats_test.go #### NATS-specific tests to add to `tests/acceptance/patterns/consumer/`: 1. **Fanout behavior** (`test_fanout.go`): + ```go func testFanout(t *testing.T, driver interface{}, caps framework.Capabilities) { // Multiple subscribers receive same message @@ -165,6 +167,7 @@ git rm pkg/drivers/nats/nats_test.go ``` 2. **Message ordering** (`test_ordering.go`): + ```go func testMessageOrdering(t *testing.T, driver interface{}, caps framework.Capabilities) { // Messages received in publish order @@ -172,6 +175,7 @@ git rm pkg/drivers/nats/nats_test.go ``` 3. **Unsubscribe behavior** (`test_unsubscribe.go`): + ```go func testUnsubscribeStopsMessages(t *testing.T, driver interface{}, caps framework.Capabilities) { // No messages after unsubscribe @@ -179,6 +183,7 @@ git rm pkg/drivers/nats/nats_test.go ``` 4. **Concurrent publish** (`concurrent_test.go` - already exists, verify NATS is included): + ```go func testConcurrentPublish(t *testing.T, driver interface{}, caps framework.Capabilities) { // Concurrent publishers don't interfere @@ -186,6 +191,7 @@ git rm pkg/drivers/nats/nats_test.go ``` 5. **Metadata handling** (`test_metadata.go`): + ```go func testPublishWithMetadata(t *testing.T, driver interface{}, caps framework.Capabilities) { // Metadata preserved (backend-dependent) @@ -201,6 +207,7 @@ git rm pkg/drivers/nats/nats_test.go #### Makefile Changes **Before**: + ```makefile test-drivers: @cd pkg/drivers/memstore && go test -v ./... @@ -211,6 +218,7 @@ test-all: test-drivers test-acceptance ``` **After**: + ```makefile # Unit tests for backend-specific behavior test-unit-backends: @@ -236,6 +244,7 @@ test-coverage: #### CI Workflow Changes **Before** (`.github/workflows/ci.yml`): + ```yaml - name: Test drivers run: make test-drivers @@ -245,6 +254,7 @@ test-coverage: ``` **After**: + ```yaml - name: Unit Tests run: make test-unit-backends @@ -272,6 +282,7 @@ test-coverage: ### Coverage Measurement **Generate coverage including driver code**: + ```bash go test -coverprofile=coverage.out \ -coverpkg=github.com/jrepp/prism-data-layer/pkg/drivers/... \ @@ -281,6 +292,7 @@ go tool cover -html=coverage.out -o coverage.html ``` **Coverage report format**: + ```text pkg/drivers/memstore/memstore.go: 92.3% of statements pkg/drivers/redis/redis.go: 88.7% of statements @@ -290,6 +302,7 @@ TOTAL DRIVER COVERAGE: 88.7% ``` **Enforcement in CI**: + ```bash COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') if (( $(echo "$COVERAGE < 85" | bc -l) )); then @@ -344,12 +357,14 @@ fi ### 4. **Better Test Organization** **Before** (scattered tests): + ```text pkg/drivers/redis/redis_test.go # Functional + unit tests mixed tests/acceptance/patterns/keyvalue/basic_test.go # Functional tests ``` **After** (clear separation): + ```text tests/unit/backends/redis/redis_unit_test.go # Driver-specific unit tests tests/acceptance/patterns/keyvalue/basic_test.go # Interface compliance tests @@ -429,6 +444,7 @@ tests/acceptance/patterns/keyvalue/basic_test.go # Interface compliance tests 4. If coverage drops >2%, add targeted acceptance tests **Validation**: + ```bash # Before migration make test-drivers test-acceptance @@ -455,6 +471,7 @@ echo "After: $AFTER" 4. Merge only when all drivers migrated successfully **Rollback Plan**: + ```bash # If migration fails, revert git revert HEAD~5..HEAD # Revert last 5 commits diff --git a/docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md b/docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md index 70273de35..10299b282 100644 --- a/docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md +++ b/docs-cms/memos/memo-033-process-isolation-bulkhead-pattern.md @@ -533,6 +533,7 @@ procmgr_process_state_transitions_total{to_state!="Syncing"} - Reasonable overhead **Recommended Configuration**: + ```go IsolationConfig{ Level: isolation.IsolationNamespace, diff --git a/docs-cms/memos/memo-034-pattern-launcher-quickstart.md b/docs-cms/memos/memo-034-pattern-launcher-quickstart.md index 8642c88d8..532890e15 100644 --- a/docs-cms/memos/memo-034-pattern-launcher-quickstart.md +++ b/docs-cms/memos/memo-034-pattern-launcher-quickstart.md @@ -119,6 +119,7 @@ go run cmd/pattern-launcher/main.go \ ``` **Expected output**: + ```text Discovering patterns in directory: ./patterns Discovered pattern: hello-pattern (version: 1.0.0, isolation: namespace) @@ -147,6 +148,7 @@ grpcurl -plaintext \ ``` **Expected**: + ```json { "processId": "ns:tenant-a:hello-pattern", @@ -157,6 +159,7 @@ grpcurl -plaintext \ ``` **Verify it's running**: + ```bash curl http://localhost:9090/health # OK - hello-pattern is healthy @@ -329,6 +332,7 @@ grpcurl -plaintext \ ``` **Expected**: + ```json { "healthy": true, diff --git a/docs-cms/memos/memo-035-nomad-local-development-setup.md b/docs-cms/memos/memo-035-nomad-local-development-setup.md index fdd4aa1c6..8e9dde8bb 100644 --- a/docs-cms/memos/memo-035-nomad-local-development-setup.md +++ b/docs-cms/memos/memo-035-nomad-local-development-setup.md @@ -298,6 +298,7 @@ Memory required: %s ``` **Experience**: + ```text πŸš€ Prism Stack Builder @@ -608,6 +609,7 @@ prismctl dev start --services=./my-services.yaml **Most issues fixed automatically by `prismctl dev`:** ### "Nomad not found" + ```bash # Just run init again prismctl dev init @@ -615,6 +617,7 @@ prismctl dev init ``` ### "Port already in use" + ```bash # Dashboard shows port conflicts prismctl dev start @@ -623,6 +626,7 @@ prismctl dev start ``` ### "Service unhealthy" + ```bash # Dashboard shows health status in real-time # Red = failed (with last error message) @@ -633,6 +637,7 @@ prismctl dev start ``` ### "Need logs" + ```bash # In dashboard, select service and press 'l' # Or: prismctl dev logs @@ -670,6 +675,7 @@ services: ``` **Now available in picker**: + ```bash prismctl dev start # [βœ“] clickhouse 9000 diff --git a/docs-cms/memos/memo-036-kubernetes-operator-development.md b/docs-cms/memos/memo-036-kubernetes-operator-development.md index a5372bbe0..6659336f7 100644 --- a/docs-cms/memos/memo-036-kubernetes-operator-development.md +++ b/docs-cms/memos/memo-036-kubernetes-operator-development.md @@ -1306,12 +1306,14 @@ func (r *PrismPatternReconciler) SetupWithManager(mgr ctrl.Manager) error { 5. Wait for cluster to start (green indicator) **Verify**: + ```bash kubectl cluster-info kubectl get nodes ``` Expected output: + ```bash NAME STATUS ROLES AGE VERSION docker-desktop Ready control-plane 1d v1.28.2 @@ -1388,6 +1390,7 @@ spec: ``` Apply it: + ```bash kubectl apply -f config/samples/prism_v1alpha1_prismstack.yaml ``` @@ -1487,6 +1490,7 @@ var _ = Describe("PrismStack Controller", func() { ``` Run tests: + ```bash make test ``` @@ -1985,11 +1989,13 @@ kubectl logs -n prism-system deployment/prism-operator-controller-manager -c man ### Issue 1: CRD Not Found After Install **Symptoms**: + ```go error: the server doesn't have a resource type "prismstack" ``` **Solution**: + ```bash # Reinstall CRDs make install @@ -2005,6 +2011,7 @@ kubectl apply -f config/crd/bases/ ### Issue 2: Operator Pod CrashLoopBackOff **Symptoms**: + ```bash $ kubectl get pods -n prism-system NAME READY STATUS RESTARTS AGE @@ -2012,6 +2019,7 @@ prism-operator-controller-manager-xxx 0/2 CrashLoopBackOff 5 2 ``` **Diagnosis**: + ```bash # Check pod logs kubectl logs -n prism-system prism-operator-controller-manager-xxx -c manager @@ -2024,6 +2032,7 @@ kubectl logs -n prism-system prism-operator-controller-manager-xxx -c manager ``` **Solutions**: + ```bash # Solution 1: Fix RBAC make manifests @@ -2046,6 +2055,7 @@ kubectl get events -n prism-system --sort-by='.lastTimestamp' - Status not updating **Diagnosis**: + ```bash # Check PrismStack status kubectl get prismstack prism-dev -o yaml | grep -A 20 status @@ -2058,6 +2068,7 @@ kubectl describe prismstack prism-dev ``` **Solutions**: + ```bash # Solution 1: Check controller is watching correct namespace kubectl logs -n prism-system deployment/prism-operator-controller-manager | grep "Starting workers" @@ -2072,11 +2083,13 @@ kubectl rollout restart deployment prism-operator-controller-manager -n prism-sy ### Issue 4: Pods Not Scheduling (Node Selector) **Symptoms**: + ```text 0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector ``` **Diagnosis**: + ```bash # Check pod scheduling failure kubectl describe pod prism-proxy-xxx @@ -2089,6 +2102,7 @@ kubectl get deployment prism-dev-proxy -o jsonpath='{.spec.template.spec.nodeSel ``` **Solutions**: + ```bash # Solution 1: Label nodes to match selector kubectl label nodes docker-desktop role=prism-proxy @@ -2107,6 +2121,7 @@ kubectl edit prismstack prism-dev - No lease object found **Diagnosis**: + ```bash # Check for lease object kubectl get lease -n prism-system @@ -2119,6 +2134,7 @@ kubectl auth can-i create leases --as=system:serviceaccount:prism-system:prism-a ``` **Solutions**: + ```bash # Solution 1: Add lease RBAC kubectl apply -f - < Result<()> { @@ -156,6 +160,7 @@ async fn main() -> Result<()> { tokio::try_join!(data_server, admin_server)?; Ok(()) } + ```text **Authentication:** @@ -163,10 +168,12 @@ async fn main() -> Result<()> { Admin API requires separate credentials: ``` + # Metadata in all admin requests metadata: x-admin-token: "admin-abc123" x-admin-user: "alice@example.com" + ```text **Pros:** @@ -243,11 +250,13 @@ admin-ui/static/ β”‚ └── health.js # Health dashboard └── lib/ └── grpc-web.js # gRPC-Web runtime + ```text **JavaScript Client:** ``` + // admin-ui/static/js/config.js import {AdminServiceClient} from './admin_grpc_web_pb.js'; @@ -264,6 +273,7 @@ async function loadConfigs() { renderConfigs(response.getConfigsList()); }); } + ```text **Pros:** @@ -291,6 +301,7 @@ async function loadConfigs() { **Implementation:** ``` + impl AdminService { async fn create_config(&self, req: CreateConfigRequest) -> Result { let actor = self.get_admin_user_from_metadata()?; @@ -313,11 +324,13 @@ impl AdminService { result } } + ```text **Storage:** ``` + CREATE TABLE audit_log ( id UUID PRIMARY KEY, timestamp TIMESTAMPTZ NOT NULL, @@ -333,6 +346,7 @@ CREATE TABLE audit_log ( INDEX idx_audit_actor ON audit_log(actor), INDEX idx_audit_operation ON audit_log(operation) ); + ```text **Pros:** @@ -351,6 +365,7 @@ CREATE TABLE audit_log ( **Docker Compose:** ``` + services: prism-proxy: image: prism/proxy:latest @@ -370,11 +385,13 @@ services: ADMIN_TOKEN_SECRET: ${ADMIN_TOKEN_SECRET} depends_on: - prism-proxy + ```text **Network Policy:** ``` + # Kubernetes NetworkPolicy example apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -397,6 +414,7 @@ spec: ports: - protocol: TCP port: 8980 # Data plane - all pods + ```text ## Security Considerations @@ -409,6 +427,7 @@ spec: - Stored in secret management system ``` + #[derive(Debug)] pub struct AdminApiKey { pub key_id: String, @@ -426,6 +445,7 @@ pub enum Permission { NamespaceAdmin, OperationalAdmin, } + ```text **Alternative: OAuth2** @@ -438,6 +458,7 @@ pub enum Permission { **Role-Based Access Control (RBAC):** ``` + roles: admin: - config:* @@ -455,6 +476,7 @@ roles: - config:read - session:read - backend:read + ```text ### Network Isolation @@ -465,6 +487,7 @@ roles: - mTLS for admin API connections ``` + # Example firewall rules - port: 8980 protocol: TCP @@ -473,6 +496,7 @@ roles: - port: 8981 protocol: TCP allow: [internal, 10.0.0.0/8] + ```text ## Performance Considerations @@ -482,6 +506,7 @@ roles: Admin UI caches configuration list: ``` + // Cache configs for 30 seconds const configCache = new Map(); const CACHE_TTL = 30000; @@ -498,6 +523,7 @@ async function getConfigs() { configCache.set('list', { data: configs, timestamp: now }); return configs; } + ```text ### Pagination @@ -505,6 +531,7 @@ async function getConfigs() { All list operations support pagination: ``` + message ListSessionsRequest { int32 page_size = 1; optional string page_token = 2; @@ -515,6 +542,7 @@ message ListSessionsResponse { optional string next_page_token = 2; int32 total_count = 3; } + ```text ### Async Operations @@ -522,6 +550,7 @@ message ListSessionsResponse { Long-running operations return operation handle: ``` + message DrainConnectionsRequest { optional string namespace = 1; optional google.protobuf.Duration timeout = 2; @@ -542,6 +571,7 @@ message GetOperationResponse { int32 progress_percent = 2; optional string error = 3; } + ```text ## Migration and Rollout @@ -579,6 +609,7 @@ message GetOperationResponse { ### Unit Tests ``` + #[cfg(test)] mod tests { #[tokio::test] @@ -597,10 +628,12 @@ mod tests { assert_eq!(err.code(), Code::Unauthenticated); } } + ```text ### Integration Tests ``` + # Test admin API grpcurl -H "x-admin-token: test-token" \ localhost:8981 \ @@ -611,10 +644,12 @@ psql -c "SELECT * FROM audit_log WHERE operation = 'CreateConfig'" # Test UI curl http://localhost:8000 + ```text ### E2E Tests ``` + # tests/e2e/test_admin_workflow.py def test_full_admin_workflow(): # Create config via UI @@ -627,6 +662,7 @@ def test_full_admin_workflow(): # Verify audit log audit = api_client.get_audit_log(operation="CreateConfig") assert len(audit) == 1 + ```text ## Monitoring and Observability @@ -634,6 +670,7 @@ def test_full_admin_workflow(): ### Metrics ``` + // Prometheus metrics for admin API lazy_static! { static ref ADMIN_REQUESTS: IntCounterVec = register_int_counter_vec!( @@ -648,11 +685,13 @@ lazy_static! { &["operation"] ).unwrap(); } + ```text ### Alerts ``` + # Prometheus alerting rules groups: - name: prism_admin @@ -668,6 +707,7 @@ groups: rate(prism_admin_requests_total{status="unauthenticated"}[5m]) > 0 annotations: summary: "Unauthorized access attempts detected" + ```text ## Open Questions diff --git a/docs-cms/rfcs/rfc-004-redis-integration.md b/docs-cms/rfcs/rfc-004-redis-integration.md index 4dae33656..9c6303a14 100644 --- a/docs-cms/rfcs/rfc-004-redis-integration.md +++ b/docs-cms/rfcs/rfc-004-redis-integration.md @@ -66,12 +66,14 @@ graph TB ### 2.2 Deployment Models **Single Redis (Development)** + ```mermaid graph LR Prism[Prism Proxy] --> Redis[Redis Standalone
Cache + PubSub + Vector] ``` **Redis Cluster (Production)** + ```mermaid graph TB Prism[Prism Proxy] diff --git a/docs-cms/rfcs/rfc-005-clickhouse-integration.md b/docs-cms/rfcs/rfc-005-clickhouse-integration.md index 5d154fcaa..bf25e990e 100644 --- a/docs-cms/rfcs/rfc-005-clickhouse-integration.md +++ b/docs-cms/rfcs/rfc-005-clickhouse-integration.md @@ -479,6 +479,7 @@ clickhouse: daily_volume = events_per_day Γ— avg_event_size_bytes compressed_size = daily_volume / compression_ratio retention_storage = compressed_size Γ— retention_days Γ— (1 + replica_count) + ```text **Example**: @@ -494,6 +495,7 @@ retention_storage = compressed_size Γ— retention_days Γ— (1 + replica_count) ### 7.2 Monitoring ``` + metrics: ingestion: - insert_rate_events_per_sec @@ -517,11 +519,13 @@ metrics: replication: - replication_lag_seconds - replica_queue_size + ```text ### 7.3 Data Lifecycle ``` + graph LR Hot[Hot Storage
SSD
Last 7 days] -->|TTL| Warm[Warm Storage
HDD
8-30 days] Warm -->|TTL| Cold[Cold Storage
S3
31-90 days] @@ -531,15 +535,18 @@ graph LR style Warm fill:#feca57 style Cold fill:#48dbfb style Delete fill:#dfe6e9 + ```text ``` + -- Tiered storage with TTL ALTER TABLE events MODIFY TTL timestamp + INTERVAL 7 DAY TO VOLUME 'hot', timestamp + INTERVAL 30 DAY TO VOLUME 'warm', timestamp + INTERVAL 90 DAY TO VOLUME 'cold', timestamp + INTERVAL 90 DAY DELETE; + ```text ## 8. Migration Path diff --git a/docs-cms/rfcs/rfc-006-python-admin-cli.md b/docs-cms/rfcs/rfc-006-python-admin-cli.md index 14b1abcf4..d39d589bd 100644 --- a/docs-cms/rfcs/rfc-006-python-admin-cli.md +++ b/docs-cms/rfcs/rfc-006-python-admin-cli.md @@ -179,6 +179,7 @@ Visit: https://idp.example.com/activate Enter code: WXYZ-1234 Waiting for authentication... + ```text *Browser opens automatically to verification URL* @@ -250,6 +251,7 @@ prismctl namespace list ``` **Service Account Token Claims**: + ```json { "iss": "https://idp.example.com", @@ -450,6 +452,7 @@ prism β”œβ”€β”€ status # Show plugin health and metrics β”œβ”€β”€ reload # Hot-reload plugin code └── logs # View plugin logs + ```text ## Command Specifications @@ -459,6 +462,7 @@ prism #### Create Namespace ``` + # Preferred: Declarative mode from config file prismctl namespace create --config namespace.yaml @@ -471,6 +475,7 @@ prismctl namespace create my-app \ --pattern keyvalue \ --consistency strong \ --cache-ttl 300 + ```text **Output (Rich table)**: @@ -510,11 +515,13 @@ prismctl namespace list --include-inactive β”‚ session-cache β”‚ redis β”‚ cache β”‚ 156 β”‚ 89,012 β”‚ β”‚ metrics-olap β”‚ clickhouse β”‚ timeseries β”‚ 4 β”‚ 12,345 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text #### Describe Namespace ``` + prismctl namespace describe my-app # Include recent errors @@ -522,6 +529,7 @@ prismctl namespace describe my-app --show-errors # Include configuration prismctl namespace describe my-app --show-config + ```text **Output**: @@ -594,11 +602,13 @@ Backend Health Status Error: Connection refused to broker-3 Last Success: 5m ago Action: Check broker-3 connectivity + ```text #### Backend Statistics ``` + # Show stats for all backends prismctl backend stats @@ -607,6 +617,7 @@ prismctl backend stats --namespace my-app # Export to JSON prismctl backend stats --output json + ```text ### Session Management @@ -614,6 +625,7 @@ prismctl backend stats --output json #### List Sessions ``` + # List all active sessions across all namespaces prismctl session list @@ -625,6 +637,7 @@ prismctl session list # Auto-scopes if .config.yaml has namespace set # Show long-running sessions prismctl session list --duration ">1h" + ```text **Output**: @@ -664,6 +677,7 @@ Statistics: Success: 3 (75%) Avg Latency: 38.05ms P99 Latency: 145ms + ```text ### Configuration Management @@ -671,6 +685,7 @@ Statistics: #### Show Configuration ``` + # Show proxy-wide configuration prismctl config show @@ -682,16 +697,19 @@ prismctl config show # Uses namespace from .config.yaml if present # Export configuration prismctl config show --output yaml > prism-config.yaml + ```text #### Validate Configuration ``` + # Validate config file before applying prismctl config validate prism-config.yaml # Dry-run mode prismctl config validate prism-config.yaml --dry-run + ```text **Output**: @@ -751,16 +769,19 @@ Backend Health: βœ“ Redis (3 instances) βœ“ ClickHouse (2 instances) βœ— Kafka (1 degraded) + ```text #### Export Metrics ``` + # Prometheus format prismctl metrics export --format prometheus > metrics.prom # JSON format with metadata prismctl metrics export --format json --include-metadata > metrics.json + ```text ### Shadow Traffic @@ -768,6 +789,7 @@ prismctl metrics export --format json --include-metadata > metrics.json #### Enable Shadow Traffic ``` + # Enable shadow traffic for Postgres version upgrade (14 β†’ 16) prismctl shadow enable user-profiles \ --source postgres-14-primary \ @@ -780,6 +802,7 @@ prismctl shadow enable user-profiles \ --target postgres-16-replica \ --ramp-up "10%,25%,50%,100%" \ --interval 1h + ```text **Output**: @@ -845,6 +868,7 @@ Query Compatibility: βœ“ Performance parity achieved βœ“ Target performing well, ready for next stage + ```text ### Plugin Management @@ -856,6 +880,7 @@ For complete plugin development guide, see [RFC-008: Plugin Development Experien #### List Plugins ``` + # List all installed plugins prismctl plugin list @@ -865,6 +890,7 @@ prismctl plugin list --status disabled # Show plugin versions prismctl plugin list --show-versions + ```text **Output**: @@ -908,11 +934,13 @@ Installing plugin: mongodb Plugin 'mongodb' installed successfully Supported operations: get, set, query, aggregate Ready to create namespaces with backend: mongodb + ```text #### Update Plugin ``` + # Update to latest version prismctl plugin update mongodb @@ -921,6 +949,7 @@ prismctl plugin update mongodb --version 1.1.0 # Dry-run mode (check compatibility without applying) prismctl plugin update mongodb --dry-run + ```text **Output (with warnings)**: @@ -971,11 +1000,13 @@ Actions: ⚠ Use --force to stop all kafka namespaces Plugin 'kafka' disabled successfully + ```text #### Plugin Status ``` + # View plugin health and detailed metrics prismctl plugin status mongodb @@ -984,6 +1015,7 @@ prismctl plugin status mongodb --show-errors # Live monitoring mode prismctl plugin status mongodb --watch + ```text **Output**: @@ -1045,11 +1077,13 @@ Reloading plugin: mongodb Plugin 'mongodb' reloaded successfully Namespaces affected: 7 (all healthy) Reload time: 2.3s (zero downtime) + ```text #### View Plugin Logs ``` + # View recent logs prismctl plugin logs mongodb @@ -1061,6 +1095,7 @@ prismctl plugin logs mongodb --level error # Show logs from specific time range prismctl plugin logs mongodb --since "1h ago" + ```text **Output**: @@ -1182,11 +1217,13 @@ tools/ β”œβ”€β”€ acceptance_test.go # testscript runner β”œβ”€β”€ go.mod └── go.sum + ```text ### Example Implementation (Namespace Commands) ``` + # src/prism_cli/commands/namespace.py import typer from rich.console import Console @@ -1330,11 +1367,13 @@ def describe( except Exception as e: console.print(f"[red]Error describing namespace: {e}[/red]") raise typer.Exit(1) + ```text ### Admin gRPC Client Wrapper ``` + # src/prism_cli/client/admin.py import grpc from typing import List, Optional @@ -1427,6 +1466,7 @@ class AdminClient: def __exit__(self, *args): self.channel.close() + ```text ## Use-Case Recommendations @@ -1468,6 +1508,7 @@ The CLI follows a hierarchical configuration search strategy: **Example `.config.yaml` (project-level)**: ``` + # .config.yaml - Project configuration for my-app namespace namespace: my-app # Default namespace for scoped commands endpoint: localhost:50052 @@ -1479,10 +1520,12 @@ backend: cache_ttl: 300 # Sessions, config, metrics will auto-scope to this namespace unless --namespace specified + ```text **Example `~/.prism/config.yaml` (user-level)**: ``` + # ~/.prism/config.yaml - Global CLI configuration default_endpoint: localhost:50052 auth: @@ -1502,10 +1545,12 @@ timeouts: logging: level: info file: ~/.prism/cli.log + ```text **Usage pattern**: ``` + # In project directory with .config.yaml (namespace: my-app): cd ~/projects/my-app prismctl session list # Auto-scopes to my-app namespace @@ -1518,15 +1563,18 @@ prismctl session list --namespace other-app # Parent directory search: cd ~/projects/my-app/src/handlers prismctl session list # Finds .config.yaml in ~/projects/my-app/ + ```text ### Environment Variables ``` + # Override config file settings export PRISM_ENDPOINT="prism.example.com:50052" export PRISM_AUTH_METHOD="oauth2" export PRISM_OUTPUT_FORMAT="json" + ```text ## Performance and UX @@ -1552,6 +1600,7 @@ export PRISM_OUTPUT_FORMAT="json" ### Unit Tests ``` + # tests/test_namespace.py from typer.testing import CliRunner from prism_cli.main import app @@ -1591,11 +1640,13 @@ def test_namespace_list_json(): data = json.loads(result.stdout) assert len(data) == 2 assert data[0]["name"] == "ns1" + ```text ### Integration Tests ``` + # tests/integration/test_admin_client.py import pytest from prism_cli.client.admin import AdminClient @@ -1621,6 +1672,7 @@ def test_create_and_list_namespace(admin_client): # Cleanup admin_client.delete_namespace("test-integration") + ```text ## Deployment @@ -1628,6 +1680,7 @@ def test_create_and_list_namespace(admin_client): ### Installation ``` + # Install via uv (development) cd prism-cli uv pip install -e . @@ -1638,11 +1691,13 @@ uv pip install prism-cli # Verify installation prismctl --version prismctl --help + ```text ### Shell Completion ``` + # Bash prismctl --install-completion bash @@ -1651,6 +1706,7 @@ prismctl --install-completion zsh # Fish prismctl --install-completion fish + ```text ## Migration Path @@ -1704,6 +1760,7 @@ Track CLI adoption and usage patterns: CLI logs structured events: ``` + { "timestamp": "2025-10-08T09:15:23.456Z", "level": "info", @@ -1712,6 +1769,7 @@ CLI logs structured events: "duration_ms": 234, "status": "success" } + ```text ## Security Considerations @@ -1741,6 +1799,7 @@ CLI logs structured events: ### All Commands ``` + prismctl namespace create [options] prismctl namespace list [options] prismctl namespace describe [options] @@ -1771,11 +1830,13 @@ prismctl shadow status prismctl version prismctl help [command] + ```text ### Global Options ``` + --endpoint # Proxy endpoint (default: localhost:50052) --output # Output format: table, json, yaml --no-color # Disable colored output @@ -1783,6 +1844,7 @@ prismctl help [command] --quiet, -q # Suppress non-error output --config # CLI config file --help, -h # Show help + ```text --- diff --git a/docs-cms/rfcs/rfc-007-cache-strategies.md b/docs-cms/rfcs/rfc-007-cache-strategies.md index 45bf7effe..6d30cafc3 100644 --- a/docs-cms/rfcs/rfc-007-cache-strategies.md +++ b/docs-cms/rfcs/rfc-007-cache-strategies.md @@ -766,6 +766,7 @@ What's your data update frequency? β”œβ”€ Rarely (hourly+) β†’ Look-Aside + long TTL + warmup β”œβ”€ Occasionally (minutes) β†’ Look-Aside + short TTL └─ Frequently (seconds) β†’ Write-Through + ```text --- diff --git a/docs-cms/rfcs/rfc-008-proxy-plugin-architecture.md b/docs-cms/rfcs/rfc-008-proxy-plugin-architecture.md index 2c871b6a9..bfb094af2 100644 --- a/docs-cms/rfcs/rfc-008-proxy-plugin-architecture.md +++ b/docs-cms/rfcs/rfc-008-proxy-plugin-architecture.md @@ -1343,6 +1343,7 @@ Backend-Specific Results: ClickHouse: βœ“ 10/10 tests passed test result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out + ```text ### CI/CD Integration @@ -1350,6 +1351,7 @@ test result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out **GitHub Actions workflow**: ``` + # .github/workflows/plugin-acceptance.yml name: Plugin Acceptance Tests @@ -1390,6 +1392,7 @@ jobs: with: name: acceptance-test-results-${{ matrix.backend }} path: target/test-results/ + ```text ### Benefits of Acceptance Test Framework @@ -1453,6 +1456,7 @@ jobs: ### Plugin Verification ``` + // Verify plugin before loading pub fn verify_plugin(plugin_path: &Path) -> Result<()> { // 1. Check file permissions (must be owned by prism user) @@ -1474,6 +1478,7 @@ pub fn verify_plugin(plugin_path: &Path) -> Result<()> { Ok(()) } + ```text ## Netflix Architecture Comparison @@ -1568,6 +1573,7 @@ Making plugins easy to create is critical for Prism's success. We prioritize: **Quick Start** - Create a new plugin in 30 seconds: ``` + # Create new plugin from template prism-plugin-init --name mongodb --language rust @@ -1588,6 +1594,7 @@ mongodb-plugin/ β”‚ β”œβ”€β”€ integration_test.rs β”‚ └── fixtures/ └── README.md + ```text **Template provides**: @@ -1604,6 +1611,7 @@ mongodb-plugin/ Rust SDK provides helpers for common patterns: ``` + use prism_plugin_sdk::prelude::*; #[plugin] @@ -1634,6 +1642,7 @@ impl BackendPlugin for MongoDbPlugin { Ok(GetResponse { value: doc }) } } + ```text **SDK Features**: @@ -1648,6 +1657,7 @@ impl BackendPlugin for MongoDbPlugin { **Local Testing Without Prism**: ``` + # 1. Start plugin in standalone mode cargo run --bin mongodb-plugin-server @@ -1658,11 +1668,13 @@ grpcurl -plaintext -d '{"namespace": "test", "config": {...}}' \ # 3. Send operations grpcurl -plaintext -d '{"operation": "get", "params": {"key": "foo"}}' \ localhost:50100 prism.plugin.BackendPlugin/Execute + ```text **Integration Testing**: ``` + #[tokio::test] async fn test_get_operation() { // SDK provides test harness @@ -1676,13 +1688,16 @@ async fn test_get_operation() { assert_eq!(response.value, expected_value); } + ```text **Hot-Reload During Development**: ``` + # Watch for changes and reload plugin cargo watch -x 'build --release' -s 'prism plugin reload mongodb' + ```text ### Language Support Strategy @@ -1702,7 +1717,9 @@ cargo watch -x 'build --release' -s 'prism plugin reload mongodb' **Template Availability**: ``` + prism-plugin-init --name mybackend --language [rust|go|python] + ```text Each template includes: @@ -1728,6 +1745,7 @@ Each template includes: **Performance Isolation** (Prevents noisy neighbor): ``` + # Kubernetes resource limits per plugin apiVersion: v1 kind: Pod @@ -1744,11 +1762,13 @@ spec: limits: cpu: "4" # Prevent CPU starvation of other plugins memory: "8Gi" # Prevent OOM affecting proxy + ```text **Network Isolation** (Prevents cross-plugin communication): ``` + # NetworkPolicy: Plugin can only talk to proxy and backend apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -1771,6 +1791,7 @@ spec: - podSelector: matchLabels: app: clickhouse # Plugin can only call ClickHouse + ```text ### Plugin Administration @@ -1780,6 +1801,7 @@ Plugin management commands are provided via the Prism Admin CLI. See [RFC-006 Pl **Quick examples**: ``` + # List installed plugins prism plugin list @@ -1798,6 +1820,7 @@ prism plugin status mongodb # Hot-reload plugin code prism plugin reload mongodb + ```text For detailed plugin admin commands, see [RFC-006 Section: Plugin Management](/rfc/rfc-006.md#plugin-management). @@ -1809,6 +1832,7 @@ For detailed plugin admin commands, see [RFC-006 Section: Plugin Management](/rf 1. **Implement BackendPlugin Trait**: ``` + use prism_plugin::{BackendPlugin, InitializeRequest, ExecuteRequest}; pub struct MyBackendPlugin { @@ -1839,26 +1863,31 @@ impl BackendPlugin for MyBackendPlugin { } } } + ```text 2. **Build as Shared Library**: ``` + [lib] crate-type = ["cdylib"] # Dynamic library [dependencies] prism-plugin-sdk = "0.1" + ```text 3. **Register Plugin**: ``` + # Add to proxy configuration plugins: - name: my-backend library: /path/to/libmy_backend_plugin.so type: in_process + ```text --- diff --git a/docs-cms/rfcs/rfc-009-distributed-reliability-patterns.md b/docs-cms/rfcs/rfc-009-distributed-reliability-patterns.md index b32a3058c..dc01728b0 100644 --- a/docs-cms/rfcs/rfc-009-distributed-reliability-patterns.md +++ b/docs-cms/rfcs/rfc-009-distributed-reliability-patterns.md @@ -70,11 +70,13 @@ Patterns are ordered by complexity and value. Prism will implement in this order β”‚ Redis β”‚ β”‚ Postgres β”‚ β”‚ S3 β”‚ β”‚ (ms) β”‚ β”‚ (10ms) β”‚ β”‚ (100ms) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text #### Prism Configuration ``` + # .config.yaml namespaces: - name: user-activity-logs @@ -112,11 +114,13 @@ namespaces: - condition: age > 30_days AND tier == warm action: demote_to_cold + ```text #### Client Code ``` + // Application sees unified interface let log = client.get("user-activity-logs", "user:12345:2025-01-15").await?; @@ -125,6 +129,7 @@ let log = client.get("user-activity-logs", "user:12345:2025-01-15").await?; // 2. Checks warm tier (Postgres) - HIT // 3. Returns result // 4. Optionally promotes to hot tier (if access_count > 10) + ```text #### Key Characteristics @@ -287,11 +292,13 @@ let order = client.get("order-writes", "order:789").await?; β”‚ 3. Process full payload β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text #### Prism Configuration ``` + namespaces: - name: video-processing-events pattern: claim-check @@ -316,11 +323,13 @@ namespaces: # OR retain for replay on_consume: false retention: 604800 # 7 days + ```text #### Client Code ``` + // Producer: Prism handles claim check automatically let event = VideoProcessingEvent { video_id: "vid123", @@ -342,6 +351,7 @@ let event: VideoProcessingEvent = client.consume("video-processing-events").awai // 4. Returns to consumer assert_eq!(event.raw_video.len(), 50_000_000); + ```text #### Key Characteristics @@ -586,11 +596,13 @@ graph TD β”‚ Redis β”‚ β”‚ Elastic β”‚ β”‚ ClickHouse β”‚ β”‚ (Cache) β”‚ β”‚ (Search)β”‚ β”‚ (Analytics) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text #### Prism Configuration ``` + namespaces: - name: user-cdc pattern: change-data-capture @@ -633,11 +645,13 @@ namespaces: backend: clickhouse operations: [INSERT, UPDATE] table: user_events + ```text #### CDC Event Format ``` + { "op": "u", // c=create, u=update, d=delete "source": { @@ -658,11 +672,13 @@ namespaces: }, "ts_ms": 1704931200000 } + ```text #### Client Code ``` + # Application does normal database operations db.execute("UPDATE users SET email = 'new@example.com' WHERE id = 42") @@ -674,6 +690,7 @@ db.execute("UPDATE users SET email = 'new@example.com' WHERE id = 42") # 5. Inserts into ClickHouse: user_events table # No dual writes needed! + ```text #### Key Characteristics @@ -851,11 +868,13 @@ let listings = client.query("product-listings", filters).await?; β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Kafka (Events) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text #### Prism Configuration ``` + namespaces: - name: order-service pattern: outbox @@ -888,11 +907,13 @@ namespaces: cleanup: strategy: delete # or mark_published retention: 86400 # Keep published events for 1 day + ```text #### Client Code ``` + // Application: Single transaction let tx = db.begin().await?; @@ -918,6 +939,7 @@ tx.commit().await?; // - Deletes from outbox (or marks published_at) // Guaranteed: If order exists in database, event will be published. + ```text #### Key Characteristics @@ -961,6 +983,7 @@ tx.commit().await?; **Use Case**: Video processing pipeline where videos are too large for Kafka, but you want pub/sub semantics. ``` + namespaces: - name: video-processing patterns: @@ -982,6 +1005,7 @@ namespaces: - name: thumbnail-generator - name: transcoder - name: metadata-extractor + ```text **How it works**: @@ -996,6 +1020,7 @@ namespaces: **Use Case**: Transactionally publish large payloads (e.g., ML model weights after training). ``` + namespaces: - name: ml-model-releases patterns: @@ -1019,6 +1044,7 @@ namespaces: destination: backend: kafka topic: model-releases + ```text **How it works**: @@ -1034,6 +1060,7 @@ namespaces: **Use Case**: High-throughput writes with automatic hot/warm/cold tiering. ``` + namespaces: - name: user-activity patterns: @@ -1060,6 +1087,7 @@ namespaces: ttl: 2592000 cold: backend: s3 + ```text **How it works**: @@ -1074,6 +1102,7 @@ namespaces: **Use Case**: Keep read models in sync with write model using change data capture. ``` + namespaces: - name: product-catalog patterns: @@ -1100,6 +1129,7 @@ namespaces: - name: product-cache backend: redis sync_from: product-cdc.products + ```text **How it works**: @@ -1142,11 +1172,13 @@ namespaces: Each pattern is a first-class `pattern` type in namespace config: ``` + namespaces: - name: my-data pattern: tiered-storage # or event-sourcing, cqrs, etc. backend: multi # Indicates multiple backends # Pattern-specific config follows + ```text ### 2. Code Generation from Patterns diff --git a/docs-cms/rfcs/rfc-010-admin-protocol-oidc.md b/docs-cms/rfcs/rfc-010-admin-protocol-oidc.md index 5df70cb63..30976f953 100644 --- a/docs-cms/rfcs/rfc-010-admin-protocol-oidc.md +++ b/docs-cms/rfcs/rfc-010-admin-protocol-oidc.md @@ -55,6 +55,7 @@ Administrator β†’ OIDC Provider β†’ Admin CLI/UI β†’ Prism Admin API β†’ Backend 2. Receive JWT with claims 3. Present JWT in gRPC metadata 4. Authorized operations + ```text ### Ports and Endpoints @@ -471,6 +472,7 @@ All requests must include: authorization: Bearer request-id: // Optional but recommended + ```text ### Error Codes @@ -490,6 +492,7 @@ Standard gRPC status codes: ### Roles ``` + roles: admin: description: Full administrative access @@ -509,6 +512,7 @@ roles: description: Read-only access permissions: - admin:read + ```text ### Permission Mapping @@ -529,6 +533,7 @@ roles: ### Authorization Middleware ``` + use tonic::{Request, Status}; use tonic::metadata::MetadataMap; @@ -578,6 +583,7 @@ impl AuthInterceptor { }.to_string() } } + ```text ## Audit Logging @@ -585,6 +591,7 @@ impl AuthInterceptor { ### Audit Entry Structure ``` + #[derive(Debug, Serialize)] pub struct AuditLogEntry { pub id: Uuid, @@ -600,11 +607,13 @@ pub struct AuditLogEntry { pub error: Option, pub metadata: serde_json::Value, } + ```text ### Storage ``` + CREATE TABLE admin_audit_log ( id UUID PRIMARY KEY, timestamp TIMESTAMPTZ NOT NULL, @@ -624,6 +633,7 @@ CREATE TABLE admin_audit_log ( INDEX idx_audit_operation ON admin_audit_log(operation), INDEX idx_audit_namespace ON admin_audit_log(namespace) ); + ```text ## Security Considerations @@ -631,6 +641,7 @@ CREATE TABLE admin_audit_log ( ### Network Isolation ``` + # Kubernetes NetworkPolicy apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -648,11 +659,13 @@ spec: ports: - protocol: TCP port: 8981 + ```text ### Rate Limiting ``` + use governor::{Quota, RateLimiter}; pub struct RateLimitInterceptor { @@ -678,11 +691,13 @@ impl RateLimitInterceptor { Ok(()) } } + ```text ### TLS Configuration ``` + use tonic::transport::{Server, ServerTlsConfig}; let tls_config = ServerTlsConfig::new() @@ -694,6 +709,7 @@ Server::builder() .add_service(AdminServiceServer::new(admin_service)) .serve("[::]:8981".parse()?) .await?; + ```text ## Deployment @@ -701,6 +717,7 @@ Server::builder() ### Docker Compose ``` + services: prism-proxy: image: prism/proxy:latest @@ -718,11 +735,13 @@ services: networks: internal: internal: true + ```text ### Kubernetes ``` + apiVersion: apps/v1 kind: Deployment metadata: @@ -762,6 +781,7 @@ spec: - port: 8981 targetPort: 8981 name: admin + ```text ## Testing @@ -769,6 +789,7 @@ spec: ### Integration Tests ``` + func TestAdminProtocol(t *testing.T) { // Start mock OIDC server oidcServer := mockoidc.NewServer(t) @@ -801,6 +822,7 @@ func TestAdminProtocol(t *testing.T) { require.NoError(t, err) assert.Equal(t, "test-namespace", resp.Namespace.Name) } + ```text ## Open Questions @@ -838,10 +860,12 @@ func TestAdminProtocol(t *testing.T) { - Refresh tokens stored securely in `~/.prism/token` (mode 0600) - Configuration: ``` + token_cache: jwt_validation_ttl: 24h # How long to cache validated JWTs jwks_cache_ttl: 24h # How long to cache public keys auto_refresh: true # Automatically refresh expired tokens + ```text - **Security Trade-offs**: - Longer caching = better performance, but delayed revocation @@ -861,6 +885,7 @@ func TestAdminProtocol(t *testing.T) { - **No online validation** = Can't check real-time revocation - **Technology Stack**: ``` + use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm}; use reqwest::Client; @@ -903,6 +928,7 @@ func TestAdminProtocol(t *testing.T) { Ok(()) } } + ```text - **Trade-offs**: - βœ… **Pros**: Lower latency, no OIDC dependency per-request, better availability @@ -923,11 +949,13 @@ func TestAdminProtocol(t *testing.T) { - Example: Group `team-analytics` β†’ Can access `analytics` namespace - Configuration: ``` + namespace_access: analytics: groups: ["team-analytics", "platform-team"] user-profiles: groups: ["team-users", "platform-team"] + ```text - **Pros**: Simple, leverages existing IdP groups, easy to understand - **Cons**: Tight coupling to IdP group structure, requires group sync @@ -937,11 +965,13 @@ func TestAdminProtocol(t *testing.T) { - IdP adds custom claims during token issuance - Configuration: ``` + let authorized_namespaces = claims.custom .get("namespaces") .and_then(|v| v.as_array()) .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect()) .unwrap_or_default(); + ```text - **Pros**: Explicit, no group interpretation needed, flexible - **Cons**: Requires custom IdP configuration, claim size limits @@ -950,6 +980,7 @@ func TestAdminProtocol(t *testing.T) { - Policy checks: `allow if user.email in namespace.allowed_users` - Configuration: ``` + # OPA policy allow { input.user.email == "alice@company.com" @@ -959,6 +990,7 @@ func TestAdminProtocol(t *testing.T) { allow { "platform-team" in input.user.groups } + ```text - **Pros**: Most flexible, centralized policy management, audit trail - **Cons**: Additional dependency (OPA), higher latency, more complex @@ -968,11 +1000,13 @@ func TestAdminProtocol(t *testing.T) { - Example: `iss: https://tenant-analytics.idp.com` β†’ `analytics` namespace - Configuration: ``` + namespaces: analytics: oidc_issuer: https://tenant-analytics.idp.com user-profiles: oidc_issuer: https://tenant-users.idp.com + ```text - **Pros**: Strong isolation, tenant-specific policies, clear boundaries - **Cons**: Complex setup, multiple IdP integrations, higher overhead @@ -992,21 +1026,25 @@ func TestAdminProtocol(t *testing.T) { - No user interaction required (headless authentication) - Flow: ``` + curl -X POST https://idp.example.com/oauth/token \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=prism-ci-service" \ -d "client_secret=" \ -d "scope=admin:read admin:write" + ```text - Configuration: ``` + # CI/CD environment PRISM_CLIENT_ID=prism-ci-service PRISM_CLIENT_SECRET= # CLI auto-detects and uses client credentials prismctl --auth=client-credentials namespace list + ```text - **Pros**: Standard OAuth2 flow, widely supported, short-lived tokens - **Cons**: Secret management required, no refresh tokens (must re-authenticate) @@ -1015,6 +1053,7 @@ func TestAdminProtocol(t *testing.T) { - Keys stored in database, validated by Prism (not IdP) - Flow: ``` + # Generate key (admin operation) prismctl serviceaccount create ci-deploy --scopes admin:write # Returns: prism_key_abc123... @@ -1022,9 +1061,11 @@ func TestAdminProtocol(t *testing.T) { # Use key export PRISM_API_KEY=prism_key_abc123... prismctl namespace create prod-analytics + ```text - Configuration: ``` + CREATE TABLE service_accounts ( id UUID PRIMARY KEY, name VARCHAR(255) NOT NULL, @@ -1034,6 +1075,7 @@ func TestAdminProtocol(t *testing.T) { expires_at TIMESTAMPTZ, last_used_at TIMESTAMPTZ ); + ```text - **Pros**: Simple, no IdP dependency, fine-grained scopes - **Cons**: Not standard OAuth2, custom implementation, key rotation complexity @@ -1042,6 +1084,7 @@ func TestAdminProtocol(t *testing.T) { - Tokens automatically rotated by Kubernetes - Flow: ``` + # Pod spec volumes: - name: prism-token @@ -1054,6 +1097,7 @@ func TestAdminProtocol(t *testing.T) { # Mount at /var/run/secrets/prism/token # CLI auto-detects and uses + ```text - **Pros**: Automatic rotation, no secret management, K8s-native - **Cons**: K8s-only, requires TokenRequest API, audience configuration @@ -1062,11 +1106,13 @@ func TestAdminProtocol(t *testing.T) { - Fetch credentials at runtime, obtain token, use, discard - Configuration: ``` + # Fetch from Vault export PRISM_CLIENT_SECRET=$(vault kv get -field=secret prism/ci-service) # Obtain token (automatically by CLI) prismctl namespace list + ```text - **Pros**: Secrets never stored on disk, audit trail in secret manager - **Cons**: Dependency on secret manager, additional latency diff --git a/docs-cms/rfcs/rfc-011-data-proxy-authentication.md b/docs-cms/rfcs/rfc-011-data-proxy-authentication.md index 920f48482..0c3086a14 100644 --- a/docs-cms/rfcs/rfc-011-data-proxy-authentication.md +++ b/docs-cms/rfcs/rfc-011-data-proxy-authentication.md @@ -57,6 +57,7 @@ Client Service β†’ [mTLS] β†’ Prism Proxy β†’ [Backend Auth] β†’ Backends Input: mTLS certificates validate client identity Output: Backend-specific credentials (mTLS, passwords, API keys) + ```text ### Ports and Security Zones @@ -303,11 +304,13 @@ NATS | JWT | NKey | Vault/K8s Secret Redis | ACL + Password | None | Vault/K8s Secret SQLite | File permissions | None | N/A (local) S3 | IAM Role | Access Keys | Instance Profile + ```text ### Postgres Authentication ``` + sequenceDiagram participant Proxy as Prism Proxy participant Vault as HashiCorp Vault @@ -346,11 +349,13 @@ sequenceDiagram Vault-->>Proxy: New {username, password} Proxy->>Proxy: Update connection pool end + ```text ### Kafka Authentication (SASL/SCRAM) ``` + sequenceDiagram participant Proxy as Prism Proxy participant Vault as HashiCorp Vault @@ -375,11 +380,13 @@ sequenceDiagram Proxy->>Kafka: ProduceRequest
Topic: user-events Kafka->>Kafka: Check ACLs:
Can prism-kafka-xyz789 write to topic? Kafka-->>Proxy: ProduceResponse + ```text ### NATS Authentication (JWT) ``` + sequenceDiagram participant Proxy as Prism Proxy participant Vault as HashiCorp Vault @@ -396,11 +403,13 @@ sequenceDiagram Proxy->>NATS: PUB events.user.login 42
{user_id: "123", timestamp: ...} NATS->>NATS: Check permissions:
Can JWT publish to events.user.login? NATS-->>Proxy: +OK + ```text ### Redis Authentication (ACL) ``` + sequenceDiagram participant Proxy as Prism Proxy participant Vault as HashiCorp Vault @@ -421,11 +430,13 @@ sequenceDiagram Proxy->>Redis: SET cache:user:123:session Redis-->>Proxy: OK + ```text ### Credential Management ``` + use vaultrs::client::VaultClient; use vaultrs::kv2; @@ -509,11 +520,13 @@ impl CredentialManager { }); } } + ```text ## End-to-End Authentication Flow ``` + sequenceDiagram participant App as Application participant Proxy as Prism Proxy @@ -556,6 +569,7 @@ sequenceDiagram Proxy-->>App: PermissionDenied (7) end + ```text ## Secrets Provider Abstraction @@ -565,6 +579,7 @@ To support multiple secret management services (Vault, AWS Secrets Manager, Goog ### Secrets Provider Interface ``` + #[async_trait] pub trait SecretsProvider: Send + Sync { /// Fetch credentials for a backend @@ -591,6 +606,7 @@ pub struct Credentials { pub lease_id: Option, pub expires_at: Option>, } + ```text ### Provider Implementations @@ -598,6 +614,7 @@ pub struct Credentials { #### HashiCorp Vault Provider ``` + use vaultrs::client::VaultClient; pub struct VaultProvider { @@ -635,11 +652,13 @@ impl SecretsProvider for VaultProvider { "vault" } } + ```text #### AWS Secrets Manager Provider ``` + use aws_sdk_secretsmanager::Client as SecretsManagerClient; use aws_sdk_secretsmanager::types::SecretString; @@ -696,11 +715,13 @@ struct SecretData { api_key: Option, metadata: Option>, } + ```text #### Google Secret Manager Provider ``` + use google_secretmanager::v1::SecretManagerServiceClient; pub struct GcpSecretsProvider { @@ -753,11 +774,13 @@ impl SecretsProvider for GcpSecretsProvider { "gcp-secret-manager" } } + ```text #### Azure Key Vault Provider ``` + use azure_security_keyvault::KeyvaultClient; use azure_identity::DefaultAzureCredential; @@ -802,6 +825,7 @@ impl SecretsProvider for AzureKeyVaultProvider { "azure-keyvault" } } + ```text ### Provider Comparison @@ -820,6 +844,7 @@ impl SecretsProvider for AzureKeyVaultProvider { ### Provider Selection Strategy ``` + pub enum ProviderConfig { Vault { address: String, @@ -867,11 +892,13 @@ pub fn create_provider(config: &ProviderConfig) -> Result, credentials: Arc>>, @@ -970,6 +997,7 @@ impl CredentialManager { }); } } + ```text ## Configuration @@ -979,6 +1007,7 @@ impl CredentialManager { #### Option 1: HashiCorp Vault (Recommended for Dynamic Credentials) ``` + # prism-proxy.yaml data_port: 8980 admin_port: 8981 @@ -1030,11 +1059,13 @@ backends: connection: servers: [nats://nats-1:4222, nats://nats-2:4222] tls_required: true + ```text #### Option 2: AWS Secrets Manager (AWS Native) ``` + # prism-proxy.yaml data_port: 8980 admin_port: 8981 @@ -1077,10 +1108,12 @@ backends: brokers: [kafka-1:9092, kafka-2:9092, kafka-3:9092] security_protocol: SASL_SSL sasl_mechanism: SCRAM-SHA-512 + ```text **AWS Secrets Manager Secret Format** (JSON): ``` + { "username": "prism-postgres-user", "password": "securepassword123", @@ -1089,11 +1122,13 @@ backends: "environment": "production" } } + ```text #### Option 3: Google Secret Manager (GCP Native) ``` + # prism-proxy.yaml data_port: 8980 admin_port: 8981 @@ -1136,11 +1171,13 @@ backends: brokers: [kafka-1:9092, kafka-2:9092, kafka-3:9092] security_protocol: SASL_SSL sasl_mechanism: SCRAM-SHA-512 + ```text #### Option 4: Azure Key Vault (Azure Native) ``` + # prism-proxy.yaml data_port: 8980 admin_port: 8981 @@ -1182,6 +1219,7 @@ backends: brokers: [kafka-1:9092, kafka-2:9092, kafka-3:9092] security_protocol: SASL_SSL sasl_mechanism: SCRAM-SHA-512 + ```text ### Multi-Provider Deployment (Hybrid Cloud) @@ -1189,6 +1227,7 @@ backends: For hybrid cloud deployments, you can configure different providers per backend: ``` + # prism-proxy.yaml data_port: 8980 admin_port: 8981 @@ -1263,6 +1302,7 @@ backends: port: 5432 database: reports ssl_mode: require + ```text ## Security Considerations @@ -1286,6 +1326,7 @@ Automatic rotation schedule: 3. On renewal failure, fetch new credentials 4. Gracefully drain old connections 5. Old credentials revoked by Vault after TTL + ```text ### Audit Requirements @@ -1303,6 +1344,7 @@ Every data access must log: ### Integration Tests ``` + #[tokio::test] async fn test_mtls_authentication() { let ca = generate_test_ca(); @@ -1332,6 +1374,7 @@ async fn test_mtls_authentication() { assert!(bad_client.is_err()); } + ```text ## Open Questions diff --git a/docs-cms/rfcs/rfc-012-prism-netgw-control-plane.md b/docs-cms/rfcs/rfc-012-prism-netgw-control-plane.md index 87374ebab..a212b696a 100644 --- a/docs-cms/rfcs/rfc-012-prism-netgw-control-plane.md +++ b/docs-cms/rfcs/rfc-012-prism-netgw-control-plane.md @@ -76,11 +76,13 @@ Organizations deploying Prism at scale face several challenges: β”‚ (Postgres, β”‚ β”‚ (Kafka, β”‚ β”‚ (SQLite, β”‚ β”‚ Redis) β”‚ β”‚ NATS) β”‚ β”‚ Postgres) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Components ``` + graph TB subgraph "prism-netgw Control Plane" API[Control Plane API
:9980] @@ -117,6 +119,7 @@ graph TB Agent2 --> Prism5 Discovery -.->|Returns nearest| Prism1 Discovery -.->|Returns nearest| Prism4 + ```text ### Deployment Model @@ -213,6 +216,7 @@ message ConfigChange { **Push Model** (preferred): prism-netgw β†’ Watch(config_version) β†’ prism-agent ← ConfigUpdate stream ← + ```text **Pull Model** (fallback for high latency): @@ -232,9 +236,11 @@ Global Health (prism-netgw) β”‚ β”‚ └── Namespace Health (operational state) β”‚ └── Network Health (connectivity, latency) └── Control Plane Health (netgw nodes) + ```text ``` + message ReportHealthRequest { string cluster_id = 1; google.protobuf.Timestamp timestamp = 2; @@ -261,6 +267,7 @@ message BackendHealth { double latency_ms = 3; string error_message = 4; } + ```text ### 4. Service Discovery @@ -268,6 +275,7 @@ message BackendHealth { **Goal**: Clients discover nearest healthy Prism gateway. ``` + message DiscoverGatewaysRequest { string namespace = 1; // Filter by namespace support string client_location = 2; // "us-east-1", "eu-west-1", etc. @@ -287,10 +295,12 @@ message Gateway { HealthStatus health = 6; int32 load_score = 7; // 0-100 (lower is better) } + ```text **DNS-based discovery** (alternative): ``` + # Round-robin DNS for Prism gateways dig prism.example.com # β†’ 10.0.1.10 (us-east-1) @@ -300,6 +310,7 @@ dig prism.example.com dig prism.example.com # β†’ 10.0.1.10 (us-east-1) [if client in North America] # β†’ 10.0.2.20 (eu-west-1) [if client in Europe] + ```text ### 5. Cross-Region Routing @@ -313,6 +324,7 @@ dig prism.example.com 3. **Data Replication**: Namespace replicated across regions (lowest latency, eventual consistency) ``` + message RouteRequest { string namespace = 1; string client_region = 2; @@ -329,6 +341,7 @@ message RouteResponse { string target_gateway = 2; repeated string fallback_gateways = 3; } + ```text ## Latency and Partition Tolerance @@ -339,30 +352,38 @@ message RouteResponse { 1. **Async Configuration Push**: Don't block on config sync ``` + prism-netgw: Config updated (version 123) β†’ Async push to all clusters (fire-and-forget) β†’ Eventually consistent (all clusters converge to version 123) + ```text 2. **Heartbeat with Jitter**: Randomize heartbeat intervals to avoid thundering herd ``` + let heartbeat_interval = Duration::from_secs(30); let jitter = Duration::from_secs(rand::thread_rng().gen_range(0..10)); sleep(heartbeat_interval + jitter).await; + ```text 3. **Batch Updates**: Accumulate config changes and push in batches ``` + Instead of: 10 individual namespace updates (10 round trips) Do: 1 batch with 10 namespace updates (1 round trip) + ```text 4. **Caching**: Prism clusters cache config locally (survive netgw downtime) ``` + prism-agent: - Fetches config from netgw periodically - Caches config on disk - Uses cached config if netgw unavailable + ```text ### Handling Network Partitions @@ -404,6 +425,7 @@ Partition scenario: Group B: ap-south-1, ap-northeast-1 (2 nodes, NO QUORUM) β†’ becomes followers Result: Only Group A can make config changes (split-brain prevented) + ```text ## API Specification @@ -411,6 +433,7 @@ Result: Only Group A can make config changes (split-brain prevented) ### gRPC Service Definition ``` + syntax = "proto3"; package prism.netgw.v1; @@ -443,6 +466,7 @@ service ControlPlaneService { // Metrics rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse); } + ```text ## Deployment @@ -450,6 +474,7 @@ service ControlPlaneService { ### Kubernetes Deployment (Multi-Region) ``` + # Deploy netgw control plane in multiple regions apiVersion: apps/v1 kind: StatefulSet @@ -486,11 +511,13 @@ spec: resources: requests: storage: 10Gi + ```text ### Agent Deployment (Per Cluster) ``` + # Deploy prism-agent on each Prism gateway cluster apiVersion: apps/v1 kind: DaemonSet @@ -518,6 +545,7 @@ spec: volumes: - name: config-cache emptyDir: {} + ```text ## Security Considerations @@ -527,11 +555,13 @@ spec: All communication between netgw and agents uses mTLS: ``` + tls: server_cert: /etc/netgw/tls/server.crt server_key: /etc/netgw/tls/server.key client_ca: /etc/netgw/tls/ca.crt client_cert_required: true + ```text ### 2. Authentication @@ -590,6 +620,7 @@ prism_netgw_config_sync_latency_ms{cluster_id="..."} 234 prism_netgw_heartbeat_success_total{cluster_id="..."} 12345 prism_netgw_heartbeat_failure_total{cluster_id="..."} 3 prism_netgw_heartbeat_latency_ms{cluster_id="..."} 156 + ```text ### Distributed Tracing diff --git a/docs-cms/rfcs/rfc-013-neptune-graph-backend.md b/docs-cms/rfcs/rfc-013-neptune-graph-backend.md index a7a767023..0f274d6dc 100644 --- a/docs-cms/rfcs/rfc-013-neptune-graph-backend.md +++ b/docs-cms/rfcs/rfc-013-neptune-graph-backend.md @@ -405,6 +405,7 @@ func (p *NeptunePlugin) BulkImport(ctx context.Context, s3Path string) error { ``` **CSV Format** for bulk load: + ```csv ~id,~label,name:String,age:Int user:1,User,Alice,30 @@ -433,6 +434,7 @@ neptune_config: ### Query Optimization **1. Use indexes for frequent lookups**: + ```gremlin // Bad: Full scan g.V().has('email', 'alice@example.com') @@ -442,6 +444,7 @@ g.V('user:alice@example.com') ``` **2. Limit traversal depth**: + ```gremlin // Bad: Unbounded traversal g.V('user:alice').repeat(out('FOLLOWS')).until(has('name', 'target')) @@ -451,6 +454,7 @@ g.V('user:alice').repeat(out('FOLLOWS')).times(3).has('name', 'target') ``` **3. Use projection to reduce data transfer**: + ```gremlin // Bad: Return full vertex g.V().hasLabel('User') @@ -514,6 +518,7 @@ g.V().has('email', 'alice@example.com').profile() Step Count Traversers Time (ms) ===================================================== NeptuneGraphStep(vertex,[email.eq(alice)]) 1 1 2.345 + ```text ## Testing Strategy @@ -521,6 +526,7 @@ NeptuneGraphStep(vertex,[email.eq(alice)]) 1 1 2.345 ### Unit Tests ``` + func TestCreateVertex(t *testing.T) { plugin := setupNeptunePlugin(t) @@ -536,11 +542,13 @@ func TestCreateVertex(t *testing.T) { require.NoError(t, err) assert.Equal(t, "user:test1", resp.Vertex.Id) } + ```text ### Integration Tests ``` + func TestGraphTraversal(t *testing.T) { plugin := setupRealNeptune(t) // Connect to test Neptune cluster @@ -565,6 +573,7 @@ func TestGraphTraversal(t *testing.T) { require.NoError(t, err) assert.Contains(t, resp.Vertices, vertexWithId("C")) } + ```text ## Migration Path diff --git a/docs-cms/rfcs/rfc-014-layered-data-access-patterns.md b/docs-cms/rfcs/rfc-014-layered-data-access-patterns.md index d3b629610..71d47c7f8 100644 --- a/docs-cms/rfcs/rfc-014-layered-data-access-patterns.md +++ b/docs-cms/rfcs/rfc-014-layered-data-access-patterns.md @@ -51,6 +51,7 @@ Modern distributed applications require complex reliability patterns, but implem β”‚ Kafka | NATS | Postgres | Redis | S3 | ClickHouse β”‚ β”‚ "Connect to and execute operations on backend" β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text **Goals:** @@ -84,6 +85,7 @@ You need **guaranteed durability** for operations - write operations must be per **Why WAL is Critical for Reliability:** ``` + sequenceDiagram participant App as Application participant Proxy as Prism Proxy @@ -110,10 +112,12 @@ sequenceDiagram Note over App,Backend: Crash Recovery Proxy->>WAL: On restart: Read uncommitted WAL entries Proxy->>Backend: Replay to backend + ```text **Client Configuration:** ``` + # Declare what you need - Prism handles the rest namespaces: - name: order-processing @@ -125,10 +129,12 @@ namespaces: replay: enabled # β†’ Prism keeps committed log for replay retention: 30days # β†’ Prism retains WAL for 30 days ordered: true # β†’ Prism guarantees order + ```text **Client Code (Producer):** ``` + # Application publishes operation - Prism ensures durability order = {"order_id": 12345, "amount": 99.99, "status": "pending"} @@ -140,10 +146,12 @@ response = client.publish("orders", order) # βœ“ Can be replayed if needed print("Order persisted:", response.offset) + ```text **Client Code (Consumer):** ``` + # Consumer must explicitly acknowledge each operation for operation in client.consume("orders"): try: @@ -157,6 +165,7 @@ for operation in client.consume("orders"): # On failure: don't ack, operation will be retried logging.error(f"Failed to process order: {e}") operation.nack() # Explicit negative ack (immediate retry) + ```text **What Happens Internally:** @@ -218,6 +227,7 @@ You need to publish large files (videos, ML models, datasets) but message queues - Automatic cleanup after consumption **Client Configuration:** + ```text namespaces: - name: video-processing @@ -230,6 +240,7 @@ namespaces: ``` **Client Code:** + ```text # Publisher: Send large video file (no special handling) video_bytes = open("movie.mp4", "rb").read() # 2.5GB @@ -253,6 +264,7 @@ process_video(video_bytes) Problem: ML team publishes 50GB model weights per training run Before Prism: Manual S3 upload + manual message with S3 key With Prism: Standard publish() API, Prism handles everything + ```text ### Pattern 3: Transactional Messaging (Outbox) @@ -268,6 +280,7 @@ You need guaranteed message delivery when updating database - no lost messages, **Client Configuration:** ``` + namespaces: - name: order-processing pattern: queue @@ -275,10 +288,12 @@ namespaces: needs: consistency: strong # β†’ Prism adds Outbox pattern delivery_guarantee: exactly_once + ```text **Client Code:** ``` + # Application code: Atomically update DB and publish event with client.transaction() as tx: # 1. Update database @@ -292,6 +307,7 @@ with client.transaction() as tx: tx.commit() # Prism handles background publishing from outbox table + ```text **What Happens Internally:** @@ -318,6 +334,7 @@ You need to stream database changes to other systems (cache, search index, analy - Guaranteed ordering per key **Client Configuration:** + ```text namespaces: - name: user-profiles @@ -336,6 +353,7 @@ namespaces: ``` **Client Code:** + ```text # Application: Normal database operations (no CDC code!) client.update("user_profiles", user_id, {"email": "new@email.com"}) @@ -367,6 +385,7 @@ def cache_invalidator(): Problem: Keep Elasticsearch search index synced with PostgreSQL Before Prism: Dual write (update DB, update ES) - race conditions With Prism: CDC automatically streams changes, ES consumes + ```text ### Pattern 5: Transactional Large Payloads (Outbox + Claim Check) @@ -382,6 +401,7 @@ You need BOTH transactional guarantees (outbox) AND large payload support (claim **Client Configuration:** ``` + namespaces: - name: ml-model-releases pattern: pubsub @@ -390,10 +410,12 @@ namespaces: consistency: strong # β†’ Prism adds Outbox max_message_size: 5GB # β†’ Prism adds Claim Check delivery_guarantee: exactly_once + ```text **Client Code:** ``` + # Application: Publish large model with transactional guarantee model_weights = load_model("model-v2.weights") # 2GB @@ -414,6 +436,7 @@ with client.transaction() as tx: tx.commit() # If commit succeeds: model will be published # If commit fails: S3 object is cleaned up, no message sent + ```text **What Happens Internally:** @@ -441,6 +464,7 @@ You need fast cached reads but cache must stay fresh when database changes. - No application cache management code **Client Configuration:** + ```text namespaces: - name: product-catalog @@ -459,6 +483,7 @@ namespaces: ``` **Client Code:** + ```text # Application: Just read - Prism handles caching product = client.get("products", product_id) @@ -482,6 +507,7 @@ product = client.get("products", product_id) # Gets updated price Problem: Product catalog with millions of reads/sec, frequent price updates Before Prism: Manual cache + manual invalidation, stale data bugs With Prism: Declare cache + CDC, Prism handles everything + ```text ### Pattern Selection Guide @@ -501,6 +527,7 @@ With Prism: Declare cache + CDC, Prism handles everything Application owners declare **requirements**, Prism selects **patterns**: ``` + # Application declares "what" they need namespaces: - name: video-uploads @@ -514,6 +541,7 @@ namespaces: # Internally translates to: # patterns: [WAL, Outbox, Claim Check, Tiered Storage] # backend: [Kafka, S3, Postgres] + ```text Application owners **never write pattern composition logic** - they declare needs, Prism handles the rest. @@ -525,6 +553,7 @@ Application owners **never write pattern composition logic** - they declare need The Prism proxy is structured to cleanly separate concerns across layers: ``` + graph TB subgraph "External" Client[Client Application] @@ -620,11 +649,13 @@ graph TB PatternChain -.->|Emit| Metrics PatternChain -.->|Emit| Traces PatternChain -.->|Emit| Logs + ```text ### Authentication and Authorization Flow ``` + sequenceDiagram participant Client participant gRPC as gRPC Server @@ -660,11 +691,13 @@ sequenceDiagram JWT-->>Auth: Error Auth-->>Client: Unauthenticated (16) end + ```text ### Pattern Layer Execution Flow ``` + sequenceDiagram participant API as API Layer (Layer 3) participant Chain as Pattern Chain @@ -710,11 +743,13 @@ sequenceDiagram P1->>Backend: UPDATE outbox published_at P1->>Obs: Metric: outbox_published++ end + ```text ### Pattern Routing and Backend Execution ``` + graph LR subgraph "Pattern Layer" Input[Pattern Input
Context] @@ -772,6 +807,7 @@ graph LR Transact --> PGOps Stream --> PGOps Batch --> RedisOps + ```text ### Three-Layer Model @@ -781,6 +817,7 @@ graph LR The **What** layer - defines the interface applications use: ``` + // Example: PubSub Service service PubSubService { rpc Publish(PublishRequest) returns (PublishResponse); @@ -792,6 +829,7 @@ message PublishRequest { bytes payload = 2; // Application doesn't know about Claim Check map metadata = 3; } + ```text **Key Characteristics:** @@ -805,6 +843,7 @@ message PublishRequest { The **How** layer - implements reliability patterns transparently: ``` + # Namespace configuration namespaces: - name: video-processing @@ -826,11 +865,13 @@ namespaces: backend: queue: kafka topic_prefix: video + ```text **Pattern Execution Order:** ``` + sequenceDiagram participant App as Application participant API as Layer 3: PubSub API @@ -851,6 +892,7 @@ sequenceDiagram Backend-->>Pat2: Acknowledged Pat2-->>API: Success API-->>App: PublishResponse + ```text #### Layer 1: Backend Execution (Implementation) @@ -858,6 +900,7 @@ sequenceDiagram The **Where** layer - connects to and executes on specific backends: ``` + // Backend-specific implementation impl KafkaBackend { async fn publish(&self, topic: &str, payload: &[u8]) -> Result { @@ -867,6 +910,7 @@ impl KafkaBackend { .map_err(|e| Error::Backend(e)) } } + ```text **Key Characteristics:** @@ -904,6 +948,7 @@ Not all patterns can be layered together. Compatibility depends on: #### Without Layering (Application Code) ``` + # Application must implement Claim Check manually def publish_video(video_id, video_bytes): if len(video_bytes) > 1_000_000: # > 1MB @@ -937,6 +982,7 @@ def consume_video(): video_bytes = msg["payload"] process_video(video_bytes) + ```text **Problems**: @@ -949,6 +995,7 @@ def consume_video(): **Configuration**: ``` + namespaces: - name: video-processing client_api: pubsub @@ -968,10 +1015,12 @@ namespaces: type: kafka brokers: [kafka-1:9092, kafka-2:9092] topic: videos + ```text **Application Code**: ``` + # Producer: Prism handles Claim Check automatically client.publish("videos", video_bytes) # Prism: @@ -983,6 +1032,7 @@ client.publish("videos", video_bytes) event = client.subscribe("videos") video_bytes = event.payload # Prism fetched from S3 automatically process_video(video_bytes) + ```text **Benefits**: @@ -999,6 +1049,7 @@ process_video(video_bytes) #### Pattern Layering ``` + sequenceDiagram participant App as Application participant Prism as Prism Proxy @@ -1031,10 +1082,12 @@ sequenceDiagram Prism->>DB: UPDATE outbox
SET published_at = NOW() end + ```text **Configuration**: ``` + namespaces: - name: ml-model-releases client_api: pubsub @@ -1060,6 +1113,7 @@ namespaces: backend: type: kafka topic: model-releases + ```text **Guarantees**: @@ -1075,6 +1129,7 @@ namespaces: #### Pattern Composition ``` + namespaces: - name: user-profiles client_api: reader @@ -1108,11 +1163,13 @@ namespaces: backend: type: postgres database: users_db + ```text **Data Flow**: ``` + graph LR App[Application] Prism[Prism Proxy] @@ -1136,6 +1193,7 @@ graph LR CDC -->|Parse changes| Kafka Kafka -->|Subscribe| Invalidator Invalidator -->|DEL user:123:profile| Cache + ```text **Benefits**: @@ -1152,10 +1210,12 @@ graph LR **Client API (Layer 3)** - Stable interface: ``` + service QueueService { rpc Publish(PublishRequest) returns (PublishResponse); rpc Subscribe(SubscribeRequest) returns (stream Message); } + ```text **Backend Strategy (Layer 2 + 1)** - Implementation details: @@ -1170,9 +1230,11 @@ service QueueService { **Application doesn't know which strategy** - same API for all: ``` + # Works with ANY backend strategy client.publish("events", payload) messages = client.subscribe("events") + ```text ### Pattern Configuration Encapsulation @@ -1180,6 +1242,7 @@ messages = client.subscribe("events") Applications declare requirements; Prism selects patterns: ``` + # Application-facing configuration namespaces: - name: video-processing @@ -1200,6 +1263,7 @@ namespaces: backend: type: kafka partitions: 20 # For throughput + ```text ## Stitching Patterns Together @@ -1209,6 +1273,7 @@ namespaces: Each pattern implements standard interfaces for composability: ``` + /// Pattern trait for composing reliability patterns #[async_trait] pub trait Pattern: Send + Sync { @@ -1240,11 +1305,13 @@ pub struct ConsumeContext { pub payload: Vec, pub metadata: HashMap, } + ```text ### Example: Claim Check Pattern Implementation ``` + pub struct ClaimCheckPattern { threshold: usize, storage: Arc, @@ -1293,6 +1360,7 @@ impl Pattern for ClaimCheckPattern { } } } + ```text ### Pattern Chain Execution @@ -1300,6 +1368,7 @@ impl Pattern for ClaimCheckPattern { Prism executes patterns in order: ``` + pub struct PatternChain { patterns: Vec>, } @@ -1321,6 +1390,7 @@ impl PatternChain { Ok(ctx) } } + ```text **Example execution with Outbox + Claim Check**: @@ -1655,6 +1725,7 @@ prism_pattern_outbox_pending_count{namespace="videos"} 5 # Pattern Chain prism_pattern_chain_duration_seconds{namespace="videos", pattern="claim-check"} 0.042 prism_pattern_chain_duration_seconds{namespace="videos", pattern="outbox"} 0.008 + ```text ### Distributed Tracing diff --git a/docs-cms/rfcs/rfc-015-plugin-acceptance-test-framework.md b/docs-cms/rfcs/rfc-015-plugin-acceptance-test-framework.md index 276205113..da1282247 100644 --- a/docs-cms/rfcs/rfc-015-plugin-acceptance-test-framework.md +++ b/docs-cms/rfcs/rfc-015-plugin-acceptance-test-framework.md @@ -37,6 +37,7 @@ The framework provides: MEMO-006 decomposes backends into thin interfaces (e.g., Redis implements 16 interfaces across 6 data models), but without interface-level testing, we can't verify compliance: **Current Approach (Backend-Type Testing)**: + ```go func TestPostgresPlugin(t *testing.T) { // Tests all PostgreSQL features mixed together @@ -54,6 +55,7 @@ func TestPostgresPlugin(t *testing.T) { - Hard to verify partial interface implementations **MEMO-006 Approach (Interface-Based Testing)**: + ```go // Test keyvalue_basic interface (works for ANY backend implementing it) func TestKeyValueBasicInterface(t *testing.T, backend TestBackend) { @@ -1163,6 +1165,7 @@ jobs: **Problem**: Backend claims to implement `keyvalue_scan` but actually doesn't support prefix filtering. **Solution**: Interface test suite validates all operations defined in `keyvalue_scan.proto`: + ```go func (s *KeyValueScanTestSuite) Run() { s.t.Run("ScanAll", s.testScanAll) @@ -1189,6 +1192,7 @@ func (s *KeyValueScanTestSuite) Run() { ### 3. Clear Contract Definition Interface test suites serve as executable specifications: + ```go // KeyValueBasicTestSuite defines EXACTLY what keyvalue_basic means: // 1. Set(key, value) stores a value @@ -1205,6 +1209,7 @@ Backends implementing `keyvalue_basic` MUST pass all 10 tests. ### 4. Incremental Implementation Backends can implement subsets of interfaces: + ```yaml # registry/backends/memstore.yaml implements: @@ -1222,6 +1227,7 @@ Tests run ONLY for declared interfaces - no false failures. ### 5. Registry-Driven Testing Backend registry (`registry/backends/*.yaml`) is single source of truth: + ```yaml # registry/backends/redis.yaml backend: redis @@ -1420,6 +1426,7 @@ Implement test suites for all 45 interfaces: ### Phase 2: Performance Baseline Tests (Q2 2026) Add performance benchmarks to interface tests: + ```go func (s *KeyValueBasicTestSuite) BenchmarkSet() { // Verify latency < P99 SLO (5ms) diff --git a/docs-cms/rfcs/rfc-016-local-development-infrastructure.md b/docs-cms/rfcs/rfc-016-local-development-infrastructure.md index 45078e5af..d60d2630b 100644 --- a/docs-cms/rfcs/rfc-016-local-development-infrastructure.md +++ b/docs-cms/rfcs/rfc-016-local-development-infrastructure.md @@ -144,6 +144,7 @@ local-dev/ └── all/ β”œβ”€β”€ docker-compose.all.yml # Start everything └── Makefile + ```text **Benefits:** @@ -159,6 +160,7 @@ local-dev/ ### Quick Start ``` + # Start Signoz cd local-dev/signoz docker-compose -f docker-compose.signoz.yml up -d @@ -168,27 +170,32 @@ open http://localhost:3301 # Configure Prism to send telemetry export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 + ```text ### Integration Points **Prism Proxy:** ``` + // Automatically uses OTEL_EXPORTER_OTLP_ENDPOINT if set // proxy/src/main.rs fn main() { init_observability()?; // Sets up OpenTelemetry // ... } + ```text **Backend Plugins:** ``` + // plugins//main.go func main() { observability.InitTracer("prism-plugin-postgres") // Reads OTEL_EXPORTER_OTLP_ENDPOINT from environment } + ```text ### Resource Usage @@ -205,6 +212,7 @@ func main() { ### Architecture ``` + sequenceDiagram participant Dev as Developer participant CLI as prismctl @@ -229,6 +237,7 @@ sequenceDiagram Dex-->>Admin: Valid Admin-->>CLI: Response CLI-->>Dev: Result + ```text ### Dex Configuration @@ -236,6 +245,7 @@ sequenceDiagram Location: `local-dev/dex/dex-config.yaml` ``` + issuer: http://localhost:5556/dex storage: @@ -287,6 +297,7 @@ connectors: # Enable password grant (for automated testing) enablePasswordDB: true + ```text ### Static Developer User @@ -294,6 +305,7 @@ enablePasswordDB: true Location: `local-dev/dex/static-users.yaml` ``` + # Pre-provisioned developer user # Password is bcrypt hash of "devpass" staticPasswords: @@ -312,6 +324,7 @@ staticPasswords: userID: "a0b1c2d3-e4f5-6789-0123-456789abcdef" groups: - "prism:viewer" + ```text **Default Credentials:** @@ -324,6 +337,7 @@ staticPasswords: Location: `local-dev/dex/docker-compose.dex.yml` ``` + version: '3.8' services: @@ -359,6 +373,7 @@ networks: prism: driver: bridge external: false + ```text ### Developer Identity Auto-Provisioning @@ -370,6 +385,7 @@ networks: #### Implementation: Auto-Login Flow ``` + // cli/src/auth/auto_login.rs pub struct AutoLoginConfig { @@ -429,11 +445,13 @@ fn is_local_dev() -> bool { .map(|v| v == "true") .unwrap_or(false) } + ```text #### CLI Integration ``` + // cli/src/commands/namespace.rs async fn list_namespaces(ctx: &Context) -> Result<()> { @@ -469,11 +487,13 @@ async fn list_namespaces(ctx: &Context) -> Result<()> { println!("{}", format_namespaces(&namespaces)); Ok(()) } + ```text #### Environment Variable Toggle ``` + # Enable auto-login (default for local dev) export PRISM_LOCAL_DEV=true @@ -487,6 +507,7 @@ prism namespace list # NAME STATUS BACKEND # analytics active postgres # events active kafka + ```text ### Resource Usage @@ -518,6 +539,7 @@ prism namespace list ### Update Process ``` + # Update Signoz cd local-dev/signoz docker-compose -f docker-compose.signoz.yml pull @@ -531,6 +553,7 @@ docker-compose -f docker-compose.dex.yml up -d # Update all cd local-dev make update-support-services + ```text ### Version Tracking @@ -538,6 +561,7 @@ make update-support-services Location: `local-dev/versions.yaml` ``` + # Prism Local Development Support Services # Last updated: 2025-10-09 @@ -574,11 +598,13 @@ backend_testing: kafka: version: "7.5.0" # See MEMO-004 for complete backend versions + ```text ### Health Checks ``` + # Check all support services cd local-dev ./scripts/health-check.sh @@ -588,6 +614,7 @@ cd local-dev # βœ… Signoz OTLP (grpc://localhost:4317): healthy # βœ… Dex OIDC (http://localhost:5556): healthy # βœ… Dex well-known config: valid + ```text ## Usage Patterns @@ -595,6 +622,7 @@ cd local-dev ### Pattern 1: Start Everything ``` + # Start all support services cd local-dev make dev-up @@ -602,11 +630,13 @@ make dev-up # Equivalent to: # docker-compose -f signoz/docker-compose.signoz.yml up -d # docker-compose -f dex/docker-compose.dex.yml up -d + ```text ### Pattern 2: Start Only Signoz ``` + # Just observability, no auth cd local-dev/signoz docker-compose -f docker-compose.signoz.yml up -d @@ -614,11 +644,13 @@ docker-compose -f docker-compose.signoz.yml up -d # Start Prism without Dex (local mode, no auth) export PRISM_AUTH_DISABLED=true cargo run --release + ```text ### Pattern 3: Start Only Dex ``` + # Just auth, no observability cd local-dev/dex docker-compose -f docker-compose.dex.yml up -d @@ -626,11 +658,13 @@ docker-compose -f docker-compose.dex.yml up -d # Test OIDC flow prism auth login # Opens browser to http://localhost:5556/dex/auth + ```text ### Pattern 4: Full Stack with Backends ``` + # Start support services make dev-up @@ -642,6 +676,7 @@ docker-compose -f docker-compose.backends.yml up redis -d export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 export PRISM_LOCAL_DEV=true cargo run --release + ```text ## Configuration Injection @@ -652,6 +687,7 @@ Support services expose configuration via standard environment variables: **Signoz (OpenTelemetry):** ``` + # OTLP endpoint for Prism components export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 @@ -662,10 +698,12 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 # Environment label export OTEL_RESOURCE_ATTRIBUTES="deployment.environment=local" + ```text **Dex (OIDC):** ``` + # Issuer URL export PRISM_OIDC_ISSUER=http://localhost:5556/dex @@ -674,12 +712,14 @@ export PRISM_OIDC_CLIENT_ID=prismctl # Auto-login (local dev only) export PRISM_LOCAL_DEV=true + ```text ### Configuration Files **Prism Proxy Config:** ``` + # proxy/config.yaml observability: tracing: @@ -695,10 +735,12 @@ authentication: issuer: ${PRISM_OIDC_ISSUER:-http://localhost:5556/dex} client_id: prism-proxy # Auto-discover JWKS from .well-known/openid-configuration + ```text **CLI Config:** ``` + # ~/.prism/config.yaml admin: endpoint: http://localhost:8090 @@ -708,6 +750,7 @@ auth: issuer: http://localhost:5556/dex client_id: prismctl auto_login: true # Local dev only + ```text ## Developer Workflows @@ -715,6 +758,7 @@ auth: ### Workflow 1: First-Time Setup ``` + # Clone repository git clone https://github.com/org/prism.git cd prism @@ -736,11 +780,13 @@ make bootstrap # πŸ”­ Signoz UI: http://localhost:3301 # πŸ” Dex OIDC: http://localhost:5556/dex # πŸ‘€ Auto-authenticated as: dev@local.prism + ```text ### Workflow 2: Daily Development ``` + # Start support services (if not running) make dev-up @@ -756,11 +802,13 @@ go run ./cmd/server prism namespace list # Auto-authenticates prism session list prism health + ```text ### Workflow 3: Testing OIDC Integration ``` + # Start Dex cd local-dev/dex docker-compose -f docker-compose.dex.yml up -d @@ -783,11 +831,13 @@ cat ~/.prism/token # "refresh_token": "Cgk...", # "expires_at": "2025-10-09T11:00:00Z" # } + ```text ### Workflow 4: Debugging with Traces ``` + # Ensure Signoz is running make dev-signoz-up @@ -808,6 +858,7 @@ open http://localhost:3301 # Filter by service: prism-proxy # Click trace to see full waterfall showing: # prism-proxy β†’ admin-service β†’ postgres + ```text ## Production Considerations diff --git a/docs-cms/rfcs/rfc-017-multicast-registry-pattern.md b/docs-cms/rfcs/rfc-017-multicast-registry-pattern.md index ab152fb20..b65a29783 100644 --- a/docs-cms/rfcs/rfc-017-multicast-registry-pattern.md +++ b/docs-cms/rfcs/rfc-017-multicast-registry-pattern.md @@ -97,6 +97,7 @@ operations: ``` **Example**: + ```python # Service A registers registry.register( @@ -143,6 +144,7 @@ operations: ``` **Example**: + ```python # IoT device registers registry.register( @@ -188,6 +190,7 @@ operations: ``` **Example**: + ```python # User joins chat room registry.register( @@ -306,6 +309,7 @@ message Identity { ``` **Filter Syntax**: + ```javascript // Equality {"service_name": "payment-service"} @@ -420,6 +424,7 @@ The Multicast Registry pattern **composes** three data access primitives: β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ Coordinated by Proxy β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ```text ### Primitive Mapping @@ -437,6 +442,7 @@ The Multicast Registry pattern **composes** three data access primitives: The pattern defines **three backend slots** that can be filled independently: ``` + pattern: multicast-registry backend_slots: @@ -454,6 +460,7 @@ backend_slots: purpose: Persist undelivered messages (optional) operations: [enqueue, dequeue, ack] candidates: [kafka, postgres, sqs, redis-stream] + ```text **Key Design Principle**: The same client API works with different backend combinations, allowing trade-offs between consistency, durability, and performance. diff --git a/docs-cms/rfcs/rfc-018-poc-implementation-strategy.md b/docs-cms/rfcs/rfc-018-poc-implementation-strategy.md index 41fcf3ac6..a4599bf41 100644 --- a/docs-cms/rfcs/rfc-018-poc-implementation-strategy.md +++ b/docs-cms/rfcs/rfc-018-poc-implementation-strategy.md @@ -594,6 +594,7 @@ The sections below show the original plan with actual completion status: **3. Python Client Library** (`clients/python/`) - βœ… Connect to proxy via gRPC - βœ… KeyValue pattern API: + ```python client = PrismClient("localhost:8980") await client.keyvalue.set("key1", b"value1") @@ -601,6 +602,7 @@ The sections below show the original plan with actual completion status: await client.keyvalue.delete("key1") keys = await client.keyvalue.scan("prefix*") ``` + - ❌ No retry logic (defer) - ❌ No connection pooling (single connection) @@ -632,6 +634,7 @@ The sections below show the original plan with actual completion status: 4. βœ… Graceful shutdown **Validation Tests**: + ```python # tests/poc1/test_keyvalue_memstore.py @@ -823,6 +826,7 @@ Demonstrate Prism working with a **real external backend** and introduce: 3. βœ… Plugin recovers from Redis connection loss **Validation Tests**: + ```go // tests/acceptance/redis_test.go @@ -861,6 +865,7 @@ func TestRedisPlugin_KeyValue(t *testing.T) { - Standalone binary: `patterns/redis/redis` **Key Configuration**: + ```go type Config struct { Address string // "localhost:6379" @@ -1052,6 +1057,7 @@ Demonstrate **second client pattern** (PubSub) and introduce: **3. Extend Python Client** (`clients/python/`) - βœ… Add PubSub API: + ```python await client.pubsub.publish("events", b"message") @@ -1077,6 +1083,7 @@ Demonstrate **second client pattern** (PubSub) and introduce: 3. βœ… Handle 100 concurrent subscribers **Validation Tests**: + ```python # tests/poc3/test_pubsub_nats.py @@ -1144,6 +1151,7 @@ async def test_fanout(): - Health checks (healthy, degraded, unhealthy) **Key Configuration**: + ```go type Config struct { URL string // "nats://localhost:4222" @@ -1249,6 +1257,7 @@ type Config struct { - Context cancellation handles unsubscribe cleanly **Key Pattern**: + ```go sub, err := n.conn.Subscribe(topic, func(msg *nats.Msg) { select { @@ -1277,6 +1286,7 @@ sub, err := n.conn.Subscribe(topic, func(msg *nats.Msg) { - Subscriptions survive reconnection **Minimal Code**: + ```go opts := []nats.Option{ nats.MaxReconnects(10), @@ -1417,6 +1427,7 @@ Demonstrate **pattern composition** implementing RFC-017 with: **5. Extend Python Client** (`clients/python/`) - βœ… Add Multicast Registry API: + ```python await client.registry.register( identity="device-1", @@ -1452,6 +1463,7 @@ Demonstrate **pattern composition** implementing RFC-017 with: 3. βœ… Handle concurrent register/multicast operations **Validation Tests**: + ```python # tests/poc4/test_multicast_registry.py @@ -1567,6 +1579,7 @@ Week 7-9: POC 4 (Multicast Registry) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ Week 10-11: POC 5 (Authentication) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ 11 weeks total (2.75 months) + ```text ### Parallel Work Opportunities @@ -1625,6 +1638,7 @@ POC 2 (Redis) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ ### Development Workflow **1. Start with Tests**: + ```bash # Write test first cat > tests/poc1/test_keyvalue.py < anyhow::Result<()> { - [ ] Logs server startup **Task 2.4: Implement plugin client forwarding** (1 day) + ```rust // proxy/src/plugin.rs use prism_pb::key_value_basic_interface_client::KeyValueBasicInterfaceClient; @@ -515,6 +523,7 @@ impl PluginClient { - [ ] Retries connection on failure **Task 2.5: Add basic error handling** (half day) + ```rust // proxy/src/error.rs use thiserror::Error; @@ -561,6 +570,7 @@ impl From for tonic::Status { #### Tasks **Task 3.1: Setup Go project structure** (half day) + ```text plugins/memstore/ β”œβ”€β”€ go.mod # Module definition @@ -593,6 +603,7 @@ require ( - [ ] Hello world binary runs **Task 3.2: Implement KeyValue storage with sync.Map** (1 day) + ```go // plugins/memstore/storage/keyvalue.go package storage @@ -679,6 +690,7 @@ func (kv *KeyValueStore) setTTL(key string, duration time.Duration) { - [ ] Unit tests for all operations **Task 3.3: Implement List storage with slices** (1 day) + ```go // plugins/memstore/storage/list.go package storage @@ -774,6 +786,7 @@ func (ls *ListStore) Length(listKey string) int64 { - [ ] Unit tests for all operations **Task 3.4: Implement gRPC server** (1 day) + ```go // plugins/memstore/server.go package main @@ -857,6 +870,7 @@ func main() { #### Tasks **Task 4.1: Setup Python project structure** (half day) + ```text clients/python/ β”œβ”€β”€ pyproject.toml # Poetry/pip dependencies @@ -899,6 +913,7 @@ dev = [ - [ ] Can import `from prism import PrismClient` **Task 4.2: Implement PrismClient main class** (half day) + ```python # clients/python/prism/client.py import grpc @@ -940,6 +955,7 @@ class PrismClient: - [ ] Exposes `keyvalue` and `list` APIs **Task 4.3: Implement KeyValue API** (1 day) + ```python # clients/python/prism/keyvalue.py from typing import Optional, AsyncIterator @@ -1086,6 +1102,7 @@ class KeyValueAPI: - [ ] Type hints for all methods **Task 4.4: Implement List API** (half day) + ```python # clients/python/prism/list.py from typing import Optional @@ -1208,6 +1225,7 @@ class ListAPI: #### Tasks **Task 5.1: Write integration tests** (1 day) + ```python # tests/poc1/test_keyvalue_memstore.py import pytest @@ -1305,6 +1323,7 @@ async def test_list_stack(): - [ ] Tests run in CI **Task 5.2: Create demo script** (half day) + ```python # examples/poc1-demo.py """ @@ -1398,6 +1417,7 @@ if __name__ == "__main__": - [ ] Demonstrates TTL expiration **Task 5.3: Create README and documentation** (half day) + ```text # POC 1: KeyValue with MemStore @@ -1459,6 +1479,7 @@ Walking skeleton demonstrating Prism's end-to-end architecture. #### Tasks **Task 6.1: Create Docker Compose setup** (half day) + ```yaml # docker-compose.yml version: '3.8' @@ -1520,6 +1541,7 @@ networks: - [ ] `docker-compose down` stops cleanly **Task 6.2: Create Makefiles for development** (half day) + ```makefile # Makefile (root) .PHONY: all proto dev-up dev-down test demo clean diff --git a/docs-cms/rfcs/rfc-027-namespace-configuration-client-perspective.md b/docs-cms/rfcs/rfc-027-namespace-configuration-client-perspective.md index 46bc1db4f..6f06e22cf 100644 --- a/docs-cms/rfcs/rfc-027-namespace-configuration-client-perspective.md +++ b/docs-cms/rfcs/rfc-027-namespace-configuration-client-perspective.md @@ -399,6 +399,7 @@ Platform enforces authorization boundaries to prevent misconfiguration: - ❌ Cannot bypass capacity limits **Example**: + ```yaml # Allowed needs: @@ -422,6 +423,7 @@ backend: - Custom retention policies beyond standard limits **Example**: + ```yaml # Requires approval annotation needs: @@ -445,6 +447,7 @@ approval: - Bypass capacity guardrails **Example**: + ```yaml # Platform team only patterns: @@ -495,6 +498,7 @@ POST /api/v1/namespaces 8. Return namespace details to client **Client Receives**: + ```json { "namespace": "order-processing", @@ -742,6 +746,7 @@ dig prism.eu-west-1.example.com ``` **Client Usage**: + ```python # Client SDK auto-discovers endpoints client = PrismClient( @@ -777,6 +782,7 @@ Response: ``` **Client Usage**: + ```python client = PrismClient( namespace="order-processing", diff --git a/docs-cms/rfcs/rfc-029-load-testing-framework-evaluation.md b/docs-cms/rfcs/rfc-029-load-testing-framework-evaluation.md index ffd1fab90..0c3a3acc8 100644 --- a/docs-cms/rfcs/rfc-029-load-testing-framework-evaluation.md +++ b/docs-cms/rfcs/rfc-029-load-testing-framework-evaluation.md @@ -525,6 +525,7 @@ Prism needs **two distinct types** of load testing: **Tool**: Custom `prism-loadtest` βœ… **Architecture**: + ```text prism-loadtest β†’ Coordinator β†’ Redis/NATS ``` @@ -547,6 +548,7 @@ prism-loadtest β†’ Coordinator β†’ Redis/NATS **Tool**: `ghz` βœ… **Architecture**: + ```text ghz β†’ Rust Proxy (gRPC) β†’ Pattern (gRPC) β†’ Redis/NATS ``` @@ -621,6 +623,7 @@ ghz β†’ Rust Proxy (gRPC) β†’ Pattern (gRPC) β†’ Redis/NATS - [ ] Compare results (pattern-level vs integration) **Example ghz Test Suite**: + ```bash # tests/load/ghz/keyvalue.sh #!/bin/bash @@ -667,6 +670,7 @@ ghz --proto proto/interfaces/keyvalue_basic.proto \ **Tasks**: 1. Add ramp-up load profile + ```go // cmd/prism-loadtest/cmd/root.go rootCmd.PersistentFlags().String("profile", "constant", "Load profile: constant, ramp-up, spike") @@ -675,6 +679,7 @@ ghz --proto proto/interfaces/keyvalue_basic.proto \ ``` 2. Add JSON output format + ```go // cmd/prism-loadtest/cmd/output.go type JSONReport struct { @@ -693,6 +698,7 @@ ghz --proto proto/interfaces/keyvalue_basic.proto \ ``` 3. Add spike load profile + ```bash # 0-30s: 100 req/sec # 30-35s: 1000 req/sec (spike) @@ -706,17 +712,20 @@ ghz --proto proto/interfaces/keyvalue_basic.proto \ **Tasks**: 1. Install ghz + ```bash go install github.com/bojand/ghz/cmd/ghz@latest ``` 2. Create ghz test suite + ```bash mkdir -p tests/load/ghz # Create test scripts for each pattern ``` 3. Add to CI/CD + ```yaml # .github/workflows/load-test.yml - name: Run integration load tests @@ -742,6 +751,7 @@ ghz --proto proto/interfaces/keyvalue_basic.proto \ ``` 4. Compare results + ```bash # Pattern-level (no gRPC overhead) ./prism-loadtest register -r 100 -d 60s diff --git a/docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md b/docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md index 00ab79dc4..d26b27a14 100644 --- a/docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md +++ b/docs-cms/rfcs/rfc-030-schema-evolution-pubsub-validation.md @@ -21,6 +21,7 @@ This RFC addresses schema evolution and validation for publisher/consumer patter Prism's pub/sub and queue patterns intentionally decouple producers from consumers: **Current Architecture:** + ```text β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Producer App β”‚ β”‚ Consumer App β”‚ @@ -236,6 +237,7 @@ namespaces: ``` **Schema URL Format:** + ```text github.com/myorg/my-service/blob/main/schemas/events/orders.created.v2.proto github.com/myorg/my-service/blob/v2.1.0/schemas/events/orders.created.v2.proto # Tagged release diff --git a/docs-cms/rfcs/rfc-031-message-envelope-protocol.md b/docs-cms/rfcs/rfc-031-message-envelope-protocol.md index 72f956cfc..a416d1c8a 100644 --- a/docs-cms/rfcs/rfc-031-message-envelope-protocol.md +++ b/docs-cms/rfcs/rfc-031-message-envelope-protocol.md @@ -28,6 +28,7 @@ The envelope wraps user payloads with metadata (routing, schema, auth, observabi Current pub/sub implementations across different backends use inconsistent metadata formats: **Kafka** (message headers): + ```text schema-id: 123 correlation-id: abc-456 @@ -35,6 +36,7 @@ timestamp: 1697200000 ``` **NATS** (custom headers): + ```text Nats-Msg-Id: xyz-789 X-Schema-URL: github.com/... @@ -42,11 +44,13 @@ X-Trace-ID: trace-123 ``` **Redis** (no native headers, metadata in payload prefix): + ```text {"meta":{"schema":"v2","ts":1697200000},"payload":} ``` **PostgreSQL** (JSON columns): + ```sql INSERT INTO events (topic, payload, metadata) VALUES ('orders', '{...}', '{"schema_version":"v2"}'); ``` @@ -696,6 +700,7 @@ message SecurityContext { **Use Case:** High-throughput messaging where producer and consumer share a secret key **Configuration (Producer):** + ```yaml encryption: enabled: true @@ -705,6 +710,7 @@ encryption: ``` **Producer Code (Go):** + ```go import ( "crypto/aes" @@ -738,6 +744,7 @@ envelope.Payload = &Any{Value: ciphertext} ``` **Consumer Code (Go):** + ```go // Load SAME key from Vault using key_id reference keyId := envelope.Security.Encryption.KeyId @@ -760,6 +767,7 @@ if err != nil { **Use Case:** Producer doesn't trust consumer's key storage; only consumer can decrypt **Configuration (Producer):** + ```yaml encryption: enabled: true @@ -769,6 +777,7 @@ encryption: ``` **Configuration (Consumer):** + ```yaml encryption: enabled: true @@ -777,6 +786,7 @@ encryption: ``` **Producer Code (Go):** + ```go import ( "crypto/rand" @@ -808,6 +818,7 @@ envelope.Payload = &Any{Value: ciphertext} ``` **Consumer Code (Go):** + ```go // Load consumer's PRIVATE key (only consumer has this) privateKey := loadPrivateKeyFromVault("vault://secrets/consumers/order-processor/private-key") @@ -832,6 +843,7 @@ if err != nil { **Use Case:** Future-proof encryption resistant to quantum computer attacks **Configuration (Producer):** + ```yaml encryption: enabled: true @@ -841,6 +853,7 @@ encryption: ``` **Producer Code (Go):** + ```go import "github.com/cloudflare/circl/kem/kyber/kyber1024" @@ -873,6 +886,7 @@ envelope.Payload = &Any{Value: ciphertext} ``` **Consumer Code (Go):** + ```go // Load consumer's Kyber private key privateKey := loadKyberPrivateKeyFromVault("vault://secrets/consumers/pq/order-processor/private-key") @@ -899,6 +913,7 @@ if err != nil { **Use Case:** Transition period; protect against both current and future threats **Configuration (Producer):** + ```yaml encryption: enabled: true @@ -908,6 +923,7 @@ encryption: ``` **Producer Code (Go):** + ```go // Hybrid approach: X25519 (classical ECDH) + Kyber1024 (post-quantum KEM) // Shared secret = KDF(x25519_secret || kyber_secret) @@ -945,6 +961,7 @@ envelope.Payload = &Any{Value: ciphertext} ``` **Consumer Code (Go):** + ```go // Load consumer's hybrid private keys x25519Private := loadX25519PrivateKey("vault://secrets/consumers/hybrid/order-processor/x25519-private") @@ -1005,6 +1022,7 @@ if err != nil { - Configuration validation MUST enforce FIPS compliance when `fips_mode: true` **Go FIPS Libraries:** + ```go // Use FIPS-validated crypto libraries import ( @@ -1022,6 +1040,7 @@ import ( ``` **Environment Setup:** + ```bash # Enable FIPS mode in Go runtime export GOFIPS=1 @@ -1034,6 +1053,7 @@ go build -tags=fips ./... ### Encryption Security Best Practices **1. Key Rotation:** + ```yaml encryption: key_rotation: @@ -1067,6 +1087,7 @@ encryption: ### Key Management Integration **Vault Integration (Recommended):** + ```yaml encryption: key_provider: vault @@ -1078,6 +1099,7 @@ encryption: ``` **AWS KMS Integration:** + ```yaml encryption: key_provider: aws_kms @@ -1090,6 +1112,7 @@ encryption: ``` **Kubernetes Secrets (For Development Only):** + ```yaml encryption: key_provider: kubernetes_secret diff --git a/docs-cms/rfcs/rfc-034-robust-process-manager.md b/docs-cms/rfcs/rfc-034-robust-process-manager.md index 8a8dbafe9..a83b3c7e8 100644 --- a/docs-cms/rfcs/rfc-034-robust-process-manager.md +++ b/docs-cms/rfcs/rfc-034-robust-process-manager.md @@ -40,6 +40,7 @@ Prism requires robust process management for: Kubelet's `pod_workers.go` (~1700 lines) implements a sophisticated state machine for managing pod lifecycles. Key components: **1. Per-Process Goroutine with Channel Communication** + ```go type podWorkers struct { podUpdates map[types.UID]chan struct{} // One channel per process @@ -50,6 +51,7 @@ type podWorkers struct { ``` **2. Four Lifecycle States** + ```go type PodWorkerState int @@ -61,6 +63,7 @@ const ( ``` **3. State Tracking Per Process** + ```go type podSyncStatus struct { ctx context.Context diff --git a/docs-cms/rfcs/rfc-035-pattern-process-launcher.md b/docs-cms/rfcs/rfc-035-pattern-process-launcher.md index 2db81b53d..031efdbfd 100644 --- a/docs-cms/rfcs/rfc-035-pattern-process-launcher.md +++ b/docs-cms/rfcs/rfc-035-pattern-process-launcher.md @@ -220,6 +220,7 @@ All requests share the same process, regardless of namespace or session. **Use Case**: Stateless patterns with no tenant-specific data (e.g., schema registry lookup) **Example**: + ```text Client A (namespace: tenant-a, session: user-1) ──┐ Client B (namespace: tenant-b, session: user-2) ──┼─→ shared:consumer (single process) @@ -241,6 +242,7 @@ Each namespace gets its own dedicated process. Multiple sessions within the same **Use Case**: Multi-tenant SaaS where tenants must be isolated (data security, billing, fault isolation) **Example**: + ```text Client A (namespace: tenant-a, session: user-1) ──┐ Client C (namespace: tenant-a, session: user-3) ──┼─→ ns:tenant-a:consumer @@ -264,6 +266,7 @@ Each session gets its own dedicated process. Maximum isolation guarantees. **Use Case**: High-security environments, compliance requirements (PCI-DSS, HIPAA), debugging **Example**: + ```text Client A (namespace: tenant-a, session: user-1) ───→ session:user-1:consumer Client B (namespace: tenant-b, session: user-2) ───→ session:user-2:consumer diff --git a/docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md b/docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md index 58bd43155..49e7550ce 100644 --- a/docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md +++ b/docs-cms/rfcs/rfc-037-mailbox-pattern-searchable-event-store.md @@ -437,6 +437,7 @@ namespaces: ## Appendix A: Example Queries **Query by Time Range**: + ```sql SELECT message_id, timestamp, topic, principal FROM mailbox @@ -446,6 +447,7 @@ LIMIT 100; ``` **Query by Principal and Topic**: + ```sql SELECT message_id, timestamp, correlation_id, body FROM mailbox @@ -455,6 +457,7 @@ ORDER BY timestamp DESC; ``` **Query by Correlation ID (Distributed Trace)**: + ```sql SELECT message_id, timestamp, topic, principal, body FROM mailbox @@ -465,6 +468,7 @@ ORDER BY timestamp ASC; ## Appendix B: SQLite Backend Details **Connection Settings**: + ```go // Optimized SQLite settings for write-heavy workload PRAGMA journal_mode=WAL; diff --git a/docs-cms/rfcs/rfc-038-admin-leader-election-raft.md b/docs-cms/rfcs/rfc-038-admin-leader-election-raft.md index 3a6ed6ab8..572d2fcf1 100644 --- a/docs-cms/rfcs/rfc-038-admin-leader-election-raft.md +++ b/docs-cms/rfcs/rfc-038-admin-leader-election-raft.md @@ -1218,6 +1218,7 @@ echo "Leader failover test PASSED" 3. **Complete cluster loss**: Bootstrap new cluster from latest snapshot **Restore procedure**: + ```bash # Stop cluster docker-compose -f docker-compose-admin-cluster.yaml down diff --git a/docs-cms/rfcs/rfc-039-backend-configuration-registry.md b/docs-cms/rfcs/rfc-039-backend-configuration-registry.md index 3fa8e2e9c..8d8283b0f 100644 --- a/docs-cms/rfcs/rfc-039-backend-configuration-registry.md +++ b/docs-cms/rfcs/rfc-039-backend-configuration-registry.md @@ -1128,6 +1128,7 @@ message Backend { Existing patterns that embed backend connection details in their config can migrate gradually: **Before (embedded config)**: + ```yaml namespace: order-processing pattern: multicast-registry @@ -1143,6 +1144,7 @@ config: ``` **After (backend references)**: + ```yaml namespace: order-processing pattern: multicast-registry diff --git a/docs-cms/rfcs/rfc-040-client-sdk-architecture.md b/docs-cms/rfcs/rfc-040-client-sdk-architecture.md index 82b084f07..fbc36dbdb 100644 --- a/docs-cms/rfcs/rfc-040-client-sdk-architecture.md +++ b/docs-cms/rfcs/rfc-040-client-sdk-architecture.md @@ -572,6 +572,7 @@ proxy: ``` Usage: + ```rust let client = Client::from_file("prism-minimal.yaml")?; ``` @@ -1289,6 +1290,7 @@ prism-client = { version = "0.1", features = ["full"] } ``` **Code Generation** (build.rs): + ```rust fn main() -> Result<(), Box> { tonic_build::configure() @@ -1305,6 +1307,7 @@ fn main() -> Result<(), Box> { ``` **Async Streams**: + ```rust // Consumer uses Stream trait pub struct Consumer { @@ -1330,6 +1333,7 @@ impl Consumer { ### Python SDK **Dependencies** (`pyproject.toml` with Poetry): + ```toml [tool.poetry] name = "prism-client" @@ -1358,6 +1362,7 @@ ruff = "^0.1" ``` **Code Generation**: + ```bash # Generate protobuf stubs python -m grpc_tools.protoc \ @@ -1370,6 +1375,7 @@ python -m grpc_tools.protoc \ ``` **Async Iterators**: + ```python # Consumer uses async iterator class Consumer: @@ -1389,6 +1395,7 @@ class Consumer: ### Go SDK **Dependencies** (`go.mod`): + ```go module github.com/prism/client-go @@ -1406,6 +1413,7 @@ require ( ``` **Code Generation** (`gen.go`): + ```go //go:generate protoc --go_out=. --go_opt=paths=source_relative \ // --go-grpc_out=. --go-grpc_opt=paths=source_relative \ @@ -1415,6 +1423,7 @@ require ( ``` **Channel-Based Consumer**: + ```go // Consumer returns channel for messages type Consumer struct { @@ -1900,6 +1909,7 @@ All SDKs include security scanning in CI: ### From Direct Backend Clients **Before** (direct Kafka): + ```python from kafka import KafkaProducer @@ -1911,6 +1921,7 @@ producer.send('orders', {'order_id': 123}) ``` **After** (Prism SDK): + ```python from prism_client import Client @@ -1953,6 +1964,7 @@ async with Client.from_file("prism.yaml") as client: - Binary size <2MB - No configuration file required (programmatic API) - Example code: + ```rust let client = Client::connect("localhost:8980")?; let producer = client.producer("orders")?; @@ -2064,6 +2076,7 @@ fn main() -> Result<(), Box> { ``` **Run**: + ```bash cargo add prism-client cargo run @@ -2083,6 +2096,7 @@ with Client("localhost:8980", api_key="dev-key") as client: ``` **Run**: + ```bash pip install prism-client python hello.py @@ -2111,6 +2125,7 @@ func main() { ``` **Run**: + ```bash go get github.com/prism/client-go go run hello.go