diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 669d653..dafe379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,27 @@ name: CI on: pull_request: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "docker/**" + - "package.json" + - "pnpm-lock.yaml" + - "tsconfig.json" + - ".github/workflows/ci.yml" push: branches: [main] tags: ["v*"] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "docker/**" + - "package.json" + - "pnpm-lock.yaml" + - "tsconfig.json" + - ".github/workflows/ci.yml" permissions: contents: write @@ -58,7 +76,7 @@ jobs: with: images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-server tags: | - type=ref,event=branch + type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} @@ -103,7 +121,7 @@ jobs: with: images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-agent tags: | - type=ref,event=branch + type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 169a3c0..845bad8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,8 +3,20 @@ name: CodeQL on: pull_request: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "package.json" + - "pnpm-lock.yaml" push: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "package.json" + - "pnpm-lock.yaml" schedule: - cron: "30 5 * * 1" # Monday 5:30 UTC diff --git a/README.md b/README.md index 4b60644..d945b5c 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,9 @@ Stop hand-editing YAML. Build observability pipelines with drag-and-drop
and
- ## Why VectorFlow? @@ -51,13 +49,17 @@ Build Vector pipelines with a drag-and-drop canvas. Browse 100+ components from Deploy pipeline configs to your entire fleet with a single click. The deploy dialog shows a full YAML diff against the previous version before you confirm. Agents pull configs automatically — no SSH, no Ansible, no manual intervention. - +

+ VectorFlow Fleet — manage and monitor all your agents +

### 📊 Real-Time Monitoring Track pipeline throughput, error rates, and host metrics (CPU, memory, disk, network) per node and per pipeline. Live event rates display directly on the pipeline canvas while you're editing. - +

+ VectorFlow Dashboard — real-time metrics per node including CPU, memory, and pipeline throughput +

### 🔄 Version Control & Rollback @@ -72,14 +74,10 @@ Every deployment creates an immutable version snapshot with a changelog. Browse - **Certificate management** — TLS cert storage referenced directly in pipeline configs - **Audit log** — immutable record of every action with before/after diffs - - ### ⚡ Alerting & Webhooks Set threshold-based alert rules on CPU, memory, disk, error rates, and more. Deliver notifications via HMAC-signed webhooks to Slack, Discord, PagerDuty, or any HTTP endpoint. - - ## 🏗️ Architecture ```mermaid diff --git a/docker/server/docker-compose.dev.yml b/docker/server/docker-compose.dev.yml index a933f60..973fda0 100644 --- a/docker/server/docker-compose.dev.yml +++ b/docker/server/docker-compose.dev.yml @@ -33,6 +33,7 @@ services: AUTH_TRUST_HOST: "true" volumes: - vfdata:/app/.vectorflow + - backups:/backups env_file: - path: .env required: false @@ -42,3 +43,5 @@ volumes: name: vectorflow-pgdata vfdata: name: vectorflow-data + backups: + name: vectorflow-backups diff --git a/src/components/flow/sink-node.tsx b/src/components/flow/sink-node.tsx index bcfc3cb..4d356cd 100644 --- a/src/components/flow/sink-node.tsx +++ b/src/components/flow/sink-node.tsx @@ -2,9 +2,8 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -23,36 +22,9 @@ type SinkNodeData = { type SinkNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function SinkNodeComponent({ data, selected }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled } = data; + const { componentDef, componentKey, metrics, disabled } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (
) {

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {(componentDef.inputTypes ?? []).map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */} diff --git a/src/components/flow/source-node.tsx b/src/components/flow/source-node.tsx index 4f0ec51..4087444 100644 --- a/src/components/flow/source-node.tsx +++ b/src/components/flow/source-node.tsx @@ -2,10 +2,9 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { Lock } from "lucide-react"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -25,36 +24,9 @@ type SourceNodeData = { type SourceNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function SourceNodeComponent({ data, selected }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled, isSystemLocked } = data; + const { componentDef, componentKey, metrics, disabled, isSystemLocked } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (
) {

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {componentDef.outputTypes.map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */} diff --git a/src/components/flow/transform-node.tsx b/src/components/flow/transform-node.tsx index c53d658..e8225b7 100644 --- a/src/components/flow/transform-node.tsx +++ b/src/components/flow/transform-node.tsx @@ -2,9 +2,8 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -23,39 +22,12 @@ type TransformNodeData = { type TransformNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function TransformNodeComponent({ data, selected, }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled } = data; + const { componentDef, componentKey, metrics, disabled } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {componentDef.outputTypes.map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */}