Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,27 @@ name: CI
on:
pull_request:
branches: [main]
paths:
- "src/**"
- "agent/**"
- "prisma/**"
- "docker/**"
- "package.json"
- "pnpm-lock.yaml"
- "tsconfig.json"
- ".github/workflows/ci.yml"
push:
branches: [main]
tags: ["v*"]
paths:
- "src/**"
- "agent/**"
- "prisma/**"
- "docker/**"
- "package.json"
- "pnpm-lock.yaml"
- "tsconfig.json"
- ".github/workflows/ci.yml"

permissions:
contents: write
Expand Down Expand Up @@ -58,7 +76,7 @@ jobs:
with:
images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-server
tags: |
type=ref,event=branch
type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
Expand Down Expand Up @@ -103,7 +121,7 @@ jobs:
with:
images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-agent
tags: |
type=ref,event=branch
type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,20 @@ name: CodeQL
on:
pull_request:
branches: [main]
paths:
- "src/**"
- "agent/**"
- "prisma/**"
- "package.json"
- "pnpm-lock.yaml"
push:
branches: [main]
paths:
- "src/**"
- "agent/**"
- "prisma/**"
- "package.json"
- "pnpm-lock.yaml"
schedule:
- cron: "30 5 * * 1" # Monday 5:30 UTC

Expand Down
16 changes: 7 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,9 @@ Stop hand-editing YAML. Build observability pipelines with drag-and-drop<br>and

<br>

<!-- TODO: Add hero screenshot — drag the pipeline editor screenshot into a GitHub issue to get a CDN URL, then uncomment:
<p align="center">
<img src="REPLACE_WITH_GITHUB_CDN_URL" alt="VectorFlow Pipeline Editor — drag-and-drop canvas with component palette, visual node graph, and configuration panel" width="800">
<img src="https://github.com/user-attachments/assets/6c1003be-9e54-46b3-8bc3-1ccfe48cbbdd" alt="VectorFlow Pipeline Editor — drag-and-drop canvas with component palette, visual node graph, and configuration panel" width="800">
</p>
-->

## Why VectorFlow?

Expand All @@ -51,13 +49,17 @@ Build Vector pipelines with a drag-and-drop canvas. Browse 100+ components from

Deploy pipeline configs to your entire fleet with a single click. The deploy dialog shows a full YAML diff against the previous version before you confirm. Agents pull configs automatically — no SSH, no Ansible, no manual intervention.

<!-- TODO: Add pipeline list screenshot -->
<p align="center">
<img src="https://github.com/user-attachments/assets/207e5f17-eca4-490a-abfe-f1d86402840f" alt="VectorFlow Fleet — manage and monitor all your agents" width="800">
</p>

### 📊 Real-Time Monitoring

Track pipeline throughput, error rates, and host metrics (CPU, memory, disk, network) per node and per pipeline. Live event rates display directly on the pipeline canvas while you're editing.

<!-- TODO: Add node detail screenshot -->
<p align="center">
<img src="https://github.com/user-attachments/assets/c7f7e5be-df2a-4877-b716-b125a343f902" alt="VectorFlow Dashboard — real-time metrics per node including CPU, memory, and pipeline throughput" width="800">
</p>

### 🔄 Version Control & Rollback

Expand All @@ -72,14 +74,10 @@ Every deployment creates an immutable version snapshot with a changelog. Browse
- **Certificate management** — TLS cert storage referenced directly in pipeline configs
- **Audit log** — immutable record of every action with before/after diffs

<!-- TODO: Add audit log screenshot -->

### ⚡ Alerting & Webhooks

Set threshold-based alert rules on CPU, memory, disk, error rates, and more. Deliver notifications via HMAC-signed webhooks to Slack, Discord, PagerDuty, or any HTTP endpoint.

<!-- TODO: Add alerts screenshot -->

## 🏗️ Architecture

```mermaid
Expand Down
3 changes: 3 additions & 0 deletions docker/server/docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ services:
AUTH_TRUST_HOST: "true"
volumes:
- vfdata:/app/.vectorflow
- backups:/backups
env_file:
- path: .env
required: false
Expand All @@ -42,3 +43,5 @@ volumes:
name: vectorflow-pgdata
vfdata:
name: vectorflow-data
backups:
name: vectorflow-backups
53 changes: 4 additions & 49 deletions src/components/flow/sink-node.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

import { memo, useMemo } from "react";
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/utils";
import type { VectorComponentDef, DataType } from "@/lib/vector/types";
import type { VectorComponentDef } from "@/lib/vector/types";
import type { NodeMetricsData } from "@/stores/flow-store";
import { getIcon } from "./node-icon";
import { NodeSparkline } from "./node-sparkline";
Expand All @@ -23,36 +22,9 @@ type SinkNodeData = {

type SinkNodeType = Node<SinkNodeData, "sink">;

const dataTypeBadgeColor: Record<DataType, string> = {
log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300",
metric:
"bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300",
trace:
"bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300",
};

function getConfigSummary(config: Record<string, unknown>): string | null {
const entries = Object.entries(config);
if (entries.length === 0) return null;

const [key, value] = entries[0];
if (value === undefined || value === null) return null;
if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`;
const display =
typeof value === "string"
? value
: Array.isArray(value)
? value.slice(0, 2).join(", ")
: String(value);

const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display;
return `${key}: ${truncated}`;
}

function SinkNodeComponent({ data, selected }: NodeProps<SinkNodeType>) {
const { componentDef, componentKey, config, metrics, disabled } = data;
const { componentDef, componentKey, metrics, disabled } = data;
const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]);
const configSummary = getConfigSummary(config);

return (
<div
Expand Down Expand Up @@ -82,28 +54,11 @@ function SinkNodeComponent({ data, selected }: NodeProps<SinkNodeType>) {
<div className="space-y-2 px-3 py-2.5">
<p className="truncate text-xs font-medium text-foreground">{componentKey}</p>

{metrics ? (
{metrics && (
<p className="truncate text-xs font-mono text-purple-400">
{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}
</p>
) : configSummary ? (
<p className="truncate text-xs text-muted-foreground">
{configSummary}
</p>
) : null}

{/* Data type badges */}
<div className="flex flex-wrap gap-1">
{(componentDef.inputTypes ?? []).map((dt) => (
<Badge
key={dt}
variant="secondary"
className={cn("px-1.5 py-0 text-xs", dataTypeBadgeColor[dt])}
>
{dt.charAt(0).toUpperCase() + dt.slice(1)}
</Badge>
))}
</div>
)}
</div>

{/* Monitoring overlay */}
Expand Down
53 changes: 4 additions & 49 deletions src/components/flow/source-node.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@

import { memo, useMemo } from "react";
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
import { Badge } from "@/components/ui/badge";
import { Lock } from "lucide-react";
import { cn } from "@/lib/utils";
import type { VectorComponentDef, DataType } from "@/lib/vector/types";
import type { VectorComponentDef } from "@/lib/vector/types";
import type { NodeMetricsData } from "@/stores/flow-store";
import { getIcon } from "./node-icon";
import { NodeSparkline } from "./node-sparkline";
Expand All @@ -25,36 +24,9 @@ type SourceNodeData = {

type SourceNodeType = Node<SourceNodeData, "source">;

const dataTypeBadgeColor: Record<DataType, string> = {
log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300",
metric:
"bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300",
trace:
"bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300",
};

function getConfigSummary(config: Record<string, unknown>): string | null {
const entries = Object.entries(config);
if (entries.length === 0) return null;

const [key, value] = entries[0];
if (value === undefined || value === null) return null;
if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`;
const display =
typeof value === "string"
? value
: Array.isArray(value)
? value.slice(0, 2).join(", ")
: String(value);

const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display;
return `${key}: ${truncated}`;
}

function SourceNodeComponent({ data, selected }: NodeProps<SourceNodeType>) {
const { componentDef, componentKey, config, metrics, disabled, isSystemLocked } = data;
const { componentDef, componentKey, metrics, disabled, isSystemLocked } = data;
const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]);
const configSummary = getConfigSummary(config);

return (
<div
Expand All @@ -81,28 +53,11 @@ function SourceNodeComponent({ data, selected }: NodeProps<SourceNodeType>) {
<div className="space-y-2 px-3 py-2.5">
<p className="truncate text-xs font-medium text-foreground">{componentKey}</p>

{metrics ? (
{metrics && (
<p className="truncate text-xs font-mono text-emerald-400">
{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}
</p>
) : configSummary ? (
<p className="truncate text-xs text-muted-foreground">
{configSummary}
</p>
) : null}

{/* Data type badges */}
<div className="flex flex-wrap gap-1">
{componentDef.outputTypes.map((dt) => (
<Badge
key={dt}
variant="secondary"
className={cn("px-1.5 py-0 text-xs", dataTypeBadgeColor[dt])}
>
{dt.charAt(0).toUpperCase() + dt.slice(1)}
</Badge>
))}
</div>
)}
</div>

{/* Monitoring overlay */}
Expand Down
53 changes: 4 additions & 49 deletions src/components/flow/transform-node.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

import { memo, useMemo } from "react";
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/utils";
import type { VectorComponentDef, DataType } from "@/lib/vector/types";
import type { VectorComponentDef } from "@/lib/vector/types";
import type { NodeMetricsData } from "@/stores/flow-store";
import { getIcon } from "./node-icon";
import { NodeSparkline } from "./node-sparkline";
Expand All @@ -23,39 +22,12 @@ type TransformNodeData = {

type TransformNodeType = Node<TransformNodeData, "transform">;

const dataTypeBadgeColor: Record<DataType, string> = {
log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300",
metric:
"bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300",
trace:
"bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300",
};

function getConfigSummary(config: Record<string, unknown>): string | null {
const entries = Object.entries(config);
if (entries.length === 0) return null;

const [key, value] = entries[0];
if (value === undefined || value === null) return null;
if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`;
const display =
typeof value === "string"
? value
: Array.isArray(value)
? value.slice(0, 2).join(", ")
: String(value);

const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display;
return `${key}: ${truncated}`;
}

function TransformNodeComponent({
data,
selected,
}: NodeProps<TransformNodeType>) {
const { componentDef, componentKey, config, metrics, disabled } = data;
const { componentDef, componentKey, metrics, disabled } = data;
const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]);
const configSummary = getConfigSummary(config);

return (
<div
Expand Down Expand Up @@ -85,28 +57,11 @@ function TransformNodeComponent({
<div className="space-y-2 px-3 py-2.5">
<p className="truncate text-xs font-medium text-foreground">{componentKey}</p>

{metrics ? (
{metrics && (
<p className="truncate text-xs font-mono text-blue-400">
{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}
</p>
) : configSummary ? (
<p className="truncate text-xs text-muted-foreground">
{configSummary}
</p>
) : null}

{/* Data type badges */}
<div className="flex flex-wrap gap-1">
{componentDef.outputTypes.map((dt) => (
<Badge
key={dt}
variant="secondary"
className={cn("px-1.5 py-0 text-xs", dataTypeBadgeColor[dt])}
>
{dt.charAt(0).toUpperCase() + dt.slice(1)}
</Badge>
))}
</div>
)}
</div>

{/* Monitoring overlay */}
Expand Down
Loading