From 5942f470fcd375b5fc7a7180ac3e68d5eee07e16 Mon Sep 17 00:00:00 2001 From: sahilrohera2003 Date: Tue, 1 Jul 2025 22:40:26 +0530 Subject: [PATCH] Add Docker publishing workflow and quick start scripts --- .github/workflows/docker-publish.yml | 70 +++++++ README.md | 196 ++++++++++++++++++++ client-dashboard/Dockerfile | 29 +++ client-dashboard/nginx.conf | 33 ++++ client-dashboard/src/components/Sidebar.tsx | 4 + client-dashboard/src/services/api.ts | 2 +- docker-compose.yml | 106 +++++++++++ get-observo.sh | 70 +++++++ log-agent/agent.config.yaml | 4 +- log-agent/src/logWatcher.js | 30 +-- logs.sh | 11 ++ quick-start.sh | 51 +++++ server/Dockerfile | 24 +++ server/src/index.js | 2 +- server/src/kafka/consumer.js | 29 ++- stop.sh | 11 ++ 16 files changed, 644 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 README.md create mode 100644 client-dashboard/Dockerfile create mode 100644 client-dashboard/nginx.conf create mode 100644 docker-compose.yml create mode 100755 get-observo.sh create mode 100755 logs.sh create mode 100755 quick-start.sh create mode 100644 server/Dockerfile create mode 100755 stop.sh diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..c73e5d4 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,70 @@ +name: Docker Build and Publish + +on: + push: + branches: [ main ] + tags: [ 'v*' ] + pull_request: + branches: [ main ] + +env: + REGISTRY: docker.io + IMAGE_NAME_SERVER: observo/server + IMAGE_NAME_DASHBOARD: observo/dashboard + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_SERVER }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_DASHBOARD }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Build and push Server image + uses: docker/build-push-action@v5 + with: + context: ./server + file: ./server/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Dashboard image + uses: docker/build-push-action@v5 + with: + context: ./client-dashboard + file: ./client-dashboard/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..6a71da7 --- /dev/null +++ b/README.md @@ -0,0 +1,196 @@ +# Observo - Self-Hosted Log Monitoring System + +πŸš€ **Observo** is a complete self-hosted log monitoring and analytics platform that provides real-time log aggregation, storage, and visualization. + +## πŸš€ Quick Start (No Repository Clone Required!) + +### Option 1: One-Command Installation +```bash +curl -sSL https://raw.githubusercontent.com/daddycoder007/Observo/main/get-observo.sh | bash +cd observo +./quick-start.sh +``` + +### Option 2: Manual Setup +```bash +# Create directory and download files +mkdir observo && cd observo +curl -o docker-compose.yml https://raw.githubusercontent.com/daddycoder007/Observo/main/docker-compose.yml +curl -o quick-start.sh https://raw.githubusercontent.com/daddycoder007/Observo/main/quick-start.sh +chmod +x quick-start.sh + +# Start Observo +./quick-start.sh +``` + +### Option 3: Direct Docker Compose +```bash +# Download and run +curl -o docker-compose.yml https://raw.githubusercontent.com/daddycoder007/Observo/main/docker-compose.yml +docker-compose up -d +``` + +## πŸ—οΈ Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Log Files │───▢│ Log Agent │───▢│ Kafka │───▢│ Server β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Dashboard │◀───│ Web UI │◀───│ REST API │◀───│ MongoDB β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## πŸ“¦ Components + +### 1. Log Agent (`@observo/log-agent`) +Install the log agent to start monitoring your log files: + +```bash +npm install -g @observo/log-agent +``` + +Create a configuration file: +```yaml +kafka: + brokers: localhost:9092 + clientId: my-log-agent + +files: + - path: /var/log/myapp.log + topic: myapp-logs + tag: output + - path: /var/log/error.log + topic: myapp-logs + tag: error +``` + +Start monitoring: +```bash +observo-agent config.yaml +``` + +### 2. Observo Server (Published Docker Image) +- **Image**: `observo/server:latest` +- **Port**: 3000 +- **Features**: Kafka consumer, MongoDB storage, REST API, WebSocket + +### 3. Observo Dashboard (Published Docker Image) +- **Image**: `observo/dashboard:latest` +- **Port**: 80 +- **Features**: React-based web interface, real-time logs, analytics + +## βš™οΈ Configuration + +### Environment Variables (.env file) +```env +# MongoDB Configuration +MONGO_ROOT_USERNAME=admin +MONGO_ROOT_PASSWORD=password123 +MONGO_DATABASE=observo +MONGO_PORT=27017 + +# Kafka Configuration +KAFKA_TOPIC_PREFIX=observo-logs +KAFKA_PORT=9092 + +# Server Configuration +SERVER_PORT=3000 +DASHBOARD_PORT=80 + +# Kafka Topics +KAFKA_TOPICS=observo-logs.output,observo-logs.error,observo-logs.access +``` + +## 🐳 Docker Images + +All images are automatically published to Docker Hub: + +- **observo/server**: `docker pull observo/server:latest` +- **observo/dashboard**: `docker pull observo/dashboard:latest` + +### Available Tags +- `latest` - Latest stable version +- `v1.0.0` - Specific version +- `main` - Latest from main branch + +## πŸ“Š Access Points + +After starting Observo: + +- **Dashboard**: http://localhost +- **API**: http://localhost:3000/api +- **MongoDB**: localhost:27017 +- **Kafka**: localhost:9092 + +## πŸ› οΈ Management Commands + +```bash +# Start Observo +./quick-start.sh + +# Stop Observo +./stop.sh + +# View logs +./logs.sh + +# Update to latest version +docker-compose pull +docker-compose up -d +``` + +## πŸ“Š API Documentation + +### Base URL: `http://localhost:3000/api` + +#### Get Logs +```http +GET /api/logs?page=1&limit=50&level=error +``` + +#### Get Statistics +```http +GET /api/logs/stats?startDate=2024-01-01&endDate=2024-01-31 +``` + +#### Get Analytics +```http +GET /api/analytics?startDate=2024-01-01&endDate=2024-01-31 +``` + +## πŸ”’ Security + +### Production Deployment +1. Change default passwords in `.env` +2. Use environment variables for sensitive data +3. Configure firewall rules +4. Enable HTTPS (use reverse proxy) +5. Use Kafka authentication if needed + +### Example Production .env +```env +MONGO_ROOT_USERNAME=observo_admin +MONGO_ROOT_PASSWORD=your_secure_password_here +KAFKA_TOPIC_PREFIX=your-company-logs +``` + +## 🀝 Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests if applicable +5. Submit a pull request + +## πŸ“„ License + +MIT License - see [LICENSE](LICENSE) file for details. + +## πŸ†˜ Support + +- πŸ“– [Documentation](https://github.com/daddycoder007/Observo/wiki) +- πŸ› [Issues](https://github.com/daddycoder007/Observo/issues) +- πŸ’¬ [Discussions](https://github.com/daddycoder007/Observo/discussions) \ No newline at end of file diff --git a/client-dashboard/Dockerfile b/client-dashboard/Dockerfile new file mode 100644 index 0000000..c718706 --- /dev/null +++ b/client-dashboard/Dockerfile @@ -0,0 +1,29 @@ +# Build stage +FROM node:18-alpine AS builder + +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci + +# Copy source code +COPY . . + +# Build the app +RUN npm run build + +# Production stage +FROM nginx:alpine + +# Copy built app to nginx +COPY --from=builder /app/build /usr/share/nginx/html + +# Copy nginx configuration +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 + +CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/client-dashboard/nginx.conf b/client-dashboard/nginx.conf new file mode 100644 index 0000000..5fad570 --- /dev/null +++ b/client-dashboard/nginx.conf @@ -0,0 +1,33 @@ +server { + listen 80; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + + # Handle React Router + location / { + try_files $uri $uri/ /index.html; + } + + # API proxy to backend + location /api/ { + proxy_pass http://host.docker.internal:3000/api/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_cache_bypass $http_upgrade; + } + + # WebSocket proxy + location /ws { + proxy_pass http://host.docker.internal:3000/ws; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } +} \ No newline at end of file diff --git a/client-dashboard/src/components/Sidebar.tsx b/client-dashboard/src/components/Sidebar.tsx index d1a77b2..1b2701d 100644 --- a/client-dashboard/src/components/Sidebar.tsx +++ b/client-dashboard/src/components/Sidebar.tsx @@ -17,6 +17,7 @@ import { import { Dashboard as DashboardIcon, ReceiptLong as LogsIcon, + Settings as SettingsIcon, ChevronLeft, ChevronRight, QueryStats, @@ -72,6 +73,9 @@ const Drawer = styled(MuiDrawer, { shouldForwardProp: (prop) => prop !== 'open' const menuItems = [ { text: 'Dashboard', icon: , path: '/' }, { text: 'Logs', icon: , path: '/logs' }, + { text: 'Analytics', icon: , path: '/analytics' }, + { text: 'Settings', icon: , path: '/settings' }, + ]; interface SidebarProps { diff --git a/client-dashboard/src/services/api.ts b/client-dashboard/src/services/api.ts index 4d18dd6..1b60c42 100644 --- a/client-dashboard/src/services/api.ts +++ b/client-dashboard/src/services/api.ts @@ -1,6 +1,6 @@ import axios from 'axios'; -const API_BASE_URL = 'http://localhost:8000/api'; +const API_BASE_URL = 'http://localhost:3000/api'; // Create axios instance with base configuration const apiClient = axios.create({ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6a4c348 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,106 @@ +version: '3.8' + +services: + mongodb: + image: mongo:7.0 + container_name: observo-mongodb + restart: unless-stopped + ports: + - "27017:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: admin + MONGO_INITDB_ROOT_PASSWORD: password + MONGO_INITDB_DATABASE: observo + volumes: + - mongodb_data:/data/db + - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + networks: + - observo-network + command: mongod --auth + + zookeeper: + image: confluentinc/cp-zookeeper:7.5.0 + container_name: observo-zookeeper + restart: unless-stopped + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ports: + - "2181:2181" + networks: + - observo-network + + kafka: + image: confluentinc/cp-kafka:7.5.0 + container_name: observo-kafka + restart: unless-stopped + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: INTERNAL://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_DELETE_TOPIC_ENABLE: 'true' + networks: + - observo-network + + server: + image: observo/server:test + container_name: observo-server + restart: unless-stopped + ports: + - "3000:3000" + environment: + MONGODB_URI: mongodb://admin:password@mongodb:27017/observo?authSource=admin + KAFKA_BROKERS: kafka:29092 + NODE_ENV: production + depends_on: + - mongodb + - kafka + networks: + - observo-network + volumes: + - ./server/logs:/logs + + dashboard: + image: observo/dashboard:test + container_name: observo-dashboard + restart: unless-stopped + ports: + - "8080:80" + environment: + REACT_APP_API_URL: http://localhost:3000 + depends_on: + - server + networks: + - observo-network + + # Optional: Kafka UI for management + kafka-ui: + image: provectuslabs/kafka-ui:latest + container_name: observo-kafka-ui + restart: unless-stopped + depends_on: + - kafka + ports: + - "8081:8080" + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 + KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181 + networks: + - observo-network + +volumes: + mongodb_data: + +networks: + observo-network: + driver: bridge \ No newline at end of file diff --git a/get-observo.sh b/get-observo.sh new file mode 100755 index 0000000..a78778d --- /dev/null +++ b/get-observo.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +echo "πŸš€ Getting Observo..." +echo "====================" + +# Create observo directory +mkdir -p observo +cd observo + +# Download docker-compose.yml +echo "πŸ“₯ Downloading docker-compose.yml..." +curl -o docker-compose.yml https://raw.githubusercontent.com/daddycoder007/Observo/main/docker-compose.yml + +# Download quick-start script +echo "πŸ“₯ Downloading quick-start script..." +curl -o quick-start.sh https://raw.githubusercontent.com/daddycoder007/Observo/main/quick-start.sh +chmod +x quick-start.sh + +# Download stop script +echo "πŸ“₯ Downloading stop script..." +curl -o stop.sh https://raw.githubusercontent.com/daddycoder007/Observo/main/stop.sh +chmod +x stop.sh + +# Download logs script +echo "πŸ“₯ Downloading logs script..." +curl -o logs.sh https://raw.githubusercontent.com/daddycoder007/Observo/main/logs.sh +chmod +x logs.sh + +# Create .env file +echo "βš™οΈ Creating .env file..." +cat > .env << EOF +# Observo Configuration +OBSERVO_VERSION=1.0.0 + +# MongoDB Configuration +MONGO_ROOT_USERNAME=admin +MONGO_ROOT_PASSWORD=password123 +MONGO_DATABASE=observo +MONGO_PORT=27017 + +# Kafka Configuration +KAFKA_TOPIC_PREFIX=observo-logs +KAFKA_PORT=9092 + +# Server Configuration +SERVER_PORT=3000 +DASHBOARD_PORT=80 + +# Kafka Topics (comma-separated) +KAFKA_TOPICS=observo-logs.output,observo-logs.error,observo-logs.access + +# Kafka Group and Client IDs +KAFKA_GROUP_ID=observo-log-consumer-group +KAFKA_CLIENT_ID=observo-log-service + +# MongoDB Collection +MONGODB_COLLECTION=logs +EOF + +echo "" +echo "βœ… Observo files downloaded successfully!" +echo "" +echo "πŸš€ To start Observo:" +echo " ./quick-start.sh" +echo "" +echo "πŸ›‘ To stop Observo:" +echo " ./stop.sh" +echo "" +echo "πŸ“‹ To view logs:" +echo " ./logs.sh" \ No newline at end of file diff --git a/log-agent/agent.config.yaml b/log-agent/agent.config.yaml index ccb1611..ea41456 100644 --- a/log-agent/agent.config.yaml +++ b/log-agent/agent.config.yaml @@ -1,8 +1,8 @@ -server_id: "my-server-1" +server_id: "my-node-server-1" kafka: brokers: - "localhost:9092" topic_prefix: "testing-logs" log_files: - - path: ./logs/output.log + - path: /Users/apple/Documents/Observo/server/logs/output.log tag: "output" diff --git a/log-agent/src/logWatcher.js b/log-agent/src/logWatcher.js index 210f72e..ce5e2cf 100644 --- a/log-agent/src/logWatcher.js +++ b/log-agent/src/logWatcher.js @@ -10,23 +10,23 @@ export const watchLogFile = (file, onLine) => { console.log(`πŸ” Starting to watch: ${file.path} with tag: ${file.tag}`); // Read existing content first - try { - const existingContent = fs.readFileSync(file.path, 'utf8'); - const lines = existingContent.split('\n').filter(line => line.trim() !== ''); + // try { + // const existingContent = fs.readFileSync(file.path, 'utf8'); + // const lines = existingContent.split('\n').filter(line => line.trim() !== ''); - console.log(`πŸ“– Found ${lines.length} existing lines in ${file.path}`); + // console.log(`πŸ“– Found ${lines.length} existing lines in ${file.path}`); - lines.forEach(line => { - console.log(`πŸ“ Processing existing line: ${line}`); - onLine({ - line, - tag: file.tag, - path: file.path, - }); - }); - } catch (error) { - console.error(`❌ Error reading existing content from ${file.path}:`, error); - } + // lines.forEach(line => { + // console.log(`πŸ“ Processing existing line: ${line}`); + // onLine({ + // line, + // tag: file.tag, + // path: file.path, + // }); + // }); + // } catch (error) { + // console.error(`❌ Error reading existing content from ${file.path}:`, error); + // } // Watch for new content let lastSize = fs.statSync(file.path).size; diff --git a/logs.sh b/logs.sh new file mode 100755 index 0000000..8ff4871 --- /dev/null +++ b/logs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "πŸ“‹ Observo Logs" +echo "===============" + +if [ -f docker-compose.yml ]; then + docker-compose logs -f +else + echo "❌ docker-compose.yml not found" + echo " Make sure you're in the correct directory" +fi \ No newline at end of file diff --git a/quick-start.sh b/quick-start.sh new file mode 100755 index 0000000..0205e7a --- /dev/null +++ b/quick-start.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +echo "πŸš€ Observo - Quick Start" +echo "========================" + +# Check if Docker is installed +if ! command -v docker &> /dev/null; then + echo "❌ Docker is not installed. Please install Docker first." + echo " Visit: https://docs.docker.com/get-docker/" + exit 1 +fi + +# Check if Docker Compose is installed +if ! command -v docker-compose &> /dev/null; then + echo "❌ Docker Compose is not installed. Please install Docker Compose first." + echo " Visit: https://docs.docker.com/compose/install/" + exit 1 +fi + +# Pull latest images +echo "πŸ“₯ Pulling latest Observo images..." +docker-compose pull + +# Start services +echo "πŸš€ Starting Observo services..." +docker-compose up -d + +# Wait for services to be ready +echo "⏳ Waiting for services to be ready..." +sleep 30 + +# Check service status +echo "πŸ” Checking service status..." +docker-compose ps + +echo "" +echo "πŸŽ‰ Observo is now running!" +echo "" +echo "🌐 Dashboard: http://localhost" +echo "πŸ”§ API: http://localhost:3000/api" +echo "πŸ—„οΈ MongoDB: localhost:27017" +echo "πŸ“¨ Kafka: localhost:9092" +echo "" +echo "πŸ“¦ To install the log agent:" +echo " npm install -g @observo/log-agent" +echo "" +echo "πŸ“– For more information:" +echo " https://github.com/daddycoder007/Observo" +echo "" +echo "πŸ›‘ To stop Observo:" +echo " docker-compose down" \ No newline at end of file diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..e11190d --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,24 @@ +FROM node:18-alpine + +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci --only=production + +# Copy source code +COPY src/ ./src/ + +# Create non-root user +RUN addgroup -g 1001 -S nodejs +RUN adduser -S observo -u 1001 + +# Change ownership +RUN chown -R observo:nodejs /app +USER observo + +EXPOSE 3000 + +CMD ["npm", "start"] \ No newline at end of file diff --git a/server/src/index.js b/server/src/index.js index 05a1fed..fdcf300 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -19,7 +19,7 @@ dotenv.config(); const app = express(); const server = createServer(app); -const PORT = process.env.PORT || 8000; +const PORT = process.env.PORT || 3000; // Initialize WebSocket initializeWebSocket(server); diff --git a/server/src/kafka/consumer.js b/server/src/kafka/consumer.js index 829e1e7..77edff1 100644 --- a/server/src/kafka/consumer.js +++ b/server/src/kafka/consumer.js @@ -6,24 +6,34 @@ import logger from '../logger.js'; let consumer = null; let isRunning = false; - -const kafka = new Kafka({ - clientId: process.env.KAFKA_CLIENT_ID || 'observo-log-service', - brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','), - retry: { - initialRetryTime: 100, - retries: 8 - } -}); +let kafka = null; export async function startKafkaConsumer() { try { + // Create Kafka instance here after environment variables are loaded + kafka = new Kafka({ + clientId: process.env.KAFKA_CLIENT_ID || 'observo-log-service', + brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','), + retry: { + initialRetryTime: 100, + retries: 8 + } + }); + const topicPrefix = process.env.KAFKA_TOPIC_PREFIX || 'testing-logs'; const topics = process.env.KAFKA_TOPICS ? process.env.KAFKA_TOPICS.split(',') : [`${topicPrefix}.output`]; // Default to output topic const groupId = process.env.KAFKA_GROUP_ID || 'observo-log-consumer-group'; + // Log the configuration for debugging + logger.info(`πŸ”§ Kafka Configuration:`, { + brokers: process.env.KAFKA_BROKERS || 'localhost:9092', + clientId: process.env.KAFKA_CLIENT_ID || 'observo-log-service', + groupId: groupId, + topics: topics + }); + consumer = kafka.consumer({ groupId }); await consumer.connect(); @@ -48,6 +58,7 @@ export async function startKafkaConsumer() { }); isRunning = true; + logger.info('βœ… Kafka consumer started successfully'); } catch (error) { logger.error('❌ Failed to start Kafka consumer:', error); diff --git a/stop.sh b/stop.sh new file mode 100755 index 0000000..31ecae6 --- /dev/null +++ b/stop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "πŸ›‘ Stopping Observo..." + +if [ -f docker-compose.yml ]; then + docker-compose down + echo "βœ… Observo services stopped" +else + echo "❌ docker-compose.yml not found" + echo " Make sure you're in the correct directory" +fi \ No newline at end of file