feat: introduce crawler agent and orders system (DAP-4811) (#56)

* feat: introduce crawler agent PoC

* ci: containerize crawler

* build: add kubernetes deployment config

* feat: process multiple page in parallel

* ci: ignore crawler subprojects

* feat: introduce orders

* refactor: extract shared types to crawler sdk

* feat(crawler-agent): fetch orders from the backend

* feat: fetch parser configs from the registry

* feat(crawler-agent): respect cron expressions

Author: alsakhaev

.github/workflows/crawler-agent.yml

@@ -0,0 +1,32 @@
+name: Crawler Agent
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'apps/crawler-agent/**'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build and Push container
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: "Checkout GitHub Action"
+        uses: actions/checkout@main
+
+      - name: "Login to GitHub Container Registry"
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: "Build Inventory Image"
+        run: |
+          docker build -f apps/crawler-agent/Dockerfile . --tag ghcr.io/dapplets/crawler-agent:latest
+          docker push ghcr.io/dapplets/crawler-agent:latest

.github/workflows/extension-artifact-upload.yml

@@ -2,6 +2,11 @@ name: Extension Artifact Upload
 on:
   pull_request:
     branches: [develop]
+  paths-ignore:
+    - 'apps/crawler-agent/**'
+    - 'apps/crawler-backend/**'
+    - 'apps/crawler-dashboard/**'
+    - 'apps/crawler-extension/**'
 jobs:
   create-artifact:
     name: Test build

.github/workflows/extension-release.yml

@@ -6,8 +6,12 @@ on:
       - master
       - develop
     paths-ignore:
+      - 'apps/crawler-agent/**'
+      - 'apps/crawler-backend/**'
+      - 'apps/crawler-dashboard/**'
+      - 'apps/crawler-extension/**'
       - 'package.json'
-      - 'package-lock.json'
+      - 'pnpm-lock.yaml'
       - 'CHANGELOG.md'
 jobs:
   release:

apps/crawler-agent/.gitignore

@@ -0,0 +1,33 @@
+# Dependencies
+node_modules/
+.pnp/
+.pnp.js
+
+# Testing
+coverage/
+.nyc_output/
+
+# Production
+dist/
+build/
+
+# Development
+.env
+.env.local
+.env.development
+.env.test
+.env.production
+
+# Logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Editor directories and files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store

apps/crawler-agent/.npmignore

@@ -0,0 +1,30 @@
+# Source
+src/
+
+# Tests
+tests/
+coverage/
+*.test.ts
+jest.config.js
+
+# Development configs
+.github/
+.vscode/
+.env*
+tsconfig.json
+.eslintrc.js
+.prettierrc
+.gitignore
+
+# Documentation source
+docs/
+
+# Misc
+*.log
+.DS_Store
+node_modules/
+examples/
+
+# Include dist and type definitions
+!dist/
+!dist/**/*.d.ts

apps/crawler-agent/.npmrc

@@ -0,0 +1,6 @@
+audit=true
+fund=false
+package-lock=true
+save-exact=true
+engine-strict=true
+resolution-mode=highest

apps/crawler-agent/Dockerfile

@@ -0,0 +1,47 @@
+FROM node:current-alpine as base
+
+RUN apk add chromium
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+    
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+FROM base AS builder
+# Set working directory
+WORKDIR /app
+
+RUN pnpm add -g turbo@^2
+COPY . .
+
+# Generate a partial monorepo with a pruned lockfile for a target workspace.
+# Assuming "web" is the name entered in the project's package.json: { name: "web" }
+RUN turbo telemetry disable
+RUN turbo prune @mweb/crawler-agent --docker
+ 
+# Add lockfile and package.json's of isolated subworkspace
+FROM base AS installer
+WORKDIR /app
+ 
+# First install the dependencies (as they change less often)
+COPY --from=builder /app/out/json/ .
+RUN pnpm install
+ 
+# Build the project
+COPY --from=builder /app/tsconfig.base.json ./tsconfig.base.json
+COPY --from=builder /app/out/full/ .
+RUN pnpm run build:crawler-agent
+ 
+FROM base AS runner
+WORKDIR /app
+ 
+# Don't run production as root
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 app
+USER app
+
+COPY --from=installer --chown=app:nodejs /app ./
+
+CMD node apps/crawler-agent/dist/node/cli.js start

apps/crawler-agent/jest.config.js

@@ -0,0 +1,26 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  roots: ['<rootDir>/src', '<rootDir>/tests'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': ['ts-jest', {
+      tsconfig: 'tsconfig.test.json'
+    }]
+  },
+  moduleNameMapper: {
+    '^@/(.*)$': '<rootDir>/src/$1'
+  },
+  setupFiles: ['<rootDir>/tests/setup.ts'],
+  collectCoverage: true,
+  coverageDirectory: 'coverage',
+  coverageReporters: ['text', 'lcov'],
+  coveragePathIgnorePatterns: [
+    '/node_modules/',
+    '/dist/',
+    '/coverage/',
+    '/tests/setup.ts'
+  ],
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node']
+};

apps/crawler-agent/kubernetes/deployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawler-agent
+  namespace: default
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: crawler-agent
+  template:
+    metadata:
+      labels:
+        app: crawler-agent
+    spec:
+      containers:
+        - name: container-name
+          image: ghcr.io/dapplets/crawler-agent:latest
+          env:
+            - name: API_URL
+              value: "https://crawler-api.apps.dapplets.org"

apps/crawler-agent/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@mweb/crawler-agent",
+  "version": "0.1.0",
+  "main": "dist/node/index.js",
+  "types": "dist/node/index.d.ts",
+  "bin": {
+    "mweb-crawler-agent": "dist/node/cli.js"
+  },
+  "scripts": {
+    "start": "node dist/node/cli.js start",
+    "build": "pnpm run clean && tsc -p tsconfig.json && chmod +x ./dist/node/cli.js && pnpm run build:inpage",
+    "build:inpage": "webpack --mode production",
+    "clean": "rm -rf dist",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:coverage": "jest --coverage",
+    "test:unit": "jest \"tests/unit/.*\\.test\\.ts$\"",
+    "test:unit:watch": "jest \"tests/unit/.*\\.test\\.ts$\" --watch",
+    "lint": "eslint src --ext .ts",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "prepublishOnly": "pnpm run build && pnpm test && pnpm run lint"
+  },
+  "files": [
+    "dist",
+    "docs"
+  ],
+  "keywords": [
+    "mutableweb",
+    "dapplets",
+    "crawler"
+  ],
+  "author": "Mutable Web",
+  "license": "MIT",
+  "dependencies": {
+    "@mweb/backend": "workspace:*",
+    "@mweb/core": "workspace:*",
+    "@mweb/crawler-sdk": "workspace:*",
+    "commander": "11.1.0",
+    "events": "^3.3.0",
+    "node-cron": "3.0.3",
+    "puppeteer": "23.10.4"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.0.0",
+    "@types/node": "^20.0.0",
+    "@types/node-cron": "3.0.11",
+    "jest": "^29.0.0",
+    "ts-jest": "^29.0.0",
+    "typescript": "^5.0.0",
+    "webpack": "^5.86.0",
+    "webpack-cli": "^5.1.4"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}