feat: introduce AI Crawler as part of the Mutable Web monorepository (#52)

* chore: initilize crawler-extension project

* feat: connect/disconnect wallet with react query

* feat: show parsed contexts in the side panel

* fix: typing errors

* fix: twitter changed their markup

* feat: generate parser config with chatgpt

* feat: sync active tab and side panel

* feat: improve parser config with chatgpt

* feat: parsed items counter

* feat: store parsed contexts to server, visualize graph

* fix: not all edges are stored

* feat(crawler-dashboard): overlay with parsed context data

* fix: inconsistent config generation

* feat: delete local parser config

* feat: reload context page button, improve styles

* feat: backend for saving of local parser config

* chore: init version

* feat: add Adapter editor, for reading

* feat(crawler-dashboard): add api url env variable

* feat: add changes highlighting; config saving

* feat(crawler-extension): add api url env variable

* feat(crawler-backend): add port env variable

* fix: catch an error in editor

* chore: prepare extension for publishing

* feat(crawler-extension): close popup after clicking on side panel

* feat: open options page if there is no api key

* refactor: remove mutation related code

* fix: Editor has 2 X-axis scrolls

* fix: update parserConfig in the Editor after saving locally; change parsed data Card styles

* feat: introduce contract

* feat: economy implemented

* build: minimization active

* feat: error badge

* feat: skip existing contexts

* chore: version bump

* fix: replace rpc url

* fix: invalid context tree in dashboard

* fix: invalid context tree in dashboard

* feat: improve texts

* feat: add empty editor to add an existed parser

* fix: rename adapters to parsers

* chore: version bump

* [+] NEAR rewards workflow

* initial change to start the metric collection

* ci: delete near rewards tracker

* build: update pnpm lock

---------

Co-authored-by: Dmitry Palchun <dima@ethernian.com>
Co-authored-by: ethernian <ethernian@TABLET-ERQOS7R4.localdomain>

Author: 3 people

apps/crawler-backend/.eslintrc.js

@@ -0,0 +1,25 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir: __dirname,
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

apps/crawler-backend/.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

apps/crawler-backend/README.md

@@ -0,0 +1,73 @@
+<p align="center">
+  <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo-small.svg" width="200" alt="Nest Logo" /></a>
+</p>
+
+[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
+[circleci-url]: https://circleci.com/gh/nestjs/nest
+
+  <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
+    <p align="center">
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
+<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
+<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
+<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
+<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
+<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
+  <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
+    <a href="https://opencollective.com/nest#sponsor"  target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
+  <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
+</p>
+  <!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
+  [![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
+
+## Description
+
+[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
+
+## Installation
+
+```bash
+$ pnpm install
+```
+
+## Running the app
+
+```bash
+# development
+$ pnpm run start
+
+# watch mode
+$ pnpm run start:dev
+
+# production mode
+$ pnpm run start:prod
+```
+
+## Test
+
+```bash
+# unit tests
+$ pnpm run test
+
+# e2e tests
+$ pnpm run test:e2e
+
+# test coverage
+$ pnpm run test:cov
+```
+
+## Support
+
+Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
+
+## Stay in touch
+
+- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
+- Website - [https://nestjs.com](https://nestjs.com/)
+- Twitter - [@nestframework](https://twitter.com/nestframework)
+
+## License
+
+Nest is [MIT licensed](LICENSE).

apps/crawler-backend/nest-cli.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://json.schemastore.org/nest-cli",
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src",
+  "compilerOptions": {
+    "deleteOutDir": true
+  }
+}

apps/crawler-backend/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@mweb/crawler-backend",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "nest start",
+    "dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@nestjs/common": "^10.0.0",
+    "@nestjs/core": "^10.0.0",
+    "@nestjs/platform-express": "^10.0.0",
+    "@noble/hashes": "^1.5.0",
+    "borsher": "^3.5.0",
+    "json-stringify-deterministic": "^1.0.12",
+    "near-api-js": "^5.0.1",
+    "reflect-metadata": "^0.1.13",
+    "rxjs": "^7.8.1"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^10.0.0",
+    "@nestjs/schematics": "^10.0.0",
+    "@nestjs/testing": "^10.0.0",
+    "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.2",
+    "@types/node": "^20.3.1",
+    "@types/supertest": "^2.0.12",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "eslint": "^8.42.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "jest": "^29.5.0",
+    "prettier": "^3.0.0",
+    "source-map-support": "^0.5.21",
+    "supertest": "^6.3.3",
+    "ts-jest": "^29.1.0",
+    "ts-loader": "^9.4.3",
+    "ts-node": "^10.9.1",
+    "tsconfig-paths": "^4.2.0",
+    "typescript": "^5.6.3"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}

apps/crawler-backend/src/app/app.module.ts

@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+import { ContextModule } from 'src/context/context.module';
+
+@Module({
+  imports: [ContextModule],
+})
+export class AppModule {}

apps/crawler-backend/src/context/context.controller.ts

@@ -0,0 +1,23 @@
+import { Body, Controller, Get, Post, Param } from '@nestjs/common';
+import { ContextService } from './context.service';
+import { StoreContextDto } from './dtos/store-context.dto';
+
+@Controller('/context')
+export class ContextController {
+  constructor(private readonly contextService: ContextService) {}
+
+  @Post()
+  async storeContext(@Body() storeContextDto: StoreContextDto) {
+    return await this.contextService.storeContext(storeContextDto);
+  }
+
+  @Get()
+  async getContexts() {
+    return this.contextService.getContexts();
+  }
+
+  @Get(':hash')
+  async getContext(@Param() params: any) {
+    return this.contextService.getContext(params.hash);
+  }
+}

apps/crawler-backend/src/context/context.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { ContextController } from './context.controller';
+import { ContextService } from './context.service';
+
+@Module({
+  imports: [],
+  controllers: [ContextController],
+  providers: [ContextService],
+})
+export class ContextModule {}