diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8fd80ac3e..383292f04 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## - # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index a5f1acd50..ff020426e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: Black on: diff --git a/.github/workflows/check_mainnet.yml b/.github/workflows/check_mainnet.yml index f704bd534..6115473d6 100644 --- a/.github/workflows/check_mainnet.yml +++ b/.github/workflows/check_mainnet.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: Run Mainnet Check and Notify Slack on: diff --git a/.github/workflows/check_testnet.yml b/.github/workflows/check_testnet.yml index 71649c9da..1016b22b9 100644 --- a/.github/workflows/check_testnet.yml +++ b/.github/workflows/check_testnet.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: Run Script and Notify Slack on: diff --git a/.github/workflows/cron_topup.yml b/.github/workflows/cron_topup.yml index 06e58d81f..2c7d434d3 100644 --- a/.github/workflows/cron_topup.yml +++ b/.github/workflows/cron_topup.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: Topup accounts on: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 08e4d486f..f0fb30092 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: MyPy Static Typechecking on: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d94c982ec..3f0875dde 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: Pylint Style Tests on: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ee45941c3..17133aa5a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,7 +1,3 @@ -## -## Copyright 2024 Ocean Protocol Foundation -## SPDX-License-Identifier: Apache-2.0 -## name: pdr-backend tests on: @@ -26,20 +22,6 @@ jobs: with: python-version: "3.11" - - uses: actions/checkout@v2 - name: Checkout Barge - with: - repository: "oceanprotocol/barge" - path: "barge" - ref: "5bd4a318c732d84a6b7fbeb193cee7fb16891a67" - - - name: Run Barge - working-directory: ${{ github.workspace }}/barge - env: - SUBGRAPH_VERSION: main - run: | - bash -x start_ocean.sh --no-aquarius --no-elasticsearch --no-provider --no-dashboard --with-thegraph --predictoor > start_ocean.log & - - name: Install dependencies working-directory: ${{ github.workspace }} run: | @@ -47,18 +29,10 @@ jobs: pip install -r requirements.txt pip install pytest - - name: Wait for contracts deployment - working-directory: ${{ github.workspace }}/barge - run: | - for i in $(seq 1 250); do - sleep 5 - [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" ] && break - done - - name: Test with pytest id: pytest run: | - coverage run --source=pdr_backend --omit=*/test/*,*/test_ganache/*,*/test_noganache/* -m pytest + coverage run --source=pdr_backend --omit=*/test/*,*/test_noganache/* -m pytest coverage report coverage xml - name: Publish code coverage diff --git a/.gitignore b/.gitignore index b7206038c..1549c70f4 100644 --- a/.gitignore +++ b/.gitignore @@ -58,69 +58,9 @@ cover/ *.mo *.pot -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py # Environments .env @@ -131,59 +71,30 @@ ENV/ env.bak/ venv.bak/ -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - # mypy .mypy_cache/ .dmypy.json dmypy.json -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ -.test_cache/ -.cache/ -dir/ - # predictoor-specific out*.txt my_ppss.yaml +my_ppss_profiling.yaml +my_ppss_makemoney.yaml my_logging.yaml logs/ csvs/ parquet_data/ lake_data/ +xy_dir/ +xy_data/ pdr_predictions.parquet -# sim state -sim_state - # pdr_backend accuracy output pdr_backend/accuracy/output/*.json # lakeinfo output validate_duplicate_rows.csv -# agent deployer -.keys.json -.deployments/ +#profiling +profile.stats \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 5aca79878..2199bbff1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -31,7 +31,7 @@ ignore= Dockerfile, entrypoint.sh, pdr_backend.egg-info, - copyright_template.tmpl + profile.stats # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. diff --git a/README.md b/README.md index f1cb6d078..cbb39e34c 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,7 @@ ## Run bots (agents) -- **[Run predictoor bot](READMEs/predictoor.md)** - make predictions, make $ -- **[Run trader bot](READMEs/trader.md)** - consume predictions, trade, make $ - -(If you're a predictoor or trader, you can safely ignore the rest of this README.) +- **[Run sim/trading](READMEs/sim.md)** - predict, trade, make $. Historical, live mock, or live real. ## Settings: PPSS @@ -36,34 +33,20 @@ pdr This will output something like: ```text -Usage: pdr sim|predictoor|trader|.. +Usage: pdr sim|trader|.. Main tools: pdr sim YAML_FILE - pdr predictoor YAML_FILE NETWORK - pdr trader APPROACH YAML_FILE NETWORK + pdr trader APPROACH YAML_FILE livemock|livereal (FIXME) ... ``` -## Atomic READMEs - -- [Get tokens](READMEs/get-tokens.md): [testnet faucet](READMEs/testnet-faucet.md), [mainnet ROSE](READMEs/get-rose-on-sapphire.md) & [OCEAN](READMEs/get-ocean-on-sapphire.md) -- [Claim payout for predictoor bot](READMEs/payout.md) -- [Predictoor subgraph](READMEs/subgraph.md). [Subgraph filters](READMEs/filters.md) -- [Run barge locally](READMEs/barge.md) - ## Flows for core team - Backend-dev - for `pdr-backend` itself - - [Local dev flow](READMEs/dev.md) + - Local dev flow: see [sim](READMEs/sim.md) - [VPS dev flow](READMEs/vps.md) - - [Release process](READMEs/release-process.md) - - [Clean code guidelines](READMEs/clean-code.md) - [Dependency management](READMEs/dependencies.md) -- [Run dfbuyer bot](READMEs/dfbuyer.md) - runs Predictoor DF rewards -- [Run publisher](READMEs/publisher.md) - publish new feeds -- [Run trueval](READMEs/trueval.md) - run trueval bot -- [Run lake](READMEs/lake-and-etl.md) - run data lake ## Repo structure @@ -71,17 +54,8 @@ This repo implements all bots in Predictoor ecosystem. Here are each of the sub- Main bots & user tools: -- `predictoor` - submit individual predictions - `trader` - buy aggregated predictions, then trade - `sim` - experiments / simulation flow -- `payout` - OCEAN & ROSE payout - -OPF-run bots & higher-level tools: - -- `trueval` - report true values to contract -- `dfbuyer` - buy feeds on behalf of Predictoor DF -- `publisher` - publish pdr data feeds -- `deployer` - deployer tool Mid-level building blocks: @@ -93,11 +67,7 @@ Data-level building blocks: - `ohlcv` - financial data pipeline - `aimodel` - AI/ML modeling engine - `lake` - data lake and analytics tools -- `subgraph` - blockchain queries, complements lake -- `accuracy` - to report % correct in webapp -- `pred_submitter` - for predictoor bots to submit >>1 predictions in 1 tx Lower-level utilities: -- `contract` - classes to wrap blockchain contracts; some simple data structures - `util` - function-based tools diff --git a/READMEs/agent-deployer.md b/READMEs/agent-deployer.md deleted file mode 100644 index 4d68ca27d..000000000 --- a/READMEs/agent-deployer.md +++ /dev/null @@ -1,182 +0,0 @@ - -# Agent Deployment Tool - -`deployer` is a streamlined command-line utility designed for efficiently generating and managing Predictoor agent deployments. - -## Usage - -### Agent Configurations - -Firstly, you need to set up your agents configuration. This is done by creating a config entry under `deployment_configs` in `ppss.yaml` file. - -Here is an example structure for your reference: - -```yaml -deployment_configs: - testnet_predictoor_deployment: - cpu: "1" - memory: "512Mi" - source: "binance" - type: "predictoor" - approach: 3 - network: "sapphire-testnet" - s_until_epoch_end: 20 - pdr_backend_image_source: "oceanprotocol/pdr-backend:latest" - agents: - - pair: "BTC/USDT" - stake_amt: 15 - timeframe: 5m - approach: 1 - - pair: "ETH/USDT" - stake_amt: 20 - timeframe: 1h - s_until_epoch_end: 60 -``` - -_Tip: Specific agent settings (like source, timeframe) will override general settings if provided._ - -### Private Keys - -Create a `.keys.json` file and format it as follows: - -```json -{ - "config_name": ["pk1", "pk2"...] -} -``` - -_Note: If you have fewer private keys than number of agents, the tool will create new wallets and update the .keys.json file._ - -### Generate Templates - -The `generate` command is used to create deployment template files based on a configuration file. - -```console -pdr deployer generate -``` - -- ``: Path to the yaml config file. -- ``: Name of the config. -- ``: Method of deployment (choices: "k8s"). -- ``: Output directory for the generated files. - -Take a note of the `config_name`, you will need it later! - -### Deploy Config - -The `deploy` command is used to deploy agents based on a specified config name. - -```console -pdr deployer deploy [-p PROVIDER] [-r REGION] [--project_id PROJECT_ID] [--resource_group RESOURCE_GROUP] [--subscription_id SUBSCRIPTION_ID] -``` - -- ``: Name of the config. -- -p, --provider: Cloud provider (optional, choices: "aws", "azure", "gcp"). (optional) -- -r, --region: Deployment zone/region (optional). -- --project_id: Google Cloud project id (optional). -- --resource_group: Azure resource group (optional). -- --subscription_id: Azure subscription id (optional). - -### Destroy Config - -The `destroy` command is used to destroy agents deployed based on a specified configuration. - -```console -pdr deployer destroy [-p PROVIDER] -``` - -- ``: Name of the config. -- -p, --provider: Cloud provider (optional, choices: "aws", "azure", "gcp"). (optional) - -### Logs - -The `logs` command is used to retrieve logs from deployed agents. - -```console -pdr deployer logs [-p PROVIDER] -``` - -- ``: Name of the config. -- -p, --provider: Cloud provider (optional, choices: "aws", "azure", "gcp"). (optional) - -### Remote Container Registry - -The `registry` command is used to manage remote registries for agent deployment. - -```console -pdr deployer registry [-p PROVIDER] [-r REGION] [--project_id PROJECT_ID] [--resource_group RESOURCE_GROUP] -``` - -- ``: Action (choices: "deploy", "destroy", "auth", "url"). -- ``: Registry name. -- -p, --provider: Cloud provider (optional, choices: "aws", "azure", "gcp"). -- -r, --region: Deployment zone/region (optional). -- --project_id: Google Cloud project id (optional). -- --resource_group: Azure resource group (optional). - -### Build - -The build command is used to build a container image. - -```console -pdr deployer build -``` - -- ``: Image name (default: "pdr_backend"). -- ``: Image tag (default: "deployer"). - -#### Push - -The `push` command is used to push container images to a remote registry. - -```console -pdr deployer push [] [] -``` - -- ``: Registry name. -- ``: Image name (default: "pdr_backend"). -- ``: Image tag (default: "deployer"). - -## Examples - -### K8S with GCP - -Outputs are truncated for brevity. - -```shell -$ pdr deployer generate ppss.yaml predictoors_cluster k8s ./predictoors_approach3 -Generated k8s templates for predictoors_cluster - Output path: ./predictoor_approach3 - Config name: predictoors_cluster - Deployment method: k8s - Number of agents: 2 -To deploy: pdr deployer deploy predictoors_cluster -``` - -```shell -$ pdr deployer deploy predictoors_cluster -p gcp -r europe-west2 --project_id id_goes_here -Deploying predictoors_cluster... -Authenticating to Kubernetes cluster... -Fetching cluster endpoint and auth data. -kubeconfig entry generated for predictoors_cluster. -Cluster is ready, deploying the agents... -namespace/predictoors_cluster created -deployment.apps/pdr-predictoor1-3-btc-usdt-5m-binance created -deployment.apps/pdr-predictoor2-3-eth-usdt-5m-binance created -``` - -```shell -$ pdr deployer logs predictoors_cluster -Getting logs for predictoors_cluster... -Getting cluster logs... -NAME READY STATUS RESTARTS AGE -pdr-predictoor1-3-btc-usdt-5m-binance-1294c5aa3-fjc65 1/1 Running 0 91s -pdr-predictoor2-3-eth-usdt-5m-binance-21dfcf3bc4-b6nnk 1/1 Running 0 91s --> Submit predict tx result: success. -==================================================================================================================================================================================== -cur_epoch=5688716, cur_block_number=4658908, cur_timestamp=1706615099, next_slot=1706615100, target_slot=1706615400. 295 s left in epoch (predict if <= 30 s left). s_per_epoch=300 -... -``` diff --git a/READMEs/barge-calls.md b/READMEs/barge-calls.md deleted file mode 100644 index 02faf6ae0..000000000 --- a/READMEs/barge-calls.md +++ /dev/null @@ -1,43 +0,0 @@ - -# Barge flow of calls - -From getting barge going, here's how it calls specific pdr-backend components and passes arguments. - -- user calls `/barge/start_ocean.sh` to get barge going - - then, `start_ocean.sh` fills `COMPOSE_FILES` incrementally. Eg `COMPOSE_FILES+=" -f ${COMPOSE_DIR}/pdr-publisher.yml"` - - `barge/compose-files/pdr-publisher.yml` sets: - - `pdr-publisher: image: oceanprotocol/pdr-backend:${PDR_BACKEND_VERSION:-latest}` - - `pdr-publisher: command: publisher` - - `pdr-publisher: networks: backend: ipv4_address: 172.15.0.43` - - `pdr-publisher: environment:` - - `RPC_URL: ${NETWORK_RPC_URL}` (= `http://localhost:8545` via `start_ocean.sh`) - - `ADDRESS_FILE: /root/.ocean/ocean-contracts/artifacts/address.json` - - (many `PRIVATE_KEY_*`) - - then, `start_ocean.sh` pulls the `$COMPOSE_FILES` as needed: - - `[ ${FORCEPULL} = "true" ] && eval docker-compose "$DOCKER_COMPOSE_EXTRA_OPTS" --project-name=$PROJECT_NAME "$COMPOSE_FILES" pull` - - then, `start_ocean.sh` runs docker-compose including all `$COMPOSE_FILES`: - - `eval docker-compose "$DOCKER_COMPOSE_EXTRA_OPTS" --project-name=$PROJECT_NAME "$COMPOSE_FILES" up --remove-orphans` - - it executes each of the `"command"` entries in compose files. - - (Eg for pdr-publisher.yml, `"command" = "publisher ppss.yaml development"`) - - Which then goes to `pdr-backend/entrypoint.sh` via `"python /app/pdr_backend/pdr $@"` - - (where `@` is unpacked as eg `publisher ppss.yaml development`) [Ref](https://superuser.com/questions/1586997/what-does-symbol-mean-in-the-context-of#:). - - Then it goes through the usual CLI at `pdr-backend/pdr_backend/util/cli_module.py` - -## How to make changes to calls - -If you made a change to pdr-backend CLI interface, then barge must call using the updated CLI command. - -How: - -- change the relevant compose file's `"command"`. Eg change `barge/compose-files/pdr-publisher.yml`'s "command" value to `publisher ppss.yaml development` -- also, change envvar setup as needed. Eg in compose file, remove `RPC_URL` and `ADDRESS_FILE` entry. -- ultimately, ask: "does Docker have everything it needs to succesfully run the component?" - -### All Barge READMEs - -- [barge.md](barge.md): the main Barge README -- [barge-calls.md](barge-calls.md): order of execution from Barge and pdr-backend code -- [release-process.md](release-process.md): pdr-backend Dockerhub images get published with each push to `main`, and sometimes other branches. In turn these are used by Barge. diff --git a/READMEs/barge.md b/READMEs/barge.md deleted file mode 100644 index eccab900f..000000000 --- a/READMEs/barge.md +++ /dev/null @@ -1,109 +0,0 @@ - - -# Barge - -Barge is a Docker container to run a local Ganache network having Predictoor contracts and (optionally) local bots. This README describes how to install Barge, and provides reference on running it with various agents. - -⚠️ If you're on MacOS or Windows, we recommend using a remotely-run Barge. See [vps flow](vps.md). - -## Contents - -Main: - -- [Install Barge](#install-barge) - -Reference: how to run barge with... - -- [No agents](#barge-basic) - just ganache network & Predictoor contracts -- [One agent](#barge-one-agent) - eg trueval bot -- [All agents](#barge-all-agents) - predictoor, trader, trueval, dfbuyer bots - -Finally: - -- [Change Barge Itself](#change-barge-itself) - -## Install Barge - -**First, ensure pre-requisites:** [Docker](https://docs.docker.com/engine/install/), [Docker Compose](https://docs.docker.com/compose/install/), [allowing non-root users](https://www.thegeekdiary.com/run-docker-as-a-non-root-user/) - -**Then, install barge.** Open a new console and... - -```console -# Grab repo -git clone https://github.com/oceanprotocol/barge -cd barge - -# Clean up previous Ocean-related dirs & containers (optional but recommended) -rm -rf ~/.ocean -./cleanup.sh -docker system prune -a --volumes -``` - -**Then, get Docker running.** To run barge, you need the Docker engine running. Here's how: - -- If you're on Linux: you're good, there's nothing extra to do - -Congrats! Barge is installed and ready to be run. - -The sections below describe different ways to run barge. They're for reference only; DO NOT run them right now. Each README will describe what to do. - -## Barge Basic - -Barge with basic Predictoor components is: - -- local chain (ganache) -- predictoor-related smart contracts deployed to chain - -To run this, go to the barge console and: - -```console -./start_ocean.sh --no-provider --no-dashboard --predictoor --with-thegraph -``` - -When barge runs, it will auto-publish DT3 tokens. Currently this is {`BTC/USDT`, Binance, 5min}, {`ETH/USDT`, Kraken, 5min} and {`XRP/USDT`, Binance, 5min}. - -## Barge One Agent - -Barge can run with any subset of the agents. - -For example, to run barge with just trueval agent: - -```console -./start_ocean.sh --no-provider --no-dashboard --predictoor --with-thegraph --with-pdr-trueval -``` - -## Barge All Agents - -To run with all agents: - -```console -./start_ocean.sh --no-provider --no-dashboard --predictoor --with-thegraph --with-pdr-trueval --with-pdr-trader --with-pdr-predictoor --with-pdr-publisher --with-pdr-dfbuyer -``` - -This will run all of the following at once: - -- local chain (ganache) -- predictoor-related smart contracts deployed to chain -- trueval agent -- trader agent -- predictoor agent -- dfbuyer agent - -## Change Barge Itself - -Some subcomponents of Barge are those from pdr-backend. If you change those components, then the new behavior becomes part of Barge upon the next Github Release of pdr-backend. See [release-process.md](release-process.md) Docker / Barge section for details. - -For each other subcomponent of Barge, you need to change its respective repo similarly. - -And for Barge core functionality, make changes to the [barge repo](https://github.com/oceanprotocol/barge) itself. - -More info: [Barge flow of calls](barge-calls.md) - -## All Barge READMEs - -- [barge.md](barge.md): the main Barge README -- [barge-calls.md](barge-calls.md): order of execution from Barge and pdr-backend code -- [release-process.md](release-process.md): pdr-backend Dockerhub images get published with each push to `main`, and sometimes other branches. In turn these are used by Barge. diff --git a/READMEs/clean-code.md b/READMEs/clean-code.md deleted file mode 100644 index a8902450e..000000000 --- a/READMEs/clean-code.md +++ /dev/null @@ -1,104 +0,0 @@ - - -# Clean Code ✨ Guidelines - -Guidelines for core devs to have clean code. - -Main policy on PRs: - -> ✨ **To merge a PR, it must be clean.** ✨ (Rather than: "merge the PR, then clean up") - -Clean code enables us to proceed with maximum velocity. - -## Summary - -Clean code means: - -- No DRY violations -- Great labels -- Dynamic type-checking -- Tik-tok, refactor-add -- No "TODOs" -- Passes smell test -- Have tests always. TDD -- Tests are fast -- Tests are clean too - -This ensures minimal tech debt, so we can proceed at maximum velocity. Senior engineers get to spend their time contributing to features, rather than cleaning up. - -Everyone can "up their game" by being diligent about this until it becomes second nature; and by reading books on it. - -The following sections elaborate: - -- [What does clean code look like?](#what-does-clean-code-look-like) -- [Benefits of clean code](#benefits-of-clean-code) -- [Reading list](#reading-list) - -## What does clean code look like? - -**No DRY violations.** This alone avoids many complexity issues. - -**Great labels.** This makes a huge difference to complexity and DX too. - -- Rule of thumb: be as specific as possible, while keeping character count sane. Eg "ohlcv_data_factory" vs "parquet_data_factory". -- In functions of 2-5 lines you can get away with super-short labels like "c" as long as it's local, and the context makes it obvious. -- Generally: "how easily can a developer understand the code?" Think of yourself that's writing for readers, where the reader is developers (including yourself). - -**Dynamic type-checking**, via @enforce_typing + type hints on variables. - -- Given that bugs often show up as type violations, think of dynamic type-checking as a robot that automatically hunts down those bugs on your behalf. Doing dynamic type-checking will save you a ton of time. -- Small exception: in 2% of cases it's overkill, eg if your type is complex or if you're fighting with mock or mypy; then skip it there. - -**Tik-tok, refactor-add.** That is: for many features, it's best to spend 90% of the effort to refactor first (and merge that PR). Then the actual change for the feature itself is near-trivial, eg 10% of the effort. - -- It's "tik tok", where "tik" = refactor (unit tests typically don't change), and "tok" = make change. -- Inspiration: this is Intel's approach to CPU development, where "tik" = change manufacturing process, "tok" = change CPU design. - -**No "TODOs".** If you have a "TODO" that makes sense for the PR, put it in the PR. Otherwise, create a separate issue. - -**Does it pass the smell test?** If you feel like your code smells, you have work to do. - -**Have tests always. Use TDD.** - -- Coverage should be >90%. -- You should be using test-driven development (TDD), ie write the tests at the same time as the code itself in very rapid cycles of 30 s - 5 min. The outcome: module & test go together like a hand & glove. -- Anti-pattern outcome: tests are very awkward, having to jump through hoops to do tests. -- If you encounter this anti-pattern in your new code or see it in existing code, refactor to get to "hand & glove". - -**Tests run fast.** Know that if a test module publishes a feed on barge, it adds another 40 seconds to overall test runtime. So, mock aggressively. I recently did this for trader/, changing runtime from 4 min to 2 s (!). - -**Tests are clean too.** They're half the code after all! That is, for tests: no DRY violations; great labels; dynamic type-checking; no TODOs; passes "smell test" - -## Benefits of clean code - -_Aka "Why this policy?"_ - -- Helps **ensure minimal technical debt** -- Which in turn means we can **proceed at maximum velocity**. We can make changes, refactor, etc with impunity -- I was going to add the point "we're no longer under time pressure to ship features." However that should never be an excuse, because having tech debt slows us down from adding new features! Compromising quality _hurts_ speed, not helps it. Quality comes for free. -- From past experience, often "clean up later" meant "never" in practical terms. Eg sometimes it's put into a separate github issue, and that issue gets ignored. -- Senior engineers should not find themselves myself on a treadmill cleaning up after merged PRs. This is high opportunity cost of them not spending time on what they're best at (eg ML). -- What senior engineers _should_ do: use PR code reviews to show others how to clean up. And hopefully this is a learning opportunity for everyone over time too:) - -## Reading list - -To "up your game", here are great books on software engineering, in order to read them. - -- Code Complete 2, by Steve McConnell. Classic book on code construction, filled to the brim with practical tips. [Link](https://www.goodreads.com/book/show/4845.Code_Complete) -- Clean Code: A Handbook of Agile Software Craftsmanship, by Robert C. Martin. [Link](https://www.goodreads.com/book/show/3735293-clean-code) -- A Philosophy of Software Design, by John Osterhout. Best book on managing complexity. Empasizes DRY. If you've been in the coding trenches for a while, this feels like a breath of fresh air and helps you to up your game further. [Link](https://www.goodreads.com/book/show/39996759-a-philosophy-of-software-design). -- Refactoring: Improving the Design of Existing Code, by Martiwn Fowler. This book is a big "unlock" on how to apply refactoring everywhere like a ninja. [Link](https://www.goodreads.com/book/show/44936.Refactoring). -- Head First Design Patterns, by Eric Freeman et al. Every good SW engineer should have design patterns in their toolbox. This is a good first book on design patterns. [Link](https://www.goodreads.com/book/show/58128.Head_First_Design_Patterns) -- Design Patterns: Elements of Reusable Object-Oriented Software, by GOF. This is "the bible" on design patterns. It's only so-so on approachability, but nonetheless the content makes it worth it. But start with "Head First Design Patterns". [Link](https://www.goodreads.com/book/show/85009.Design_Patterns) -- The Pragmatic Programmer: From Journeyman to Master, by Andy Hunt and Dave Thomas. Some people really love this, I found it so-so. But definitely worth a read to round out your SW engineering. [Link](https://www.goodreads.com/book/show/4099.The_Pragmatic_Programmer) - -A final one. In general, _when you're coding, you're writing_. Therefore, books on crisp writing are also books about coding (!). The very top of this list is [Strunk & White Elements of Style](https://www.goodreads.com/book/show/33514.The_Elements_of_Style). It's sharper than a razor blade. - -## Recap - -Each PR should always be both "make it work" _and_ "make it good (clean)". ✨ - -It will pay off, quickly. diff --git a/READMEs/dependencies.md b/READMEs/dependencies.md index 2d7e9edc0..17b1d34ba 100644 --- a/READMEs/dependencies.md +++ b/READMEs/dependencies.md @@ -1,8 +1,3 @@ - - # Dependency Management This is a document for devs to understand how dependencies are managed in the `pdr-backend` project. @@ -15,9 +10,6 @@ It also serves users who want to understand any issues and caveats in the instal For more details, see the [the pytest-asyncio changelog](https://pytest-asyncio.readthedocs.io/en/latest/reference/changelog.html#id1), under Known Issues. The library itself recommends using version 0.21 until the issue is resolved. -- `web3` is frozen at version 6.20.2. This is because of external dependencies with Barge and transactions. We are currently working on a solution to upgrade to the latest version. -More details can be found [here](https://github.com/oceanprotocol/pdr-backend/issues/1592) in the issue. - ## For new dependencies: Types Management For type checking, we use mypy, which may require dependencies for type hints. @@ -60,11 +52,6 @@ These are due to the nature of the pytest tests and are not expected to be resol ### "Temporary" suppressions: check and remove -#### Warnings related to selenium/dash-testing -- `ignore:.*HTTPResponse.getheader\(\) is deprecated.*` -> due to usage of getheader in selenium - -If you upgrade selenium or dash[testing], you should check if these warnings are still present and remove the ignore statement in `pytest.ini` if they are not. - #### Warnings related to plotly - `ignore:.*setDaemon\(\) is deprecated, set the daemon attribute instead.*:DeprecationWarning` -> due to usage of `kaleido` and `plotly` diff --git a/READMEs/dev.md b/READMEs/dev.md deleted file mode 100644 index cc1e4bf94..000000000 --- a/READMEs/dev.md +++ /dev/null @@ -1,136 +0,0 @@ - -# Usage for Backend Devs - -This is for core devs to improve pdr-backend repo itself. - -## Install pdr-backend - -Follow directions to install pdr-backend in [predictoor.md](predictoor.md) - -## Setup Barge - -**Local barge.** If you're on ubuntu, you can run barge locally. - -- First, [install barge](barge.md#install-barge). -- Then, run it. In barge console: `./start_ocean.sh --no-provider --no-dashboard --predictoor --with-thegraph` - -**Or, remote barge.** If you're on MacOS or Windows, run barge on VPS. - -- Follow the instructions in [vps.md](vps.md) - -### Setup dev environment - -Open a new "work" console and: - -```console -# Setup virtualenv -cd pdr-backend -source venv/bin/activate - -# Set PRIVATE_KEY -export PRIVATE_KEY="0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58" - -# Unit tests default to using "development" network -- a locally-run barge. -# If you need another network such as barge on VPS, then override the endpoints for the development network -``` - -All other settings are in [`ppss.yaml`](../ppss.yaml). Some of these are used in unit tests. Whereas most READMEs make a copy `my_ppss.yaml`, for development we typically want to operate directly on `ppss.yaml`. - -### Local Usage: Testing - -In work console, run tests: - -```console -# (ensure PRIVATE_KEY set as above) - -# run a single test. The "-s" is for more output. -# note that pytest does dynamic type-checking too:) -pytest pdr_backend/util/test_noganache/test_util_constants.py::test_util_constants -s - -# run all tests in a file -pytest pdr_backend/util/test_noganache/test_util_constants.py -s - -# run a single test that flexes network connection -pytest pdr_backend/util/test_ganache/test_contract.py::test_get_contract_filename -s - -# run all regular tests; see details on pytest markers to select specific suites -pytest -``` - -### Local Usage: Linting - -In work console, run linting checks. - -```console -# auto-fix some pylint complaints like whitespace. CI doesn't modify files; we do -black ./ - -# run linting on code style. Use same setup as CI -pylint --rcfile .pylintrc * pdr_backend/* - -# mypy does static type-checking and more. Use same setup as CI -mypy --config-file mypy.ini ./ -``` - -### Local Usage: Check code coverage - -In work console: - -```console -coverage run --omit="*test*" -m pytest # Run all. For subset, add eg: pdr_backend/lake -coverage report # show results -``` - -### Local Usage: Run a custom agent - -Let's say you want to change the trader agent, and use off-the-shelf agents for everything else. Here's how. - -In barge console: - -```console -# (Hit ctrl-c to stop existing barge) - -# Run all agents except trader -./start_ocean.sh --predictoor --with-thegraph --with-pdr-trueval --with-pdr-predictoor --with-pdr-publisher --with-pdr-dfbuyer -``` - -In work console: - -```console -#(ensure envvars set as above) - -# run trader agent, approach 1 -pdr trader 1 ppss.yaml development -# or -pdr trader 1 ppss.yaml barge-pytest -``` - -(You can track at finer resolution by writing more logs to the [code](../pdr_backend/predictoor/approach3/predictoor_agent3.py), or [querying Predictoor subgraph](subgraph.md).) - -## Remote Usage - -In the CLI, simply point to a different network: - -```console -# run on testnet -pdr trader ppss.yaml sapphire-testnet - -# or, run on mainnet -pdr trader ppss.yaml sapphire-mainnet -``` - -## Dependencies -See [dependencies.md](dependencies.md) for more details. - - -## Run licenseheaders -Run this once a year to update the license headers. -Since this is not something we do everyday, do not include the dependency in setup.py. Instead, install it manually using pip: - -```console -pip install licenseheaders -licenseheaders -cy -t ./copyright_template.tmpl -x venv/**.py -``` diff --git a/READMEs/dfbuyer.md b/READMEs/dfbuyer.md deleted file mode 100644 index 8b98ee9a0..000000000 --- a/READMEs/dfbuyer.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# Run a DFBuyer Bot - -This README describes how to run a dfbuyer bot. - -## Install pdr-backend - -Follow directions in [predictoor.md](predictoor.md) - -## Local Network - -First, [install barge](barge.md#install-barge). - -Then, run barge. In barge console: - -```console -#run barge with all bots (agents) except dfbuyer -./start_ocean.sh --no-provider --no-dashboard --predictoor --with-thegraph --with-pdr-trueval --with-pdr-predictoor --with-pdr-publisher --with-pdr-trader -``` - -Open a new console and: - -```console -# Setup virtualenv -cd pdr-backend -source venv/bin/activate - -# Set envvar -export PRIVATE_KEY="0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58" -``` - -Copy [`ppss.yaml`](../ppss.yaml) into your own file `my_ppss.yaml` and change parameters as you see fit. The section "dfbuyer_ss" has parameters for this bot. - -Then, run dfbuyer bot. In console: - -```console -pdr dfbuyer my_ppss.yaml development -``` - -The bot will consume "weekly_spending_limit" worth of assets each week. This amount is distributed equally among all DF eligible assets. (This parameter is set in the yaml file.) - -![flow](https://user-images.githubusercontent.com/25263018/269256707-566b9f5d-7e97-4549-b483-2a6700826769.png) - -## Remote Usage - -In the CLI, simply point to a different network: - -```console -# run on testnet -pdr dfbuyer my_ppss.yaml sapphire-testnet - -# or, run on mainnet -pdr dfbuyer my_ppss.yaml sapphire-mainnet -``` diff --git a/READMEs/diagrams/lake.html b/READMEs/diagrams/lake.html deleted file mode 100644 index 2d678a4b0..000000000 --- a/READMEs/diagrams/lake.html +++ /dev/null @@ -1,114 +0,0 @@ - - - ETL Architecture - - - - - - -
- erDiagram - SUBGRAPH_predictPredictions ||--o{ PDR_PREDICTIONS : yields - SUBGRAPH_predictPayouts ||--o{ PDR_PAYOUTS : yields - SUBGRAPH_predictTrueVals ||--o{ PDR_TRUEVALS : yields - SUBGRAPH_predictSlots ||--o{ PDR_SLOTS : yields - SUBGRAPH_predictSubscriptions ||--o{ PDR_SUBSCRIPTIONS : yields - PDR_PREDICTIONS ||--o{ BRONZE_PDR_PREDICTIONS : yields - PDR_PAYOUTS ||--o{ BRONZE_PDR_PREDICTIONS : yields - PDR_PREDICTIONS { - string ID - string contract - string pair - string timeframe - boolean predvalue - float stake - boolean truevalue - int timestamp - string source - float payout - int slot - string user - } - PDR_PAYOUTS { - string ID - string user - int timestamp - string token - int slot - float payout - boolean predvalue - boolean truevalue - float stake - } - PDR_TRUEVALS { - string ID - boolean truevalue - int timestamp - string token - int slot - float revenue - float roundSumStakesUp - float roundSumStakes - } - PDR_SLOTS { - string ID - int timestamp - int slot - boolean truevalue - float roundSumStakesUp - float roundSumStakes - } - PDR_SUBSCRIPTIONS { - string ID - string pair - string timeframe - string source - string tx_id - float last_price_value - int timestamp - string user - } - BRONZE_PDR_PREDICTIONS { - string ID - string slot_id - string contract - int slot - string user - string pair - string timeframe - string source - boolean predvalue - boolean truevalue - float stake - float revenue - float payout - int timestamp - int last_event_timestamp - } -
- - \ No newline at end of file diff --git a/READMEs/get-ocean-on-sapphire.md b/READMEs/get-ocean-on-sapphire.md deleted file mode 100644 index 35de92878..000000000 --- a/READMEs/get-ocean-on-sapphire.md +++ /dev/null @@ -1,75 +0,0 @@ - -# Get OCEAN on Sapphire Mainnet - -OCEAN has address [`0x39d22B78A7651A76Ffbde2aaAB5FD92666Aca520`](https://explorer.oasis.io/mainnet/sapphire/token/0x39d22B78A7651A76Ffbde2aaAB5FD92666Aca520) on Sapphire. - -There are two approaches to acquire it: - -1. Use an OCEAN-ROSE DEX on Sapphire -2. Bridge OCEAN from Eth mainnet - -(1) is much simpler. (2) isn't subject to slippage. We cover both below. - -## Approach 1: OCEAN-ROSE DEX on Sapphire - -How to use: - -1. Go to **[Illuminex OCEAN-ROSE pool](https://illuminex.xyz/swap?inputCurrency=0x39d22B78A7651A76Ffbde2aaAB5FD92666Aca520&inputChainId=23294&outputCurrency=0x8Bc2B030b299964eEfb5e1e0b36991352E56D2D3&outputChainId=23294)**. -2. Specify amount to send, and destination address -3. Click "Swap" - -Notes: - -- If you see a "Insufficient liquidity" warning, then click the settings gear and toggle "Use secure router = on". -- Be careful for slippage, always. (Though there _should_ be decent liquidity. [Pool info](https://illuminex.xyz/pools/0x841dd137A2B380DA4568f6745aEAc20EDa910313)) - -## Approach 2: Bridge OCEAN from Eth mainnet - -**This approach has three steps:** - -1. [Get OCEAN on Eth Mainnet](#get-ocean-on-eth-mainnet) -1. [Transfer OCEAN to Sapphire Mainnet via Celer](#transfer-ocean-to-sapphire-mainnet-via-celer) -1. [Verify Your Tokens on Sapphire Mainnet](#verify-your-tokens-on-sapphire-mainnet) - -## Get OCEAN on Eth Mainnet - -**First, create a Wallet.** For optimal support and the ability to transfer ROSE between paratimes, we recommend the Oasis Wallet extension for Chrome. To proceed, [follow instructions in its docs](https://docs.oasis.io/general/manage-tokens/oasis-wallets/browser-extension/#install-the-oasis-wallet-via-chrome-web-store). - -**Then, purchase OCEAN.** First, acquire OCEAN at an exchange like Binance, Coinbase, Gate.io, or KuCoin. Then, withdraw that OCEAN to your web3 wallet. - -⚠️ Sending to the wrong address or network will likely result in the loss of your tokens. To minimize risk: send a small test amount first, and double-check the receiving address and network. - -## Transfer OCEAN to Sapphire Mainnet via Celer - -**First, Navigate to Celer bridge.** [Here](https://cbridge.celer.network/1/23294/OCEAN). - -**Then, Connect Your Wallet.** Click on the "Connect Wallet" button. Follow the prompts to select and connect your web3 wallet. - -**Then, Configure Transfer Settings:** - -- Ensure the "From" field is set to "Ethereum Mainnet." -- Confirm "Oasis Sapphire" is selected as the destination on the "To" field. -- Input the amount of OCEAN you desire to transfer in the designated field. - -**Finally, Execute the Transfer:** - -- Click on the "Transfer" button. -- Review the transaction details. -- Confirm the transaction from your wallet. - -## Verify Your Tokens on Sapphire Mainnet - -Bridging requires several minutes to complete. Your OCEAN will be sent to the address you executed the transaction from. - -To verify: - -- See the tx on [Oasis Explorer](https://explorer.sapphire.oasis.io/address/0x39d22B78A7651A76Ffbde2aaAB5FD92666Aca520/transactions). -- Or, import the OCEAN token contract address `0x39d22B78A7651A76Ffbde2aaAB5FD92666Aca520` into your wallet. Set symbol = `OCEAN` and decimal precision = `18`. - -Congrats! You've successfully transferred OCEAN to Sapphire Mainnet. - -> [!WARNING] -> Disclaimer: This guide does not provide financial advice. Always conduct thorough research or consult a professional before making investment decisions. Make sure to verify URLs and ensure all information aligns with the current operations and networks of the related platforms as cryptocurrency ecosystems frequently update and evolve. Always ensure to use official and secure websites when conducting transactions. diff --git a/READMEs/get-rose-on-sapphire.md b/READMEs/get-rose-on-sapphire.md deleted file mode 100644 index 99aaf15d9..000000000 --- a/READMEs/get-rose-on-sapphire.md +++ /dev/null @@ -1,96 +0,0 @@ - -# Get ROSE on Sapphire Mainnet - -Steps: - -1. [Install Oasis wallet](#1-install-oasis-wallet) -2. [Purchase ROSE](#2-purchase-rose) -3. [Prepare Sapphire EVM account](#3-prepare-sapphire-evm-account) -4. [Transfer ROSE from Oasis Account to Sapphire](#4-transfer-rose-from-oasis-account-to-sapphire) - -Bonus: [Sapphire \* Metamask](#5-bonus-sapphire-evm-account-in-metamask) - -## 1. Install Oasis Wallet - -First, some background: - -- **ParaTimes.** "ParaTimes" are subnets in the Oasis chain ecosystem, all reporting to the same "consensus layer". Sapphire, Cipher, and Emerald are Oasis ParaTimes. Each ParaTime has its own independent state. So, ROSE held at the consensus layer is not the same as in Sapphire. But you can swap between them via "deposits" and "withdrawals" in the Oasis wallet. -- **Sapphire.** Each ParaTime has its own specialty. Sapphire's specialty is EVM-compatibility & confidentiality. So, it uses EVM accounts, versus Oasis-native accounts (starts with "0x" vs with "oasis"). -- **EVM accounts.** EVM = Ethereum Virtual Machine. EVM-compatible means that it runs Ethereum-style contracts. EVM accounts = Ethereum-style accounts, including an address that starts with "0x" and support for ERC20 tokens. - -The **Oasis wallet extension (for Chrome) is the best wallet to transfer ROSE among ParaTimes.** (Metamask is good for pure EVM work in Sapphire. We'll work with that later.) - -To install it, go to [Oasis Wallet extension docs](https://docs.oasis.io/general/manage-tokens/oasis-wallets/browser-extension/#install-the-oasis-wallet-via-chrome-web-store) and follow its instructions. - -## 2. Purchase ROSE - -First, acquire ROSE at an exchange like Binance, Coinbase, Gate.io, or KuCoin. For a full list, see [CoinGecko Oasis page](https://www.coingecko.com/en/coins/oasis-network) -> "Markets" section. - -Then, you'll be withdrawing ROSE to your web3 wallet. - -- If the exchange supports withdrawal directly to Oasis Sapphire Paratime: great, do that directly! Then you can skip step 3. -- It it doesn't support direct: first, send to Oasis Emerald Network. Then, follow step 3. - -⚠️ Sending to the wrong address or network will likely result in the loss of your tokens. To minimize risk: send a small test amount first, and double-check the receiving address and network. - -## 3. Prepare Sapphire EVM account - -First, make sure you have an EVM account ready for use in Sapphire: you have an EVM address, and you know its private key. - -Then, import it into the Oasis wallet extension: - -1. Click "Import" button -2. Select "Ethereum-compatible Private Key" -3. Enter account name - whatever you like; click ok -4. Now you will see your new account listed, in the "Ethereum-compatible Account" subsection - -Now, let's check your EVM wallet's ROSE balance, via the Oasis wallet extension. - -1. There's a circle on the top-right corner. Click it. Now it shows all your Oasis-native _and_ EVM-native accounts. -2. Select your EVM account by clicking on it. Now it has a green check next to it. -3. On the app's very top-left (you may need to scroll), click the left-pointing arrow. Now it shows your EVM account's summary. -4. Click "ParaTimes". Now it shows ROSE balance in each ParaTime for each. Note that your Sapphire account has 0.0 ROSE. - -## 4. Transfer ROSE from Oasis Account to Sapphire - -Here, we use the Oasis wallet extension to transfer ROSE from your Oasis-native account to your EVM Sapphire account. - -1. Go to the "ParaTimes" section -2. In the "Sapphire" box, click on "To ParaTime" -3. In the "Amount" field, specify the # ROSE; in the "To" field, enter your EVM account address (0x...). -4. Click "Next". Click to confirm. -5. The wallet will show an animation; then it will say "submitted"; then it will say "Successful" and share tx details. - -To verify that the tx went through: - -- Verify "withdraw" side: in Oasis wallet, in the Oasis account you sent from, in its tx history, click on the most recent tx. Copy the the txHash value. Finally, go to [oasisscan.com](https://www.oasisscan.com/) and paste in the txHash value. -- But note that its "to" address is an oasis address, _not_ your EVM address. This address is the middleman bridge from your Oasis-native sending account to your EVM-native receiving account. So we still need to verify the "deposit" side, in the EVM account. -- Verify "deposit" side: via the Oasis wallet extension, check your EVM wallet's balance as described above). The balance should have increased by the amount of ROSE that you sent. - -The official Oasis guide has details: [How to Transfer ROSE into a ParaTime](https://docs.oasis.io/general/manage-tokens/how-to-transfer-rose-into-paratime) - -## 5. Bonus: Sapphire EVM Account in Metamask - -Metamask supports Sapphire EVM. Here's how to use it. - -To add Sapphire network: - -1. Go to `https://docs.oasis.io/dapp/sapphire/`. -2. In that page click "click here to register Sapphire Mainnet on Metamask". -3. Confirm the prompts to switch to Sapphire network - -To add your Sapphire EVM account: - -1. In Metamask, in top middle "account" subsection, click the dropdown -2. Click "Add account or hardware wallet". Follow the prompt. - -To see your ROSE balance: - -1. In Metamask, click "Tokens" -2. Observe the ROSE balance. - -> [!WARNING] -> Disclaimer: This guide does not provide financial advice. Always conduct thorough research or consult a professional before making investment decisions. Make sure to verify URLs and ensure all information aligns with the current operations and networks of the related platforms as cryptocurrency ecosystems frequently update and evolve. Always ensure to use official and secure websites when conducting transactions. diff --git a/READMEs/get-tokens.md b/READMEs/get-tokens.md deleted file mode 100644 index d1bd07171..000000000 --- a/READMEs/get-tokens.md +++ /dev/null @@ -1,12 +0,0 @@ - -# Get Tokens - -Sapphire accounts and wallets follow EVM standard. - -Your wallet account on Sapphire needs both ROSE (for gas) and OCEAN (for using Predictoor). - -- On Sapphire testnet, get (fake) [ROSE & OCEAN via the faucet guide](./testnet-faucet.md). -- On Sapphire mainnet, get [ROSE via this guide](get-rose-on-sapphire.md) and [OCEAN via this guide](get-ocean-on-sapphire.md). diff --git a/READMEs/images/dockerbranch.png b/READMEs/images/dockerbranch.png deleted file mode 100644 index bd695e748..000000000 Binary files a/READMEs/images/dockerbranch.png and /dev/null differ diff --git a/READMEs/images/dockerbuild.png b/READMEs/images/dockerbuild.png deleted file mode 100644 index 800ce7408..000000000 Binary files a/READMEs/images/dockerbuild.png and /dev/null differ diff --git a/READMEs/images/etl_checkpoints.png b/READMEs/images/etl_checkpoints.png deleted file mode 100644 index b63c0155d..000000000 Binary files a/READMEs/images/etl_checkpoints.png and /dev/null differ diff --git a/READMEs/images/etl_swap_from_temp_to_prod_atomic.png b/READMEs/images/etl_swap_from_temp_to_prod_atomic.png deleted file mode 100644 index 9628b8289..000000000 Binary files a/READMEs/images/etl_swap_from_temp_to_prod_atomic.png and /dev/null differ diff --git a/READMEs/images/gql_data_factory_fetch.png b/READMEs/images/gql_data_factory_fetch.png deleted file mode 100644 index eeaa50115..000000000 Binary files a/READMEs/images/gql_data_factory_fetch.png and /dev/null differ diff --git a/READMEs/images/gql_data_factory_fetch_1_write_n.png b/READMEs/images/gql_data_factory_fetch_1_write_n.png deleted file mode 100644 index 0f0d5b204..000000000 Binary files a/READMEs/images/gql_data_factory_fetch_1_write_n.png and /dev/null differ diff --git a/READMEs/images/gql_use_cache_and_swap.png b/READMEs/images/gql_use_cache_and_swap.png deleted file mode 100644 index 7c0449e38..000000000 Binary files a/READMEs/images/gql_use_cache_and_swap.png and /dev/null differ diff --git a/READMEs/images/lake_tables_diagram.png b/READMEs/images/lake_tables_diagram.png deleted file mode 100644 index 29e13e059..000000000 Binary files a/READMEs/images/lake_tables_diagram.png and /dev/null differ diff --git a/READMEs/lake-and-etl.md b/READMEs/lake-and-etl.md deleted file mode 100644 index af26b972a..000000000 --- a/READMEs/lake-and-etl.md +++ /dev/null @@ -1,251 +0,0 @@ - - -# Lake & ETL Overview - -This README describes how you can operate the lake and use analytics to understand the world of predictoor. - -First, we want to make sure we get all the data we'll need! Then, we'll process this data so it's easy to chart and understand things. - -To complete this we'll be interacting different components in our stack, such as the: Lake, GQLDataFactory, and the ETL. - - -## Content -- [**ETL Tables & Architecture**](#etl-tables--architecture) -- [**Lake - "Storage"**](#lake---storage) -- [**GQL Data Factory & RAW**](#gql-data-factory--raw-data) -- [**Data Warehouse - "Database"**](#data-warehouse---database) -- [**ETL**](#etl) -- [**Checkpoints**](#checkpoints) -- [**Dumb Components**](#dumb-components) -- [**Export to parquet files**](#export-to-parquet-file) -- [**Examples**](#examples) -- [**Dos and Don'ts**](#dos-and-donts) - - -## ETL Tables & Architecture -To provide summaries on Predictoor users and systems, we want to fetch all new data and make it available for analysis. - -The GraphQL Data Factory & ETL helps us achieve this by breaking the process down into 3 steps. -1. Ingesting + Loading Data - GQLDataFactory -1. Processing the data - ETL -1. Describing, visualization, and querying the data - -The following diagram presents the ETL tables and workflow. Please maintain the [diagram](diagrams/lake.html) and image so they are up-to-date. - -![Lake & ETL Tables](images/lake_tables_diagram.png) - - -## Lake - "Storage" -For most users, the lake can simply be thought of disk storage. - -It keeps all records stored as a timeseries CSV with a max number of records, where you can search and sample sections of the data if required. - -In python, the lake (raw) data can be accessed via CSVDataStore (a CSV wrapper) and the CLI Lake module (cli_module_lake.py). - -As a user, you can operate the lake via the CLI command: `pdr lake raw update ppss.yaml sapphire-mainnet` - -Some features include: -1. The lake has a few commands: `describe`, `validate`, `update`, `drop`, and `query` -1. The lake is made up of many timeseries objects that should exist from `lake_ss.st_ts` and `lake_ss.end_ts`. -1. All the data inside lake folders should exist between `st_ts` and `end_ts`. -1. The `st_ts` and `end_ts` checkpoints come from `ppss.yaml` and the `lake_ss` component. -1. You can set `st_ts` and `end_ts` to relative times ("3 hours ago" -> "now"), so new records are always fetched and processed. -1. The lake is designed so raw data is fetched only once from subgraph and then saved into `lake_data/` and DuckDB. - -**Note:** The lake currently does not support backfilling. If you need to, drop all data, adjust `st_ts` and `end_ts`, and then fetch again. - -![GQLDF Fetch & Write](images/gql_data_factory_fetch_1_write_n.png) - - -## GQL Data Factory & Raw Data -Is responsible for running a series of graphql fetches against subgraphs and then saves this raw data against 1->N buckets. - -GQLDF features a buffer, and will write out records as they arrive. Because ingesting is expensive, we use this routine to write it to (local) disk as CSVs, an into our (local) Data Warehouse (DuckDB). - -When the GQL Data Factory updates, it looks at each query and their respective `lake_data/` folder to figure out where to resume from. Once all data has been fetched and loaded into `lake_data/` and the Data Warehouse (DuckDB), the update routine ends. - -**Note:** The lake is designed to only be filled once. By default GQL Data Factory will not delete data from `lake_data/`. - -**[DX-Self-Healing Lake]** - -If you delete RAW CSV data sequentially from the end, i.e. "drop_tail()" then GQLDF will attempt to self-heal and simply fill in the data that it's missing. - -![GQLDF Fetch & Write](images/gql_data_factory_fetch.png) - -**[DX-How to clear your Lake and start from scratch]** -1. Delete the `lake_data` folder -```console -sudo rm -r lake_data/ -``` - -2. Fetch the raw data (if required, update `st_ts` and `end_ts`) -```console -pdr lake raw update ppss.yaml sapphire-mainnet -``` - -## Data Warehouse - "Database" -For most users, the Data Warehouse (DuckDB) is just a database. - -To deliver a complete data stack capable of storing, processing, and enriching all of our data, we load all CSVs (lake) into DuckDB (DW) and then process this data through a series of SQL queries (ETL) inside the DW. - -Currently, the DW (DB) can be accessed via the DuckDBDataStore (DuckDB Wrapper) and operated with via the CLI module `lake` commands `pdr lake etl update ppss.yaml sapphire-mainnet` (cli_module_lake.py) - -Some features include: -1. The DW is atomic and designed to be destroyed/rebuilt by leveraging raw data from `lake_data`. There are no migrations. -1. To rebuild the DW, please use a combination of `lake etl drop` and `lake etl update` commands. -1. The DW is rebuilt using a series of SQL queries that Extract-Transform-Load (ETL) into new tables. -1. All ETL SQL queries run on DuckDB and follow a "mini-batch" routine to be as efficient as possible. -1. ETL tables use a "Medallion Achitecture" (bronze, silver, gold) to deliver a structured ETL architecture. -1. All the data inside ETL tables exist between `st_ts` and `end_ts`. - -**Note:** All ETL is done on the DW to be efficient and future-proof ourselves towards distributed DBs. Do not jump between python -> sql -> python, this is computationally and memory expensive. All queries are designed to function as mini-batch opeations to reduce the amount of scans, joins, inserts, and updates. - -**[DX-How to clear your Data Warehouse without clearing your lake]** -1. Drop all records from the Data Warehouse -```console -pdr lake raw drop ppss.yaml sapphire-mainnet 2020-01-01 -pdr lake etl drop ppss.yaml sapphire-mainnet 2020-01-01 -``` - -2. Fetch if needed, load from lake, and run the full ETL pipeline -```console -pdr lake etl update ppss.yaml sapphire-mainnet -``` - - -## ETL -Is responsible for running a series of queries that transform and enrich the data so it can provide standard calculations accuately and valuable insights more readily. - -### Update from CLI -From the cli - `pdr lake etl update ppss.yaml sapphire-mainnet` - -### Data Flow -As you are developing and building around the data and lake, you might need to update how the system works, and be able to follow this workflow. -1. GQLDF completes succesfully. Fetch from source RAW => into CSV & DuckDB. -1. ETL runs SQL queries over the RAW data to clean it, join w/ other tables, transform => into Bronze Tables -1. Aggregating + Summarizing it => into Silver & Gold tables. -1. Serving this data into dahboards and other locations. - -### Workflow -As you are developing and building around the lake, your workflow migh look like this. -1. Writing & Updating ETL code -1. Dropping ETL tables via CLI -1. Rebuilding ETL tables via CLI - -### Limitations -1. No backfilling data before `st_ts`. If you want to backfill drop all records or select a new `lake_path`. - -## Checkpoints -In order to only process new data, we want to "maintain a checkpoint" that helps us know where to start/resume from. - -The simplest way to do this is to use the first and last event timestamps from our data. - -### GQL Data Factory -All CSVs should be in-sync... with nearly the same [st_ts => end_ts] across all folders. -1. GQLDF fetches [st_ts => end_ts] from CSVs if available so it can resume, otherwise it uses ppss.yaml `st_ts` -1. For each GQL, we fetch all records [st_ts => end_ts] and save them as they arrive -1. Write raw data to CSV [st_ts => end_ts] -1. Write raw data to DW [st_ts => end_ts] - -![GQLDF Fetch & Write](images/gql_use_cache_and_swap.png) - -### ETL -All ETL tables should be in-sync with nearly the same [st_ts => end_ts]. -1. ETL looks at CSV and ETL Tables last_records to identify how to resume, otherwise it uses ppss.yaml. -1. ETL first processes RAW records, before building ETL records/tables -1. RAW records are processed into NewEvents (INSERT) and UpdateEvents(UPDATE) and stored in temporary tables: [TempTable] & [TempUpdateTable]. -1. NewEvents and UpdateEvents are used to yield/update ETL tables. -1. ETL tabels follow a "Medallion Architecture": bronze, silver, gold - -![ETL Update Routine](images/etl_checkpoints.png) - -**Exceptions to the rule** -This doesn't always have to be the case. -1. RAW + ETL records could have been dropped. -2. CSV + GQLDF could be far ahead. - -**[DX-How to resolve checkpoint and data problems]** -1. Run `describe` and `validate` to make sure there are no errors. -2. Drop all RAW + ETL Tables from DW and rebuild the DW from scratch. -3. RAW Tables are rebuilt from CSV records. -4. ETL Tables are rebuilt from RAW tables. - -All systems [GQLDF + ETL + CSV + DuckDB] should be working out-of-the-box by using ppss.yaml & the cli. - - -## Examples & How To's - -### Run Lake end to end - -1. Fetch the data and create the Lake with **RAW** and **ETL** data -```console -pdr lake etl update ./ppss.yaml sapphire-mainnet -``` -2. Validate the Lake -```console -pdr lake validate ./ppss.yaml sapphire-mainnet -``` -3. Inspect the Lake -```console -pdr lake describe ./ppss.yaml sapphire-mainnet -``` -4. Query data from the Lake -```console -pdr lake query ./lake_data "SELECT * FROM pdr_predictions" -``` - - -### Modify Lake data -5. Drop data from the Lake at **ETL** level starting from a specified date -```console -pdr lake etl drop ppss.yaml sapphire-mainnet 2024-01-10 -``` - -6. Drop data from the Lake at **RAW** level starting from a specified date -```console -pdr lake raw drop ppss.yaml sapphire-mainnet 2024-01-10 -``` - -7. Recover data at **RAW** level starting from a specified date -```console -pdr lake raw update ppss.yaml sapphire-mainnet -``` - -8. Recover data at **ETL** level starting from a specified date -```console -pdr lake etl update ppss.yaml sapphire-mainnet -``` - -9. Validate the Lake -```console -pdr lake validate ppss.yaml sapphire-mainnet -``` - -## DO'S and DONT'S -**Don't**: - - !! Don't modify the CSV files in any way, otherwise data is going to be eronated !! - -**Do's**: - - If data is eronated or there is any issue with the Lake, reset the lake - - Reset the Lake by deleting `lake_data/` folder - - Recreate the Lake by running `lake raw update` and `lake etl update` commands - - -## "Dumb" Components -Lake components should be self-contained, "dumb", and perform one role. - -GQLDF is upstream from ETL and does not have any knowledge of it. Do not create dependencies where there are none. I.E. If you delete CSV data from `lake_data`, do not expect this change to be reflected in RAW or ETL tables. Similarly, ETL should assume that GQLDF is performing it's role as expected. - -Rather, help improve the lake validation and CLI tools to improve the lake DX and operations. Think about the entry points, validations, and [how to improve the user flow](https://github.com/oceanprotocol/pdr-backend/issues/1087#issuecomment-2155527087) so these checks being done outside core systems, once, while remaining user friendly. - -## Export to parquet files -By default, the ETL process exports DuckDB lake data into Parquet files stored in the exports directory. This setup allows the dashboard to read the data without interfering with the direct connection to the lake. - -The process of exporting is done periodically at a specified `seconds_between_parquet_exports` frequency inside `PPSS.yaml` config file and can be changed to the user needs. - -To nuke and re-export parquet files for periodical cleanups, the `number_of_files_after_which_re_export_db` it's used, and can be easily configured. - -Additionally, the export process can be disabled entirely by setting `export_db_data_to_parquet_files = False` in the configuration file. \ No newline at end of file diff --git a/READMEs/payout.md b/READMEs/payout.md deleted file mode 100644 index 81c93c7a6..000000000 --- a/READMEs/payout.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# Claim Payout for Predictoor Bot - -This README describes how you can claim the $ based on running a predictoor bot. - -First, congratulations on your participation and progress in making predictions! Whether you've made accurate or erroneous predictions, it's now time to claim your earnings from correct predictions. - -There are two sections below: to claim OCEAN payout, and ROSE payout, respectively. - -## Steps to Request OCEAN Rewards - -### 1. Preparation - -Ensure you pause or stop any ongoing prediction submissions. You can use `Ctrl-C` to stop the predictions. This is crucial as active submissions can interfere with the payout process. - -### 2. Execute Payout - -From console: - -```console -pdr claim_OCEAN ppss.yaml -``` - -### 3. Completion - -Once the payout module concludes its operation, your balance will reflect the updated amount. - -### 4. Verification - -It's good practice to run the payout module again. This ensures any failed blockchain calls from the previous attempt are addressed and verifies that all eligible payouts have been claimed. - -### Steps to Request ROSE Rewards - -- 1. Preparation - -- Make sure you requested your Payout mentioned above, between the DF round end and ROSE rewards distribution(on Monday, 4 days after the DF round ends), for all your predictions from the DF round to be counted in ROSE rewards calculation. -- Ensure you pause or stop any ongoing prediction submissions. You can use `Ctrl-C` to stop the predictions. This is crucial as active submissions can interfere with the payout process. - -- 2. Claim ROSE Rewards - -From console: - -```console -pdr claim_ROSE ppss.yaml -``` - -- 3. Completion - -Once the claim module concludes its operation, your balance will reflect the updated amount. - -- 4. Verification - -It's good practice to run the payout module again. This ensures any failed blockchain calls from the previous attempt are addressed and verifies that all eligible payouts have been claimed. - -## FAQ - -- Q: how often to request? -- A: While you can request a payout at any time, we suggest doing so periodically. The payout module processes requests in batches, handling up to 250 slots per transaction for efficiency. diff --git a/READMEs/predictoor-dashboard.md b/READMEs/predictoor-dashboard.md deleted file mode 100644 index 3a62a3b18..000000000 --- a/READMEs/predictoor-dashboard.md +++ /dev/null @@ -1,71 +0,0 @@ -# Predictoor Dashboard - -**The Predictoor dashboard provides insights into blockchain data through interactive plots and tables.** - -## **How it works** - -Live data from a specified supported blockchain is fetched and stored in a local database. By running a Dash dashboard, this data is processed and displayed in various formats to provide valuable information and control. - -## **How to setup and run** - -To set up and run the Predictoor dashboard, follow these steps: - -1. **Fetch chain data into the lake and export to paruqet files** - -The first step is to fetch the data from the blockchain using the lake's ETL, by this command: - -```console -pdr lake etl update ./my_ppss.yaml sapphire-mainnet -``` - -By default the lake is going to automatically export data from lake to parquet files. -Be aware that the export is executed periodically and could be configured and even disabled using this PPSS lake config params: -- export_db_data_to_parquet_files -- seconds_between_parquet_exports -- number_of_files_after_which_re_export_db - -For more information on how the lake works and how to configure it, refer to [`this readme`](./lake-and-etl.md). - -2. **Configure ppss(Optional)** - -This step is optional but very useful and **highly recommended if you are running a Predictoor bot**. - -By configuring the **ppss -> predictoor_ss -> my_addresses** list and providing one or multiple predictoor addresses, you can easily track those addresses. The app will automatically read the addresses and display all relevant data when it starts. Additionally, if you modify settings and select different Predictoors and feeds, you can easily reset the dashboard to your Predictoor settings. - -3. **Run the dash app from command line** - -After fetching the chain data locally into the lake and exporting it into parquet files, the next step is to read, process, and display the data by running the dashboard with the following command: - -```console -pdr dashboard ./my_ppss.yaml sapphire-mainnet -``` - -This command will open a browser window where you can select predictoors and feeds to visualize their performance. - -## **How to use** - -By default, if ppss is not configured with predictoor addresses, the plots will be empty until at least one predictoor and feed are selected. Otherwise you will see stats on the provided predictoor addresses. - -After **selecting one or more predictors and feeds combinations** where predictions has been submitted, and the payout method called, the plots and metrics will be populated with the relevant data. - -You can then observe how your predictoor bots or others have been performing on different feeds and even compare them. - -**IMPORTANT: Only predictions data where the payout method has been called are visible.** - -The displayed charts are: -1. **Accuracy** - how predictoor accuracy has evolved durring the selected period of time. -2. **Profit** - what is the profit in OCEAN over the period. Here you can clearly see if you are profitable or not. -3. **Costs** - how much predictoor has staked for each epoch. - - -To summarize the stats across multiple feeds and Predictoors, follow the **displayed metrics**. These include: **Average Accuracy, Total Profit, and Average Stake**. These metrics make it easy to track overall statistics for the selected Predictoors and users. - - -Furthermore, you can select different periods of time for the data calculation, so you can easily see stats on **last day, last week, or last month**. - -## **How to contribute** - -We are constantly looking for ways to improve the Predictoor dashboard to help predictoors become more profitable and are open to suggestions and ideas. - -If you are a predictoor and want to help us improve this tool, [join our Discord channel](https://discord.gg/Tvqd7Z648H) and drop a message, or open a GitHub issue. - diff --git a/READMEs/predictoor.md b/READMEs/predictoor.md deleted file mode 100644 index d1ba6c3c0..000000000 --- a/READMEs/predictoor.md +++ /dev/null @@ -1,397 +0,0 @@ - - -# Run a Predictoor Bot - -This README shows how to earn $ by running a predictoor bot on mainnet. - -**Main flow:** - -1. **[Install](#1-install-pdr-backend-repo)** -1. **[Simulate modeling & trading](#2-simulate-modeling-and-trading)** -1. **[Run bot on testnet](#3-run-predictoor-bot-on-sapphire-testnet)** -1. **[Run bot on mainnet](#4-run-predictoor-bot-on-sapphire-mainnet)** -1. **[Claim payout](#5-claim-payout)** -1. **[Run dashboard to monitor and analyze performance](#6-run-dashboard)** - -Once you're done the main flow, you can go beyond, with any of: - -- [Optimize model](#optimize-model) -- [Right-size staking](#right-size-staking) -- [Run local network](#run-local-network) -- [Run many bots](#run-many-bots). Steps: [give keys](#many-bots-give-keys), [templates](#many-bots-generate-templates), [deploy agents](#many-bots-deploy-agents), [monitor agents](#many-bots-monitor-agents), [destroy agents](#many-bots-destroy-agents) - -## 1. Install pdr-backend Repo - -Prerequisites: -- Python 3.12. Earlier _will_ fail, e.g. can't find `UTC`. [Details](https://blog.miguelgrinberg.com/post/it-s-time-for-a-change-datetime-utcnow-is-now-deprecated) -- Ubuntu MacOS. _Not_ Windows. - -In a new console: - -```console -# clone the repo and enter into it -git clone https://github.com/oceanprotocol/pdr-backend -cd pdr-backend - -# create & activate virtualenv -python -m venv venv -source venv/bin/activate - -# install modules in the environment -pip install -r requirements.txt - -# add pwd to bash path -export PATH=$PATH:. -``` - -You need a local copy of Ocean contract addresses [`address.json`](https://github.com/oceanprotocol/contracts/blob/main/addresses/address.json). In console: -```console -# make directory if needed -mkdir -p ~/.ocean; mkdir -p ~/.ocean/ocean-contracts; mkdir -p ~/.ocean/ocean-contracts/artifacts/ - -# copy from github to local directory. Or, use wget if Linux. Or, download via browser. -curl https://raw.githubusercontent.com/oceanprotocol/contracts/main/addresses/address.json -o ~/.ocean/ocean-contracts/artifacts/address.json -``` - -If you're running MacOS, then in console: - -```console -# so that sapphire.py works. Details in #66 -codesign --force --deep --sign - venv/sapphirepy_bin/sapphirewrapper-arm64.dylib - -# so that xgboost works. Details in #1339 -brew install libomp -``` - -## 2. Simulate Modeling and Trading - -Simulation allows us to quickly build intuition, and assess the performance of the data / predicting / trading strategy (backtest). - -Copy [`ppss.yaml`](../ppss.yaml) into your own file `my_ppss.yaml` and change parameters as you see fit. - -```console -cp ppss.yaml my_ppss.yaml -``` - -Let's run the simulation engine. In console: -```console -pdr sim my_ppss.yaml -``` - -What the engine does does: -1. Set simulation parameters. -1. Grab historical price data from exchanges and stores in `lake_data/` dir. It re-uses any previously saved data. -1. Run through many 5min epochs. At each epoch: - - Build a model - - Predict - - Trade - - Log to console and `logs/out_