'tdr' branch to reduce complexity, speed runtime. For #1712

.github/dependabot.yml

@@ -1,8 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
-
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:

.github/workflows/black.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: Black
 
 on:

.github/workflows/check_mainnet.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: Run Mainnet Check and Notify Slack
 
 on:

.github/workflows/check_testnet.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: Run Script and Notify Slack
 
 on:

.github/workflows/cron_topup.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: Topup accounts
 
 on:

.github/workflows/mypy.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: MyPy Static Typechecking
 
 on:

.github/workflows/pylint.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: Pylint Style Tests
 
 on:

.github/workflows/pytest.yml

@@ -1,7 +1,3 @@
-##
-## Copyright 2024 Ocean Protocol Foundation
-## SPDX-License-Identifier: Apache-2.0
-##
 name: pdr-backend tests
 
 on:
@@ -26,39 +22,17 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: actions/checkout@v2
-        name: Checkout Barge
-        with:
-          repository: "oceanprotocol/barge"
-          path: "barge"
-          ref: "5bd4a318c732d84a6b7fbeb193cee7fb16891a67"
-
-      - name: Run Barge
-        working-directory: ${{ github.workspace }}/barge
-        env:
-          SUBGRAPH_VERSION: main
-        run: |
-          bash -x start_ocean.sh --no-aquarius --no-elasticsearch --no-provider --no-dashboard --with-thegraph --predictoor > start_ocean.log &
-
       - name: Install dependencies
         working-directory: ${{ github.workspace }}
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install pytest
 
-      - name: Wait for contracts deployment
-        working-directory: ${{ github.workspace }}/barge
-        run: |
-          for i in $(seq 1 250); do
-            sleep 5
-            [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" ] && break
-            done
-
       - name: Test with pytest
         id: pytest
         run: |
-          coverage run --source=pdr_backend --omit=*/test/*,*/test_ganache/*,*/test_noganache/* -m pytest
+          coverage run --source=pdr_backend --omit=*/test/*,*/test_noganache/* -m pytest
           coverage report
           coverage xml
       - name: Publish code coverage

.gitignore

@@ -58,69 +58,9 @@ cover/
 *.mo
 *.pot
 
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
 
 # Environments
 .env
@@ -131,59 +71,30 @@ ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-.test_cache/
-.cache/
-dir/
-
 # predictoor-specific
 out*.txt
 my_ppss.yaml
+my_ppss_profiling.yaml
+my_ppss_makemoney.yaml
 my_logging.yaml
 logs/
 csvs/
 parquet_data/
 lake_data/
+xy_dir/
+xy_data/
 pdr_predictions.parquet 
 
-# sim state
-sim_state
-
 # pdr_backend accuracy output
 pdr_backend/accuracy/output/*.json
 
 # lakeinfo output
 validate_duplicate_rows.csv
 
-# agent deployer
-.keys.json
-.deployments/
+#profiling
+profile.stats

.pylintrc

@@ -31,7 +31,7 @@ ignore=
   Dockerfile,
   entrypoint.sh,
   pdr_backend.egg-info,
-  copyright_template.tmpl
+  profile.stats
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.

README.md

@@ -2,10 +2,7 @@
 
 ## Run bots (agents)
 
-- **[Run predictoor bot](READMEs/predictoor.md)** - make predictions, make $
-- **[Run trader bot](READMEs/trader.md)** - consume predictions, trade, make $
-
-(If you're a predictoor or trader, you can safely ignore the rest of this README.)
+- **[Run sim/trading](READMEs/sim.md)** - predict, trade, make $. Historical, live mock, or live real.
 
 ## Settings: PPSS
 
@@ -36,52 +33,29 @@ pdr
 This will output something like:
 
 ```text
-Usage: pdr sim|predictoor|trader|..
+Usage: pdr sim|trader|..
 
 Main tools:
   pdr sim YAML_FILE
-  pdr predictoor YAML_FILE NETWORK
-  pdr trader APPROACH YAML_FILE NETWORK
+  pdr trader APPROACH YAML_FILE livemock|livereal (FIXME)
 ...
 ```
 
-## Atomic READMEs
-
-- [Get tokens](READMEs/get-tokens.md): [testnet faucet](READMEs/testnet-faucet.md), [mainnet ROSE](READMEs/get-rose-on-sapphire.md) & [OCEAN](READMEs/get-ocean-on-sapphire.md)
-- [Claim payout for predictoor bot](READMEs/payout.md)
-- [Predictoor subgraph](READMEs/subgraph.md). [Subgraph filters](READMEs/filters.md)
-- [Run barge locally](READMEs/barge.md)
-
 ## Flows for core team
 
 - Backend-dev - for `pdr-backend` itself
-  - [Local dev flow](READMEs/dev.md)
+  - Local dev flow: see [sim](READMEs/sim.md)
   - [VPS dev flow](READMEs/vps.md)
-  - [Release process](READMEs/release-process.md)
-  - [Clean code guidelines](READMEs/clean-code.md)
   - [Dependency management](READMEs/dependencies.md)
-- [Run dfbuyer bot](READMEs/dfbuyer.md) - runs Predictoor DF rewards
-- [Run publisher](READMEs/publisher.md) - publish new feeds
-- [Run trueval](READMEs/trueval.md) - run trueval bot
-- [Run lake](READMEs/lake-and-etl.md) - run data lake
 
 ## Repo structure
 
 This repo implements all bots in Predictoor ecosystem. Here are each of the sub-directories in the repo.
 
 Main bots & user tools:
 
-- `predictoor` - submit individual predictions
 - `trader` - buy aggregated predictions, then trade
 - `sim` - experiments / simulation flow
-- `payout` - OCEAN & ROSE payout
-
-OPF-run bots & higher-level tools:
-
-- `trueval` - report true values to contract
-- `dfbuyer` - buy feeds on behalf of Predictoor DF
-- `publisher` - publish pdr data feeds
-- `deployer` - deployer tool
 
 Mid-level building blocks:
 
@@ -93,11 +67,7 @@ Data-level building blocks:
 - `ohlcv` - financial data pipeline
 - `aimodel` - AI/ML modeling engine
 - `lake` - data lake and analytics tools
-- `subgraph` - blockchain queries, complements lake
-- `accuracy` - to report % correct in webapp
-- `pred_submitter` - for predictoor bots to submit >>1 predictions in 1 tx
 
 Lower-level utilities:
 
-- `contract` - classes to wrap blockchain contracts; some simple data structures
 - `util` - function-based tools