From 637daefc7544438bc1597aba345998d8efc0bdcb Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Wed, 25 Feb 2026 17:56:04 +0000 Subject: [PATCH] Release 2026-02-25-17-56 --- CHANGELOG.md | 4 + README.md | 10 +- docker-compose.prd.yaml | 2 +- docker-compose.validator.yaml | 2 +- docs/architecture.md | 24 +-- docs/gateway-guide.md | 130 ++++++++++++- docs/miner-setup.md | 75 +++++++- docs/subnet-rules.md | 20 +- neurons/miner/agents/vericore_example.py | 163 ++++++++++++++++ neurons/miner/gateway/.env.example | 3 + neurons/miner/gateway/app.py | 24 +++ .../gateway/providers/tests/test_vericore.py | 121 ++++++++++++ neurons/miner/gateway/providers/vericore.py | 38 ++++ neurons/miner/scripts/gateway_lib/config.py | 59 +++--- neurons/miner/scripts/link_vericore.py | 181 ++++++++++++++++++ neurons/miner/scripts/services.py | 9 +- .../miner/scripts/test_agent_lib/execution.py | 2 +- .../miner/scripts/test_agent_lib/preflight.py | 12 +- neurons/validator/main.py | 2 +- neurons/validator/models/numinous_client.py | 10 + neurons/validator/models/vericore.py | 55 ++++++ neurons/validator/numinous_client/client.py | 25 +++ neurons/validator/utils/config.py | 4 +- neurons/validator/version.py | 2 +- 24 files changed, 902 insertions(+), 75 deletions(-) create mode 100644 neurons/miner/agents/vericore_example.py create mode 100644 neurons/miner/gateway/providers/tests/test_vericore.py create mode 100644 neurons/miner/gateway/providers/vericore.py create mode 100644 neurons/miner/scripts/link_vericore.py create mode 100644 neurons/validator/models/vericore.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0641d1..afbdf4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release Notes +## [2.1.1] - 2026-02-25 +- **Sandbox**: Increase agent sandbox execution timeout to 240 seconds +- **Integration**: Vericore API integration + ## [2.1.0] - 2026-02-09 - **Bittensor Upgrade**: Upgraded to Bittensor version 10.1.0 - **Bittensor CLI Upgrade**: Upgraded to Bittensor CLI version 9.18.0 diff --git a/README.md b/README.md index 3e78cc0..0aed1a1 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@
-# **Numinous** +# **Numinous** [Discord](https://discord.gg/qKPeYPc3) • [Dashboard](https://app.hex.tech/1644b22a-abe5-4113-9d5f-3ad05e4a8de7/app/Numinous-031erYRYSssIrH3W3KcyHg/latest) • [Website](https://numinouslabs.io/) • [Twitter](https://x.com/numinous_ai) • -[Network](https://taostats.io/subnets/6/chart) +[Network](https://taostats.io/subnets/6/chart) ---
## Introduction -Numinous (Subnet 6) is a **forecasting protocol** whose goal is to aggregate agents into **superhuman LLM forecasters**. The key principle is that instead of scoring predictions ($f(X)$) the subnet scores the underlying agentic models ($X$). +Numinous (Subnet 6) is a **forecasting protocol** whose goal is to aggregate agents into **superhuman LLM forecasters**. The key principle is that instead of scoring predictions ($f(X)$) the subnet scores the underlying agentic models ($X$). Miners send forecasting agents which are subsequently evaluated by validators in sandboxes with access to a curated set of tools and data. **Agent execution and code are entirely visible to the subnet protocol.** @@ -37,7 +37,7 @@ Validators spin up parallel sandboxes where miners are evaluated on batches of e ### Key Components * **The Sandbox:** Isolated execution environment with strict resource limits. - * **The Gateway:** A signing proxy allowing agents to access **Chutes (SN64)** for compute, **Desearch (SN22)** for live data, and **OpenAI** for GPT-5 models without exposing validator keys. + * **The Gateway:** A signing proxy allowing agents to access **Chutes (SN64)** for compute, **Desearch (SN22)** for live data, **OpenAI** for GPT-5 models, and **Vericore** for statement verification without exposing validator keys. * **Forecasting logic:** Agents execute once per event; only agent which were registered prior to broadcasting execute. 📖 **[Read the full system architecture](docs/architecture.md)** @@ -50,7 +50,7 @@ To survive in the Numinous arena, agents must adhere to strict constraints. Viol ### Execution Rules -1. **Timeout:** Execution must complete within **210 seconds**. +1. **Timeout:** Execution must complete within **240 seconds**. 2. **Cost:** API usage limits depend on each service and are paid by the miner. 3. **Caching:** Do not use dynamic timestamps or random seeds in prompts. This would break our caching system making agent executions differ between validators. 4. **Activation:** Code submitted before **00:00 UTC** activates the following day. You can update your code at most once every 3 days. diff --git a/docker-compose.prd.yaml b/docker-compose.prd.yaml index c787adb..c241f12 100644 --- a/docker-compose.prd.yaml +++ b/docker-compose.prd.yaml @@ -31,7 +31,7 @@ services: # Production configuration command: > - bash -c "python neurons/validator.py --netuid 6 --subtensor.network finney --wallet.name ifkey --wallet.hotkey ifhkey --db.directory /root/infinite_games/database --numinous.env prod --sandbox.max_concurrent 50 --sandbox.timeout_seconds 210 --validator.sync_hour 0 --logging.debug" + bash -c "python neurons/validator.py --netuid 6 --subtensor.network finney --wallet.name ifkey --wallet.hotkey ifhkey --db.directory /root/infinite_games/database --numinous.env prod --sandbox.max_concurrent 50 --sandbox.timeout_seconds 240 --validator.sync_hour 0 --logging.debug" logging: driver: "json-file" diff --git a/docker-compose.validator.yaml b/docker-compose.validator.yaml index b46fad2..facaefe 100644 --- a/docker-compose.validator.yaml +++ b/docker-compose.validator.yaml @@ -30,7 +30,7 @@ services: - HOST_WALLET_PATH=${HOST_WALLET_PATH:-${HOME}/.bittensor/wallets} command: > - bash -c "python neurons/validator.py --netuid 6 --subtensor.network finney --wallet.name ${WALLET_NAME} --wallet.hotkey ${WALLET_HOTKEY} --db.directory /root/infinite_games/database --numinous.env prod --sandbox.max_concurrent 50 --sandbox.timeout_seconds 210 --logging.debug" + bash -c "python neurons/validator.py --netuid 6 --subtensor.network finney --wallet.name ${WALLET_NAME} --wallet.hotkey ${WALLET_HOTKEY} --db.directory /root/infinite_games/database --numinous.env prod --sandbox.max_concurrent 50 --sandbox.timeout_seconds 240 --logging.debug" logging: driver: "json-file" diff --git a/docs/architecture.md b/docs/architecture.md index 76e93eb..d108d9f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,7 +59,7 @@ Validators continuously: - Fetch new prediction events - Download and execute miner agent code in sandboxes - Calculate an average Brier scores upon event resolutions -- Update subnet weights on the Bittensor chain +- Update subnet weights on the Bittensor chain **Process Flow:** ``` @@ -76,7 +76,7 @@ The validators spin up 50 parallel sandboxes where 50 miners are evaluated on th Agents run in isolated Docker containers with: - No internet access -- 210s execution timeout +- 240s execution timeout - Limited CPU/memory - Access to a defined set of external APIs via a signing proxy - Cost limits that depend on each service (paid by miner) @@ -161,25 +161,25 @@ For a binary event $E_q$, an agent $i$ sends a prediction $p_i$ for the probabil - $o_q = 0$ otherwise. The Brier score $S(p_i, o_q)$ for the prediction is given by: -- **If $o_q = 1$:** - +- **If $o_q = 1$:** + $$S(p_i, 1) = (1 - p_i)^2$$ - -- **If $o_q = 0$:** + +- **If $o_q = 0$:** $$S(p_i, 0) = p_i^2.$$ -The lower the score the better. This strictly proper scoring rule incentivizes miners to report their true beliefs. +The lower the score the better. This strictly proper scoring rule incentivizes miners to report their true beliefs. ## Scoring Process -1. A batch of binary events resolves -2. We calculate the Brier score for each miner's prediction +1. A batch of binary events resolves +2. We calculate the Brier score for each miner's prediction 3. We average the Brier scores across all the events in the batch 4. Winner-take-all: the miner with the lowest Brier score on one batch gets all the rewards -**Window based Scoring** All the events batches are 3 days batches and are generated daily. They contain approximately 100 events each. The score of a miner at any given time is a function of the latest event batch which resolved. The immunity period has a length of 7 days thus when a miner registers it is only scored once within the immunity period. +**Window based Scoring** All the events batches are 3 days batches and are generated daily. They contain approximately 100 events each. The score of a miner at any given time is a function of the latest event batch which resolved. The immunity period has a length of 7 days thus when a miner registers it is only scored once within the immunity period. -**Spot scoring** We only consider one prediction per miner. In the future as the network capacity improves we might move to a scoring which weights multiple predictions per miners. **Currently, only agents which were activated prior to a given event being broadcasted will forecast this event.** This means that on a given event all the miners which forecasted that event did so roughly at the same time. +**Spot scoring** We only consider one prediction per miner. In the future as the network capacity improves we might move to a scoring which weights multiple predictions per miners. **Currently, only agents which were activated prior to a given event being broadcasted will forecast this event.** This means that on a given event all the miners which forecasted that event did so roughly at the same time. --- @@ -210,7 +210,7 @@ def agent_main(event_data: dict) -> dict: ## Constraints - Max code size: 2MB -- Execution timeout: 210s +- Execution timeout: 240s - No direct internet access (must use gateway for external APIs) - Available libraries: see sandbox requirements diff --git a/docs/gateway-guide.md b/docs/gateway-guide.md index e0e76a9..27198d0 100644 --- a/docs/gateway-guide.md +++ b/docs/gateway-guide.md @@ -9,10 +9,11 @@ The Gateway API provides miner agents with access to external services during sa - **Desearch AI**: Web search, social media search, and content crawling - **OpenAI**: GPT-5 series models with built-in web search - **Perplexity**: Reasoning LLMs with built-in web search +- **Vericore**: Statement verification with evidence-based metrics All requests are cached to optimize performance and reduce costs. -**Cost Limits:** $0.01 (default) or $0.10 (linked account) per sandbox run for Chutes and Desearch. OpenAI: $1.00 per run (requires linked account, no free tier). Perplexity: $0.10 per run (requires linked account, no free tier). +**Cost Limits:** $0.01 (default) or $0.10 (linked account) per sandbox run for Chutes and Desearch. OpenAI: $1.00 per run (requires linked account, no free tier). Perplexity: $0.10 per run (requires linked account, no free tier). Vericore: $0.10 per run (requires linked account, no free tier). **Security:** API keys are securely stored using external secret management and never exposed to validators. @@ -1052,6 +1053,127 @@ print(f"Sources: {citations}") --- +## Vericore Endpoints + +Vericore provides statement verification with evidence-based metrics including sentiment, conviction, source credibility, and more. + +### POST /api/gateway/vericore/calculate-rating + +Verify a statement against web evidence and get detailed metrics. + +**URL:** `{SANDBOX_PROXY_URL}/api/gateway/vericore/calculate-rating` + +**Request Body:** +```json +{ + "run_id": "550e8400-e29b-41d4-a716-446655440000", + "statement": "Bitcoin will reach $100k by end of 2026", + "generate_preview": false +} +``` + +**Parameters:** + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `run_id` | string (UUID) | Yes | - | Execution tracking ID from environment | +| `statement` | string | Yes | - | Statement to verify against web evidence | +| `generate_preview` | boolean | No | false | Generate a preview URL for the results | + +**Response:** +```json +{ + "batch_id": "mlzjxglo15m23k", + "request_id": "req-mlzjxgmc4amr6", + "preview_url": "", + "evidence_summary": { + "total_count": 12, + "neutral": 37.5, + "entailment": 1.03, + "contradiction": 61.46, + "sentiment": -0.07, + "conviction": 0.82, + "source_credibility": 0.93, + "narrative_momentum": 0.48, + "risk_reward_sentiment": -0.15, + "political_leaning": 0.0, + "catalyst_detection": 0.12, + "statements": [ + { + "statement": "Evidence text from source...", + "url": "https://example.com/article", + "contradiction": 0.87, + "neutral": 0.12, + "entailment": 0.01, + "sentiment": -0.5, + "conviction": 0.75, + "source_credibility": 0.85, + "narrative_momentum": 0.5, + "risk_reward_sentiment": -0.5, + "political_leaning": 0.0, + "catalyst_detection": 0.3 + } + ] + }, + "cost": 0.05 +} +``` + +**Response Fields:** + +| Field | Type | Description | +|-------|------|-------------| +| `batch_id` | string | Batch identifier | +| `request_id` | string | Request identifier | +| `preview_url` | string | Preview URL (empty if `generate_preview` is false) | +| `evidence_summary.total_count` | integer | Number of evidence sources found | +| `evidence_summary.entailment` | float | Aggregated entailment score | +| `evidence_summary.contradiction` | float | Aggregated contradiction score | +| `evidence_summary.sentiment` | float | Aggregated sentiment (-1.0 to 1.0) | +| `evidence_summary.conviction` | float | Aggregated conviction level | +| `evidence_summary.source_credibility` | float | Average source credibility | +| `evidence_summary.statements` | array | Individual evidence sources with per-source metrics | + +**Example (using httpx):** +```python +import os +import httpx + +PROXY_URL = os.getenv("SANDBOX_PROXY_URL") +RUN_ID = os.getenv("RUN_ID") + +response = httpx.post( + f"{PROXY_URL}/api/gateway/vericore/calculate-rating", + json={ + "run_id": RUN_ID, + "statement": "Bitcoin will reach $100k by end of 2026", + }, + timeout=120.0, +) + +result = response.json() + +summary = result["evidence_summary"] +total = summary["total_count"] +contradiction = summary["contradiction"] +sentiment = summary["sentiment"] +conviction = summary["conviction"] +credibility = summary["source_credibility"] +``` + +**Error Handling:** + +| Status Code | Description | Recommended Action | +|-------------|-------------|-------------------| +| 503 | Service Unavailable | Retry with exponential backoff | +| 429 | Rate limit exceeded | Retry with exponential backoff | +| 401 | Authentication failed | Contact validator | +| 500 | Internal server error | Retry with fallback | + +**Note:** Vericore has no free tier. You must link your API key to use Vericore. Each call costs $0.05. + +--- + ## Caching The gateway implements request-level caching to increase consensus stabilit among validators, optimize performance, reduce API costs. @@ -1066,7 +1188,7 @@ The gateway implements request-level caching to increase consensus stabilit amon - The `run_id` field is excluded from cache key calculation - This means identical requests from different executions hit the same cache -This is crucial to increase the consensus stability per validator given the variance of LLMs when hit twice with the same prompt. +This is crucial to increase the consensus stability per validator given the variance of LLMs when hit twice with the same prompt. **Prompt rules**. Use consistent prompts across executions to ensure that the cache is hit. In practice, **DO NOT** include dynamic timestamps or random data in prompts. @@ -1165,14 +1287,14 @@ def query_llm_with_retry(prompt: str, max_retries: int = 3) -> Optional[str]: ### Timeout Management -Plan your execution time to stay within the 210-second sandbox limit: +Plan your execution time to stay within the 240-second sandbox limit: ```python import time start_time = time.time() timeout_buffer = 10 # seconds -max_time = 200 # 210s limit - 10s buffer +max_time = 230 # 240s limit - 10s buffer def time_remaining(): elapsed = time.time() - start_time diff --git a/docs/miner-setup.md b/docs/miner-setup.md index ed7b21a..fa9eb63 100644 --- a/docs/miner-setup.md +++ b/docs/miner-setup.md @@ -14,12 +14,12 @@ For system architecture details, see [architecture.md](./architecture.md). For gateway API reference (Chutes AI, Desearch AI), see [gateway-guide.md](./gateway-guide.md). The key rules to follow as a miner are the following: -- **The sandbox times out after 210s** +- **The sandbox times out after 240s** - **The total cost limit on API calls depends on each service and its paid by the miner** - **DO NOT include dynamic timestamps or random data in prompts to make sure our caching system is hit across different validator executions**. - **A forecasting agent can only be updated at most once every 3 days** -All events are currently 3 days events. The length of the immunity period is 7 days to ensure any time before registration. +All events are currently 3 days events. The length of the immunity period is 7 days to ensure any time before registration. --- @@ -33,12 +33,14 @@ All events are currently 3 days events. The length of the immunity period is 7 d - **Desearch AI API key** (for local testing with web/Twitter search) - **OpenAI API key** (for local testing with GPT-5 models) - **Perplexity API key** (for local testing with reasoning LLMs) +- **Vericore API key** (for local testing with statement verification) **Get API Keys:** - Chutes AI: https://chutes.ai/app - Desearch AI: https://desearch.ai/ - OpenAI: https://platform.openai.com/api-keys - Perplexity: https://www.perplexity.ai/settings/api +- Vericore: https://vericore.ai **⚠️ OpenAI Security Recommendation:** @@ -393,6 +395,57 @@ def parse_prediction(text: str) -> float: return 0.5 ``` +### Using Vericore (Statement Verification) + +```python +import os +import httpx +from typing import Dict, Any + +PROXY_URL = os.getenv("SANDBOX_PROXY_URL", "http://sandbox_proxy") +RUN_ID = os.getenv("RUN_ID") + +if not RUN_ID: + raise ValueError("RUN_ID environment variable is required but not set") + +VERICORE_URL = f"{PROXY_URL}/api/gateway/vericore" + +def agent_main(event_data: Dict[str, Any]) -> Dict[str, Any]: + """Uses Vericore to verify event statement against web evidence.""" + + statement = f"{event_data['title']}. {event_data['description']}" + + response = httpx.post( + f"{VERICORE_URL}/calculate-rating", + json={ + "statement": statement, + "run_id": RUN_ID, + }, + timeout=120.0, + ) + + result = response.json() + summary = result["evidence_summary"] + + # Use evidence metrics to derive prediction + entailment = summary.get("entailment", 0.0) + contradiction = summary.get("contradiction", 0.0) + neutral = summary.get("neutral", 0.0) + + total = entailment + contradiction + neutral + if total > 0: + prediction = entailment / total + else: + prediction = 0.5 + + prediction = max(0.0, min(1.0, prediction)) + + return { + "event_id": event_data["event_id"], + "prediction": prediction + } +``` + ## Important Notes 1. **Always use `SANDBOX_PROXY_URL`** - Never hardcode API URLs @@ -461,7 +514,7 @@ Analyze this event: {event_data['description']}""" ### Timeout Management -**Leave buffer time for retries** - With a 210-second timeout, plan your execution: +**Leave buffer time for retries** - With a 240-second timeout, plan your execution: - Multiple retries: Account for exponential backoff delays - Fallback logic: Always have a quick fallback (return 0.5) if time runs out @@ -472,7 +525,7 @@ Analyze this event: {event_data['description']}""" import time start_time = time.time() -timeout = 200 # Leave 10s buffer before hard 210s limit +timeout = 230 # Leave 10s buffer before hard 240s limit def check_time_remaining(): elapsed = time.time() - start_time @@ -657,6 +710,19 @@ You'll be prompted for: **Note:** Perplexity has no free tier. You must link your account to use Perplexity models. +### Vericore (Statement Verification) + +Link your Vericore account for evidence-based statement verification: + +```bash +numi services link vericore +``` + +You'll be prompted for: +- Your Vericore API key (get from https://vericore.ai) + +**Note:** Vericore has no free tier. You must link your account to use Vericore. Each call costs $0.05. + **Important:** Re-link after each agent upload - each code version needs its own link. Check your linked services anytime: @@ -683,6 +749,7 @@ numi services link chutes # Link Chutes API key numi services link desearch # Link Desearch API key numi services link openai # Link OpenAI API key numi services link perplexity # Link Perplexity API key +numi services link vericore # Link Vericore API key numi services list # Check linked services # Local Testing diff --git a/docs/subnet-rules.md b/docs/subnet-rules.md index 838e085..a1cd09f 100644 --- a/docs/subnet-rules.md +++ b/docs/subnet-rules.md @@ -7,7 +7,7 @@ This document defines the operational rules, constraints, and scoring mechanisms for the Numinous forecasting subnet. All miners must understand and follow these rules to participate successfully. The key rules are the following (they will be repeated in context below): -- **The sandbox times out after 210s** +- **The sandbox times out after 240s** - **The total cost limit on API calls depends on each service and is paid by the miner** - **DO NOT include dynamic timestamps or random data in prompts to make sure our caching system is hit across different validator executions**. @@ -71,14 +71,14 @@ Interval 1 (Day 2): Prediction reused → 0.65 Interval N (Day N): Prediction reused → 0.65 ``` -We do this currently for **efficient ressource usage**. This will change in the future since there is clear benefit in having multiple forecasting schedules. +We do this currently for **efficient ressource usage**. This will change in the future since there is clear benefit in having multiple forecasting schedules. ## Resource Limits | Resource | Limit | Consequence if Exceeded | |----------|-------|-------------------------| -| **Execution Timeout** | 210 seconds | Hard kill, no prediction recorded | +| **Execution Timeout** | 240 seconds | Hard kill, no prediction recorded | | **Code Size** | 2MB | Upload rejected | | **Cost Limit** | Depends on service (see linking) | Run exited | | **Python Version** | 3.11+ | - | @@ -86,7 +86,7 @@ We do this currently for **efficient ressource usage**. This will change in the | **Libraries** | Only in `sandbox/requirements.txt` | Import errors | **Timeout Handling:** -- Agent killed after 210 seconds +- Agent killed after 240 seconds - No prediction recorded = missing prediction - Imputed prediction = 0.5 - Test locally to avoid this! @@ -218,7 +218,7 @@ Visit https://chutes.ai/app to see which models are "hot" (have active instances | Failure Type | Penalty | How to Avoid | |--------------|---------|--------------| -| **Timeout (>210s)** | No prediction, imputed 0.5 → Brier score = 0.25 | Optimize code, test locally, add timeouts to API calls | +| **Timeout (>240s)** | No prediction, imputed 0.5 → Brier score = 0.25 | Optimize code, test locally, add timeouts to API calls | | **Python Error** | No prediction, imputed 0.5 → Brier score = 0.25 | Test with `numi test-agent`, add error handling | | **Invalid Output** | No prediction, imputed 0.5 → Brier score = 0.25 | Validate return format: `{"event_id": str, "prediction": float}` | | **Out of Range** | Clipped to [0.01, 0.99] | Ensure prediction in [0.0, 1.0] before returning | @@ -227,7 +227,7 @@ Visit https://chutes.ai/app to see which models are "hot" (have active instances | **429 from Chutes** | Rate limit exceeded | Implement exponential backoff, reduce API calls | ## Missing prediction -The miner is imputed a prediction of 0.5 in all cases of missing prediction: +The miner is imputed a prediction of 0.5 in all cases of missing prediction: - for intervals before registration when registering a miner - if the code does not return a prediction or fails @@ -246,7 +246,7 @@ The miner is imputed a prediction of 0.5 in all cases of missing prediction: - Bittensor coldkey + hotkey pair - Registration on subnet (netuid 6 mainnet, 155 testnet) - TAO for registration (cost fluctuates based on demand) -- Immunity period after registration +- Immunity period after registration ## Hotkey Verification @@ -264,7 +264,7 @@ Your submitted code is verified against your registered wallet before execution. Yes you'd want to register the closest possible to midnight which is the activation date. **Q: Does my agent re-execute for every 24hs interval?** -A: No. At the moment your agent executes **once per event**. The same prediction is automatically reused for all intervals until cutoff. +A: No. At the moment your agent executes **once per event**. The same prediction is automatically reused for all intervals until cutoff. **Q: Can I update a prediction for an event?** A: No. Once submitted, predictions are final. To change strategy, submit new agent code (active next day for new events only). @@ -276,7 +276,7 @@ A: Approximately 100 events per day across all event types. A: At the next **00:00 UTC** after submission. **Q: What happens if my agent times out?** -A: Execution is killed after 210 seconds. No prediction is recorded. You get imputed prediction of 0.5, resulting in Brier score of 0.25. +A: Execution is killed after 240 seconds. No prediction is recorded. You get imputed prediction of 0.5, resulting in Brier score of 0.25. **Q: What if I get a 503 error from Chutes?** A: You requested a cold model. Use hot models (check https://chutes.ai/app) and implement exponential backoff retry logic with a fallback. @@ -291,7 +291,7 @@ A: No, you can submit once every three days, so please ensure you really test it Before submitting your agent, ensure: - ✅ Code implements `agent_main(event_data) -> {"event_id": str, "prediction": float}` -- ✅ Execution time < 210 seconds (tested locally) +- ✅ Execution time < 240 seconds (tested locally) - ✅ Code size < 2MB - ✅ Uses only libraries in `sandbox/requirements.txt` - ✅ Returns predictions in [0.0, 1.0] range diff --git a/neurons/miner/agents/vericore_example.py b/neurons/miner/agents/vericore_example.py new file mode 100644 index 0000000..695c81d --- /dev/null +++ b/neurons/miner/agents/vericore_example.py @@ -0,0 +1,163 @@ +import asyncio +import os +import time +from datetime import datetime + +import httpx +from pydantic import BaseModel + +RUN_ID = os.getenv("RUN_ID") +if not RUN_ID: + raise ValueError("RUN_ID environment variable is required but not set") + +PROXY_URL = os.getenv("SANDBOX_PROXY_URL", "http://sandbox_proxy") +VERICORE_URL = f"{PROXY_URL}/api/gateway/vericore" + +MAX_RETRIES = 3 +BASE_BACKOFF = 1.5 + +TOTAL_COST = 0.0 + + +class AgentData(BaseModel): + event_id: str + title: str + description: str + cutoff: datetime + metadata: dict + + +async def retry_with_backoff(func, max_retries: int = MAX_RETRIES): + for attempt in range(max_retries): + try: + return await func() + except httpx.TimeoutException as e: + if attempt < max_retries - 1: + delay = BASE_BACKOFF ** (attempt + 1) + print(f"[RETRY] Timeout, retrying in {delay}s...") + await asyncio.sleep(delay) + else: + raise Exception(f"Max retries exceeded: {e}") + except httpx.HTTPStatusError as e: + try: + error_detail = e.response.json().get("detail", str(e)) + except Exception: + error_detail = e.response.text if hasattr(e.response, "text") else str(e) + + if e.response.status_code == 429: + if attempt < max_retries - 1: + delay = BASE_BACKOFF ** (attempt + 1) + print(f"[RETRY] Rate limited (429), retrying in {delay}s...") + await asyncio.sleep(delay) + else: + raise Exception(f"Rate limit exceeded: {error_detail}") + else: + raise Exception(f"HTTP {e.response.status_code}: {error_detail}") + except Exception: + raise + + +def clip_probability(prediction: float) -> float: + return max(0.0, min(1.0, prediction)) + + +def derive_prediction(evidence_summary: dict) -> float: + total = evidence_summary.get("total_count", 0) + if total == 0: + return 0.5 + + entailment = evidence_summary.get("entailment", 0.0) + contradiction = evidence_summary.get("contradiction", 0.0) + + # Use entailment vs contradiction as the primary signal + directional_total = entailment + contradiction + if directional_total <= 0: + # No directional evidence -- use sentiment as a weak signal + sentiment = evidence_summary.get("sentiment", 0.0) + return clip_probability(0.5 + sentiment * 0.15) + + support_ratio = entailment / directional_total + + # Map support_ratio [0,1] to prediction range [0.1, 0.9] to avoid extremes + prediction = 0.1 + support_ratio * 0.8 + + # Nudge by sentiment (small adjustment, capped at +/-0.1) + sentiment = evidence_summary.get("sentiment", 0.0) + prediction += sentiment * 0.1 + + return clip_probability(prediction) + + +async def verify_statement(event: AgentData) -> dict: + global TOTAL_COST + print("[VERICORE] Verifying event statement...") + + statement = f"{event.title}. {event.description}" + + async def vericore_call(): + async with httpx.AsyncClient(timeout=120.0) as client: + payload = { + "statement": statement, + "generate_preview": False, + "run_id": RUN_ID, + } + response = await client.post( + f"{VERICORE_URL}/calculate-rating", + json=payload, + ) + response.raise_for_status() + return response.json() + + result = await retry_with_backoff(vericore_call) + + cost = result.get("cost", 0.01) + TOTAL_COST += cost + + summary = result["evidence_summary"] + total = summary["total_count"] + + print(f"[VERICORE] Found {total} evidence sources") + print(f"[VERICORE] Entailment: {summary.get('entailment', 'N/A')}") + print(f"[VERICORE] Contradiction: {summary.get('contradiction', 'N/A')}") + print(f"[VERICORE] Sentiment: {summary.get('sentiment', 'N/A')}") + print(f"[VERICORE] Conviction: {summary.get('conviction', 'N/A')}") + print(f"[VERICORE] Source credibility: {summary.get('source_credibility', 'N/A')}") + print(f"[VERICORE] Cost: ${cost:.4f} | Total: ${TOTAL_COST:.4f}") + + prediction = derive_prediction(summary) + print(f"[VERICORE] Derived prediction: {prediction:.4f}") + + return { + "event_id": event.event_id, + "prediction": prediction, + } + + +async def run_agent(event: AgentData) -> dict: + global TOTAL_COST + TOTAL_COST = 0.0 + + start_time = time.time() + + try: + result = await verify_statement(event) + except Exception as e: + print(f"[AGENT] Error: {e}") + result = { + "event_id": event.event_id, + "prediction": 0.5, + } + + elapsed = time.time() - start_time + print(f"[AGENT] Complete in {elapsed:.2f}s") + print(f"[AGENT] Total run cost: ${TOTAL_COST:.4f}") + + return result + + +def agent_main(event_data: dict) -> dict: + event = AgentData.model_validate(event_data) + print(f"\n[AGENT] Running Vericore verification for event: {event.event_id}") + print(f"[AGENT] Title: {event.title}") + + return asyncio.run(run_agent(event)) diff --git a/neurons/miner/gateway/.env.example b/neurons/miner/gateway/.env.example index 2958425..1d07e04 100644 --- a/neurons/miner/gateway/.env.example +++ b/neurons/miner/gateway/.env.example @@ -14,3 +14,6 @@ OPENAI_API_KEY=your-openai-api-key-here # Perplexity API Key (Get your key from: https://www.perplexity.ai/settings/api) PERPLEXITY_API_KEY=your-perplexity-api-key-here + +# Vericore API Key (Get your key from: https://vericore.ai) +VERICORE_API_KEY=your-vericore-api-key-here diff --git a/neurons/miner/gateway/app.py b/neurons/miner/gateway/app.py index dd95b17..42fa9ca 100644 --- a/neurons/miner/gateway/app.py +++ b/neurons/miner/gateway/app.py @@ -11,6 +11,7 @@ from neurons.miner.gateway.providers.desearch import DesearchClient from neurons.miner.gateway.providers.openai import OpenAIClient from neurons.miner.gateway.providers.perplexity import PerplexityClient +from neurons.miner.gateway.providers.vericore import VericoreClient from neurons.validator.models import numinous_client as models from neurons.validator.models.chutes import ChuteStatus from neurons.validator.models.chutes import calculate_cost as calculate_chutes_cost @@ -18,6 +19,7 @@ from neurons.validator.models.desearch import calculate_cost as calculate_desearch_cost from neurons.validator.models.openai import calculate_cost as calculate_openai_cost from neurons.validator.models.perplexity import calculate_cost as calculate_perplexity_cost +from neurons.validator.models.vericore import calculate_cost as calculate_vericore_cost logger = logging.getLogger(__name__) @@ -298,6 +300,28 @@ async def perplexity_chat_completion(request: models.PerplexityInferenceRequest) ) +@gateway_router.post("/vericore/calculate-rating", response_model=models.GatewayVericoreResponse) +@cached_gateway_call +@handle_provider_errors("Vericore") +async def vericore_calculate_rating( + request: models.VericoreCalculateRatingRequest, +) -> models.GatewayVericoreResponse: + api_key = os.getenv("VERICORE_API_KEY") + if not api_key: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="VERICORE_API_KEY not configured", + ) + + client = VericoreClient(api_key=api_key) + result = await client.calculate_rating( + statement=request.statement, + generate_preview=request.generate_preview, + ) + + return models.GatewayVericoreResponse(**result.model_dump(), cost=calculate_vericore_cost()) + + app.include_router(gateway_router) diff --git a/neurons/miner/gateway/providers/tests/test_vericore.py b/neurons/miner/gateway/providers/tests/test_vericore.py new file mode 100644 index 0000000..2c69f82 --- /dev/null +++ b/neurons/miner/gateway/providers/tests/test_vericore.py @@ -0,0 +1,121 @@ +import json + +import pytest +from aiohttp import ClientResponseError +from aioresponses import aioresponses + +from neurons.miner.gateway.providers.vericore import VericoreClient +from neurons.validator.models.vericore import VericoreResponse + +MOCK_VERICORE_RESPONSE = { + "batch_id": "batch-abc-123", + "request_id": "req-xyz-456", + "preview_url": "", + "evidence_summary": { + "total_count": 3, + "neutral": 37.5, + "entailment": 1.03, + "contradiction": 61.46, + "sentiment": -0.07, + "conviction": 0.82, + "source_credibility": 0.93, + "narrative_momentum": 0.48, + "risk_reward_sentiment": -0.15, + "political_leaning": 0.0, + "catalyst_detection": 0.12, + "statements": [ + { + "statement": "Evidence supports the claim based on recent data.", + "url": "https://example.com/article1", + "contradiction": 0.87, + "neutral": 0.12, + "entailment": 0.01, + "sentiment": -0.5, + "conviction": 0.75, + "source_credibility": 0.85, + "narrative_momentum": 0.5, + "risk_reward_sentiment": -0.5, + "political_leaning": 0.0, + "catalyst_detection": 0.3, + }, + ], + }, +} + + +class TestVericoreClient: + @pytest.fixture + def client(self): + return VericoreClient(api_key="test_api_key") + + async def test_calculate_rating_success(self, client: VericoreClient): + with aioresponses() as mocked: + mocked.post( + "https://api.verify.vericore.ai/calculate-rating/v2", + status=200, + body=json.dumps(MOCK_VERICORE_RESPONSE).encode("utf-8"), + ) + + result = await client.calculate_rating( + statement="Bitcoin will reach $100k by end of 2026" + ) + + assert isinstance(result, VericoreResponse) + assert result.batch_id == "batch-abc-123" + assert result.request_id == "req-xyz-456" + assert result.evidence_summary.total_count == 3 + assert result.evidence_summary.support is None + assert result.evidence_summary.refute is None + assert result.evidence_summary.contradiction == 61.46 + assert result.evidence_summary.sentiment == -0.07 + assert len(result.evidence_summary.statements) == 1 + assert result.evidence_summary.statements[0].contradiction == 0.87 + assert result.evidence_summary.statements[0].sentiment == -0.5 + + async def test_calculate_rating_with_preview(self, client: VericoreClient): + with aioresponses() as mocked: + mocked.post( + "https://api.verify.vericore.ai/calculate-rating/v2", + status=200, + body=json.dumps(MOCK_VERICORE_RESPONSE).encode("utf-8"), + ) + + result = await client.calculate_rating( + statement="Test statement", generate_preview=True + ) + + assert isinstance(result, VericoreResponse) + assert result.batch_id == "batch-abc-123" + + async def test_calculate_rating_server_error(self, client: VericoreClient): + with aioresponses() as mocked: + mocked.post( + "https://api.verify.vericore.ai/calculate-rating/v2", + status=500, + body=b"Internal server error", + ) + + with pytest.raises(ClientResponseError) as exc: + await client.calculate_rating(statement="Test statement") + + assert exc.value.status == 500 + + async def test_calculate_rating_authentication_error(self, client: VericoreClient): + with aioresponses() as mocked: + mocked.post( + "https://api.verify.vericore.ai/calculate-rating/v2", + status=401, + body=b"Unauthorized", + ) + + with pytest.raises(ClientResponseError) as exc: + await client.calculate_rating(statement="Test statement") + + assert exc.value.status == 401 + + def test_client_initialization_invalid_api_key(self): + with pytest.raises(ValueError, match="Vericore API key is not set"): + VericoreClient(api_key="") + + with pytest.raises(ValueError, match="Vericore API key is not set"): + VericoreClient(api_key=None) diff --git a/neurons/miner/gateway/providers/vericore.py b/neurons/miner/gateway/providers/vericore.py new file mode 100644 index 0000000..d7a7c29 --- /dev/null +++ b/neurons/miner/gateway/providers/vericore.py @@ -0,0 +1,38 @@ +import aiohttp + +from neurons.validator.models.vericore import VericoreResponse + + +class VericoreClient: + __api_key: str + __base_url: str + __timeout: aiohttp.ClientTimeout + __headers: dict[str, str] + + def __init__(self, api_key: str): + if not api_key: + raise ValueError("Vericore API key is not set") + + self.__api_key = api_key + self.__base_url = "https://api.verify.vericore.ai" + self.__timeout = aiohttp.ClientTimeout(total=120) + self.__headers = { + "Authorization": f"api-key {self.__api_key}", + "Content-Type": "application/json", + } + + async def calculate_rating( + self, statement: str, generate_preview: bool = False + ) -> VericoreResponse: + body = { + "statement": statement, + "generate_preview": str(generate_preview).lower(), + } + + url = f"{self.__base_url}/calculate-rating/v2" + + async with aiohttp.ClientSession(timeout=self.__timeout, headers=self.__headers) as session: + async with session.post(url, json=body) as response: + response.raise_for_status() + data = await response.json() + return VericoreResponse.model_validate(data) diff --git a/neurons/miner/scripts/gateway_lib/config.py b/neurons/miner/scripts/gateway_lib/config.py index c310d43..b89e087 100644 --- a/neurons/miner/scripts/gateway_lib/config.py +++ b/neurons/miner/scripts/gateway_lib/config.py @@ -7,41 +7,51 @@ GATEWAY_ENV_PATH = Path("neurons/miner/gateway/.env") +API_KEYS = [ + ("CHUTES_API_KEY", "Chutes", "https://chutes.ai"), + ("DESEARCH_API_KEY", "Desearch", "https://desearch.ai"), + ("OPENAI_API_KEY", "OpenAI", "https://platform.openai.com/api-keys"), + ("PERPLEXITY_API_KEY", "Perplexity", "https://www.perplexity.ai/settings/api"), + ("VERICORE_API_KEY", "Vericore", "https://vericore.ai"), +] + + +def _is_key_set(env_content: str, key: str) -> bool: + if f"{key}=" not in env_content: + return False + value = env_content.split(f"{key}=")[1].split("\n")[0].strip() + return value != "" + def check_env_vars() -> dict[str, bool]: if not GATEWAY_ENV_PATH.exists(): - return {"CHUTES_API_KEY": False, "DESEARCH_API_KEY": False} + return {key: False for key, _, _ in API_KEYS} env_content = GATEWAY_ENV_PATH.read_text() - return { - "CHUTES_API_KEY": "CHUTES_API_KEY=" in env_content - and not env_content.split("CHUTES_API_KEY=")[1].split("\n")[0].strip() == "", - "DESEARCH_API_KEY": "DESEARCH_API_KEY=" in env_content - and not env_content.split("DESEARCH_API_KEY=")[1].split("\n")[0].strip() == "", - } + return {key: _is_key_set(env_content, key) for key, _, _ in API_KEYS} def setup_api_keys(force_all: bool = False) -> bool: console.print() - console.print("[cyan]🔑 API Key Setup[/cyan]") + console.print("[cyan]API Key Setup[/cyan]") console.print() console.print("[dim]You can get your API keys from:[/dim]") - console.print("[dim] • Chutes: [link=https://chutes.ai]https://chutes.ai[/link][/dim]") - console.print("[dim] • Desearch: [link=https://desearch.ai]https://desearch.ai[/link][/dim]") + for _, name, url in API_KEYS: + console.print(f"[dim] - {name}: [link={url}]{url}[/link][/dim]") console.print() env_status = check_env_vars() - chutes_key = None - desearch_key = None - - if not env_status["CHUTES_API_KEY"] or force_all: - chutes_key = Prompt.ask("[cyan]Chutes API Key[/cyan]") - chutes_key = chutes_key.strip() if chutes_key else None + keys_to_set: dict[str, str] = {} - if not env_status["DESEARCH_API_KEY"] or force_all: - desearch_key = Prompt.ask("[cyan]Desearch API Key[/cyan]") - desearch_key = desearch_key.strip() if desearch_key else None + for env_var, name, _ in API_KEYS: + if not env_status[env_var] or force_all: + value = Prompt.ask( + f"[cyan]{name} API Key[/cyan] [dim](Enter to skip)[/dim]", default="" + ) + value = value.strip() if value else "" + if value: + keys_to_set[env_var] = value existing_content = "" if GATEWAY_ENV_PATH.exists(): @@ -60,11 +70,8 @@ def update_or_add_key(lines: list[str], key: str, value: str) -> list[str]: lines.append(f"{key}={value}") return lines - if chutes_key: - lines = update_or_add_key(lines, "CHUTES_API_KEY", chutes_key) - - if desearch_key: - lines = update_or_add_key(lines, "DESEARCH_API_KEY", desearch_key) + for env_var, value in keys_to_set.items(): + lines = update_or_add_key(lines, env_var, value) try: GATEWAY_ENV_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -76,13 +83,13 @@ def update_or_add_key(lines: list[str], key: str, value: str) -> list[str]: GATEWAY_ENV_PATH.write_text(new_content) console.print() - console.print(f"[green]✓[/green] API keys saved to [cyan]{GATEWAY_ENV_PATH}[/cyan]") + console.print(f"[green]OK[/green] API keys saved to [cyan]{GATEWAY_ENV_PATH}[/cyan]") console.print() return True except Exception as e: console.print() - console.print(f"[red]✗[/red] Failed to save API keys: {e}") + console.print(f"[red]Failed to save API keys: {e}[/red]") console.print() return False diff --git a/neurons/miner/scripts/link_vericore.py b/neurons/miner/scripts/link_vericore.py new file mode 100644 index 0000000..837369b --- /dev/null +++ b/neurons/miner/scripts/link_vericore.py @@ -0,0 +1,181 @@ +import base64 +import time +import typing +from getpass import getpass +from pathlib import Path + +import click +import httpx +from rich.console import Console +from rich.panel import Panel +from rich.prompt import Confirm, Prompt + +from neurons.miner.scripts.numinous_config import ENV_URLS +from neurons.miner.scripts.wallet_utils import load_keypair, prompt_wallet_selection + +console = Console() + + +def link_vericore_impl( + wallet: typing.Optional[str] = None, + hotkey: typing.Optional[str] = None, + env: typing.Optional[str] = None, + wallet_path: typing.Optional[Path] = None, +) -> None: + console.print() + console.print( + Panel.fit( + "[bold cyan]Link Vericore API Key[/bold cyan]", + border_style="cyan", + padding=(1, 2), + ) + ) + console.print() + + if not env: + env_choice = Prompt.ask( + "[bold cyan]Select environment[/bold cyan]", choices=["test", "prod"], default="test" + ) + env = env_choice.lower() + + console.print(f"[dim]Network:[/dim] [yellow]{env.upper()}[/yellow]") + console.print() + + if not wallet or not hotkey: + wallet, hotkey = prompt_wallet_selection(wallet_path) + + console.print() + with console.status(f"[cyan]Loading hotkey {wallet}/{hotkey}...[/cyan]"): + hotkey_keypair = load_keypair(wallet, hotkey, wallet_path) + + if not hotkey_keypair: + console.print() + console.print( + Panel.fit( + f"[red]Failed to load hotkey:[/red] {wallet}/{hotkey}", + border_style="red", + ) + ) + console.print() + raise click.Abort() + + console.print( + f"[green]OK[/green] Loaded hotkey: [yellow]{hotkey_keypair.ss58_address}[/yellow]" + ) + + console.print() + console.print( + Panel.fit( + "[bold cyan]Vericore API Key Setup[/bold cyan]\n\n" + "[dim]Get your API key from:[/dim] [cyan]https://vericore.ai[/cyan]\n\n" + "[yellow]Budget:[/yellow]\n" + " - Miner-paid: [green]$0.05 per call[/green]\n" + " - No free tier available\n\n" + "[dim]Your API key will be securely stored in AWS Secrets Manager.[/dim]", + border_style="cyan", + ) + ) + console.print() + + api_key = getpass("Enter your Vericore API key: ") + if not api_key or not api_key.strip(): + console.print("[red]API key cannot be empty[/red]") + raise click.Abort() + + api_key = api_key.strip() + + console.print() + console.print("[bold cyan]Ready to link:[/bold cyan]") + console.print(f" [dim]Hotkey:[/dim] {hotkey_keypair.ss58_address[:16]}...") + console.print(" [dim]Service:[/dim] Vericore") + console.print(" [dim]Budget:[/dim] Miner-paid ($0.05/call)") + console.print(f" [dim]Network:[/dim] {env.upper()}") + console.print() + + if not Confirm.ask("[yellow]Proceed with linking?[/yellow]", default=True): + console.print("[dim]Cancelled[/dim]") + raise click.Abort() + + console.print() + with console.status("[cyan]Storing credentials in backend...[/cyan]"): + success, error_msg = _store_vericore_credentials(env, hotkey_keypair, api_key) + + if not success: + console.print() + console.print( + Panel.fit( + f"[red]Failed to link Vericore API key[/red]\n\n" + f"[yellow]Error:[/yellow] {error_msg}\n\n" + "[dim]Please check:[/dim]\n" + " - API key is valid\n" + " - Wallet has been registered as miner\n" + " - Network connection is stable", + border_style="red", + ) + ) + console.print() + raise click.Abort() + + console.print() + console.print( + Panel.fit( + "[bold green]Successfully linked Vericore API key![/bold green]\n\n" + f"[dim]Hotkey:[/dim] {hotkey_keypair.ss58_address[:16]}...\n" + f"[dim]Service:[/dim] Vericore\n" + f"[dim]Budget:[/dim] Miner-paid ($0.05/call)\n\n" + "[yellow]Your agent runs will now use your Vericore API key[/yellow]", + border_style="green", + padding=(1, 2), + ) + ) + console.print() + + +def _store_vericore_credentials( + env: str, keypair, api_key: str +) -> typing.Tuple[bool, typing.Optional[str]]: + api_url = ENV_URLS[env] + timestamp = int(time.time()) + payload = f"{keypair.ss58_address}:{timestamp}" + signature = keypair.sign(payload.encode()) + signature_base64 = base64.b64encode(signature).decode() + public_key_hex = keypair.public_key.hex() + + try: + with httpx.Client(timeout=30.0) as client: + response = client.post( + f"{api_url}/api/v3/miner/services/link", + json={ + "service_name": "vericore", + "auth_type": "api_key", + "credential_data": { + "api_key": api_key, + }, + }, + headers={ + "Authorization": f"Bearer {signature_base64}", + "Miner-Public-Key": public_key_hex, + "Miner": keypair.ss58_address, + "X-Payload": payload, + "Content-Type": "application/json", + }, + ) + + if response.status_code == 200: + return True, None + + error_detail = "Unknown error" + try: + error_data = response.json() + error_detail = error_data.get("detail", str(response.text)) + except Exception: + error_detail = response.text or f"HTTP {response.status_code}" + + return False, error_detail + + except httpx.TimeoutException: + return False, "Request timed out" + except httpx.ConnectError: + return False, "Connection failed" + except Exception as e: + return False, str(e) diff --git a/neurons/miner/scripts/services.py b/neurons/miner/scripts/services.py index 825f5fe..0e93ea0 100644 --- a/neurons/miner/scripts/services.py +++ b/neurons/miner/scripts/services.py @@ -15,6 +15,7 @@ from neurons.miner.scripts.link_desearch import link_desearch_impl from neurons.miner.scripts.link_openai import link_openai_impl from neurons.miner.scripts.link_perplexity import link_perplexity_impl +from neurons.miner.scripts.link_vericore import link_vericore_impl from neurons.miner.scripts.numinous_config import ENV_URLS from neurons.miner.scripts.wallet_utils import load_keypair, prompt_wallet_selection @@ -33,6 +34,7 @@ def services(): numi services link chutes # Link Chutes directly numi services link openai # Link OpenAI directly numi services link perplexity # Link Perplexity directly + numi services link vericore # Link Vericore directly numi services unlink # Unlink a service \b @@ -43,6 +45,7 @@ def services(): numi services link chutes numi services link openai numi services link perplexity + numi services link vericore numi services unlink chutes """ pass @@ -172,6 +175,7 @@ def link( - chutes: Link Chutes API key - openai: Link OpenAI API key - perplexity: Link Perplexity API key + - vericore: Link Vericore API key \b Examples: @@ -180,12 +184,13 @@ def link( numi services link chutes # Link Chutes directly numi services link openai # Link OpenAI directly numi services link perplexity # Link Perplexity directly + numi services link vericore # Link Vericore directly """ if not service_name: console.print() service_choice = Prompt.ask( "[bold cyan]Select service to link[/bold cyan]", - choices=["desearch", "chutes", "openai", "perplexity"], + choices=["desearch", "chutes", "openai", "perplexity", "vericore"], default="desearch", ) service_name = service_choice.lower() @@ -199,6 +204,8 @@ def link( link_openai_impl(wallet, hotkey, env, wallet_path) elif service_name == "perplexity": link_perplexity_impl(wallet, hotkey, env, wallet_path) + elif service_name == "vericore": + link_vericore_impl(wallet, hotkey, env, wallet_path) else: console.print(f"[red]✗ Unknown service:[/red] {service_name}") raise click.Abort() diff --git a/neurons/miner/scripts/test_agent_lib/execution.py b/neurons/miner/scripts/test_agent_lib/execution.py index a476b92..605af27 100644 --- a/neurons/miner/scripts/test_agent_lib/execution.py +++ b/neurons/miner/scripts/test_agent_lib/execution.py @@ -14,7 +14,7 @@ console = Console() -SANDBOX_TIMEOUT_SECONDS = 210 +SANDBOX_TIMEOUT_SECONDS = 240 def get_gateway_url() -> str: diff --git a/neurons/miner/scripts/test_agent_lib/preflight.py b/neurons/miner/scripts/test_agent_lib/preflight.py index e4299a6..4aa655c 100644 --- a/neurons/miner/scripts/test_agent_lib/preflight.py +++ b/neurons/miner/scripts/test_agent_lib/preflight.py @@ -222,14 +222,14 @@ def run_preflight_checks() -> bool: console.print() console.print( Panel.fit( - f"[yellow]💡 To set up API keys manually, edit:[/yellow]\n" + f"[yellow]To set up API keys manually, edit:[/yellow]\n" f" [cyan]{GATEWAY_ENV_PATH}[/cyan]\n\n" - "[yellow]Required keys:[/yellow]\n" - " CHUTES_API_KEY=your_key_here\n" - " DESEARCH_API_KEY=your_key_here\n\n" "[yellow]Get your keys from:[/yellow]\n" - " • Chutes: [link=https://chutes.ai]https://chutes.ai[/link]\n" - " • Desearch: [link=https://desearch.ai]https://desearch.ai[/link]", + " - Chutes: [link=https://chutes.ai]https://chutes.ai[/link]\n" + " - Desearch: [link=https://desearch.ai]https://desearch.ai[/link]\n" + " - OpenAI: [link=https://platform.openai.com/api-keys]https://platform.openai.com/api-keys[/link]\n" + " - Perplexity: [link=https://www.perplexity.ai/settings/api]https://www.perplexity.ai/settings/api[/link]\n" + " - Vericore: [link=https://vericore.ai]https://vericore.ai[/link]", border_style="yellow", ) ) diff --git a/neurons/validator/main.py b/neurons/validator/main.py index 61fb508..065ade4 100644 --- a/neurons/validator/main.py +++ b/neurons/validator/main.py @@ -130,7 +130,7 @@ async def main(): subtensor=AsyncSubtensor(config=config), logger=logger, max_concurrent_sandboxes=config.get("sandbox", {}).get("max_concurrent", 50), - timeout_seconds=config.get("sandbox", {}).get("timeout_seconds", 210), + timeout_seconds=config.get("sandbox", {}).get("timeout_seconds", 240), sync_hour=validator_sync_hour, validator_uid=validator_uid, validator_hotkey=validator_hotkey, diff --git a/neurons/validator/models/numinous_client.py b/neurons/validator/models/numinous_client.py index c446080..966b4c3 100644 --- a/neurons/validator/models/numinous_client.py +++ b/neurons/validator/models/numinous_client.py @@ -20,6 +20,7 @@ ) from neurons.validator.models.openai import OpenAIResponse from neurons.validator.models.perplexity import PerplexityCompletion +from neurons.validator.models.vericore import VericoreResponse class NuminousEvent(BaseModel): @@ -373,6 +374,15 @@ class GatewayPerplexityCompletion(PerplexityCompletion, GatewayCallResponse): pass +class VericoreCalculateRatingRequest(GatewayCall): + statement: str = Field(..., description="Statement to verify") + generate_preview: bool = Field(default=False, description="Generate a preview URL") + + +class GatewayVericoreResponse(VericoreResponse, GatewayCallResponse): + pass + + class MinerWeight(BaseModel): miner_uid: int miner_hotkey: str diff --git a/neurons/validator/models/vericore.py b/neurons/validator/models/vericore.py new file mode 100644 index 0000000..31ef51d --- /dev/null +++ b/neurons/validator/models/vericore.py @@ -0,0 +1,55 @@ +from decimal import Decimal +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + +VERICORE_COST_PER_CALL = Decimal("0.05") + + +class VericoreStatement(BaseModel): + statement: str + url: str + contradiction: float = Field(..., ge=0, le=1) + neutral: float = Field(..., ge=0, le=1) + entailment: float = Field(..., ge=0, le=1) + sentiment: float + conviction: float = Field(..., ge=0, le=1) + source_credibility: float = Field(..., ge=0, le=1) + narrative_momentum: float = Field(..., ge=0, le=1) + risk_reward_sentiment: float + political_leaning: float = 0.0 + catalyst_detection: float = Field(..., ge=0, le=1) + + model_config = ConfigDict(extra="allow") + + +class VericoreEvidenceSummary(BaseModel): + total_count: int + support: Optional[float] = None + neutral: float + refute: Optional[float] = None + entailment: Optional[float] = None + contradiction: Optional[float] = None + sentiment: float + conviction: float + source_credibility: float + narrative_momentum: float + risk_reward_sentiment: float + political_leaning: float = 0.0 + catalyst_detection: float + statements: list[VericoreStatement] + + model_config = ConfigDict(extra="allow") + + +class VericoreResponse(BaseModel): + batch_id: str + request_id: str + preview_url: str + evidence_summary: VericoreEvidenceSummary + + model_config = ConfigDict(extra="allow") + + +def calculate_cost() -> Decimal: + return VERICORE_COST_PER_CALL diff --git a/neurons/validator/numinous_client/client.py b/neurons/validator/numinous_client/client.py index 14c7fb2..52b6b10 100644 --- a/neurons/validator/numinous_client/client.py +++ b/neurons/validator/numinous_client/client.py @@ -24,7 +24,9 @@ PostAgentRunsRequestBody, PostPredictionsRequestBody, PostScoresRequestBody, + VericoreCalculateRatingRequest, ) +from neurons.validator.models.vericore import VericoreResponse from neurons.validator.utils.config import NuminousEnvType from neurons.validator.utils.git import commit_short_hash from neurons.validator.utils.logger.logger import NuminousLogger @@ -359,6 +361,29 @@ async def desearch_ai_search(self, body: dict | DesearchAISearchRequest): data = await response.json() return AISearchResponse.model_validate(data) + async def vericore_calculate_rating( + self, body: dict | VericoreCalculateRatingRequest + ) -> VericoreResponse: + if isinstance(body, dict): + try: + body = VericoreCalculateRatingRequest.model_validate(body) + except ValidationError as e: + raise ValueError(f"Invalid parameters: {e}") + + data = body.model_dump_json() + auth_headers = self.make_auth_headers(data=data) + + async with self.create_session( + other_headers={**auth_headers, "Content-Type": "application/json"} + ) as session: + path = "/api/gateway/vericore/calculate-rating" + + async with session.post(path, data=data) as response: + response.raise_for_status() + + data = await response.json() + return VericoreResponse.model_validate(data) + async def get_weights(self): auth_headers = self.make_get_auth_headers() diff --git a/neurons/validator/utils/config.py b/neurons/validator/utils/config.py index 866c6e5..03804b7 100644 --- a/neurons/validator/utils/config.py +++ b/neurons/validator/utils/config.py @@ -53,8 +53,8 @@ def get_config(): parser.add_argument( "--sandbox.timeout_seconds", type=int, - default=210, - help="Timeout for agent execution in seconds (default: 210)", + default=240, + help="Timeout for agent execution in seconds (default: 240)", ) parser.add_argument( "--validator.sync_hour", diff --git a/neurons/validator/version.py b/neurons/validator/version.py index a15373a..f271541 100644 --- a/neurons/validator/version.py +++ b/neurons/validator/version.py @@ -1,4 +1,4 @@ -__version__ = "2.1.0" +__version__ = "2.1.1" version_split = __version__.split(".")