added visualization work

.gitignore

@@ -2,6 +2,7 @@
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
+viz/*.json
 
 db_relations/*
 # User-specific stuff

crates/av-client/examples/news_analysis.rs

@@ -42,6 +42,7 @@ use av_core::{Config, Error};
 use av_models::news::NewsSentiment;
 use std::collections::HashMap;
 use tokio::time::{Duration, Instant, sleep};
+use tracing::error;
 
 /// Sentiment analysis results for a group of securities
 #[derive(Debug)]
@@ -101,7 +102,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
   // Load configuration
   let config = Config::from_env().map_err(|e| {
-    eprintln!("Failed to load configuration. Make sure ALPHA_VANTAGE_API_KEY is set.");
+    error!("Failed to load configuration. Make sure ALPHA_VANTAGE_API_KEY is set.");
     e
   })?;
 

crates/av-client/examples/portfolio_tracker.rs

@@ -44,6 +44,7 @@ use std::io::{self, Write};
 use av_models::{fundamentals::TopGainersLosers, time_series::GlobalQuote};
 use std::collections::HashMap;
 use tokio::time::{Duration, sleep};
+use tracing::error;
 
 /// Portfolio holding information
 #[derive(Debug, Clone)]
@@ -91,8 +92,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
   // Load configuration from environment
   let config = Config::from_env().map_err(|e| {
-    eprintln!("Failed to load configuration. Make sure ALPHA_VANTAGE_API_KEY is set.");
-    eprintln!("Error: {}", e);
+    error!("Failed to load configuration. Make sure ALPHA_VANTAGE_API_KEY is set. Error: {}", e);
     e
   })?;
 

crates/av-database/postgres/src/repository.rs

@@ -37,6 +37,7 @@ use chrono::{DateTime, Utc};
 use diesel::prelude::*;
 use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
 use diesel::result::Error as DieselError;
+use log::error;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -1427,7 +1428,7 @@ impl CryptoRepository for CryptoRepositoryImpl {
         match result {
           Ok(_) => total_affected += 1,
           Err(e) => {
-            eprintln!("Error upserting market {:?}/{:?}: {}", market.exchange, market.base, e)
+            error!("Error upserting market {:?}/{:?}: {}", market.exchange, market.base, e)
           }
         }
       }
@@ -1487,7 +1488,7 @@ impl CryptoRepository for CryptoRepositoryImpl {
 
         match result {
           Ok(_) => saved_count += 1,
-          Err(e) => eprintln!("Error upserting social data for sid {}: {}", social.sid, e),
+          Err(e) => error!("Error upserting social data for sid {}: {}", social.sid, e),
         }
       }
 
@@ -1549,7 +1550,7 @@ impl CryptoRepository for CryptoRepositoryImpl {
 
         match result {
           Ok(_) => saved_count += 1,
-          Err(e) => eprintln!("Error upserting technical data for sid {}: {}", technical.sid, e),
+          Err(e) => error!("Error upserting technical data for sid {}: {}", technical.sid, e),
         }
       }
 

tests/integration/crypto_update_integrations

@@ -485,7 +485,7 @@ async fn main() -> Result<()> {
         info!("Manual updates completed: {} basic, {} social, {} technical",
             basic_count, social_count, technical_count);
     } else {
-        eprintln!("Manual updates contain validation errors, skipping...");
+        error!("Manual updates contain validation errors, skipping...");
     }
 
     // Final statistics

viz/README.md

@@ -0,0 +1,136 @@
+# Data Visualization - POC
+
+A static, self-contained D3.js visualization showing relationships between news articles and stock symbols.
+
+**crypto_heatmap.html** is a simple heatmap visualization of the crypto names by market cap. Highlights a problem with symbol mapping. see 
+
+Will provide more complete instructions in a future commit.
+
+This is my first foray into frontend work and will move this to a separate repo but for now it is actually useful for debugging and ideas
+
+## Quick Start
+
+### 1. Extract Data from Database
+
+```bash
+cd viz
+./extract_data.sh
+```
+
+This will:
+- Extract ticker symbols from article titles using regex
+- Query the PostgreSQL database for the top 50 most-mentioned symbols
+- Fetch 200 recent articles containing ticker symbols
+- Generate relationships by matching symbols in article titles
+- Save everything to `news_data.json`
+
+### 2. View Visualization
+
+Open `index.html` in a web browser:
+
+```bash
+# Option 1: Direct open
+open index.html  # macOS
+xdg-open index.html  # Linux
+start index.html  # Windows
+
+# Option 2: Simple HTTP server (recommended for production)
+python3 -m http.server 8000
+# Then visit: http://localhost:8000
+```
+
+## Features
+
+### Visualization Elements
+
+- **Nodes**:
+  - 🔵 **Blue circles** = Stock symbols (NVDA, TSLA, AAPL, etc.)
+    - Size proportional to number of article mentions
+    - Larger nodes = more frequently mentioned symbols
+  - 🟣 **Purple circles** = News articles
+    - Smaller, uniform size
+    - Click to open article URL
+
+- **Links** (Edges):
+  - ⚪ Gray lines connecting symbols to articles that mention them
+  - Derived from regex pattern matching in article titles
+
+### Interactive Features
+
+1. **Hover**: View detailed information:
+   - Symbols: Show ticker, company name, and article count
+   - Articles: Show full title and publish date
+2. **Click**: Click article nodes to open the source URL in a new tab
+3. **Drag**: Drag nodes to manually arrange the graph
+4. **Zoom/Pan**: Scroll to zoom, drag background to pan
+
+### Stats Panel
+
+Real-time statistics showing:
+- Number of symbols displayed (50)
+- Number of articles displayed (200)
+- Number of connections (68+)
+
+## Customization
+
+### Adjust Time Range
+
+Edit `extract_news_data.sql` line 30:
+```sql
+WHERE a.ct >= CURRENT_DATE - INTERVAL '30 days'  -- Change to '60 days', '7 days', etc.
+```
+
+### Change Symbol Limit
+
+Edit `extract_news_data.sql` line 17:
+```sql
+LIMIT 25  -- Change to 50, 100, etc.
+```
+
+### Modify Visual Appearance
+
+Edit `index.html`:
+
+**Colors** (around line 210):
+```javascript
+const typeColors = {
+    'Equity': '#3b82f6',        // Change colors here
+    'Cryptocurrency': '#f59e0b',
+    'ETF': '#10b981',
+    // Add more types as needed
+};
+```
+
+**Node Sizes** (around line 288):
+```javascript
+size: Math.sqrt(symbol.article_count) * 8 + 10  // Adjust multiplier
+```
+
+**Force Simulation** (around line 386):
+```javascript
+.force('charge', d3.forceManyBody().strength(-300))  // Adjust repulsion
+.force('link', d3.forceLink(links).distance(100))    // Adjust link distance
+```
+
+
+
+
+
+## Performance Notes
+
+- **Recommended limits**:
+  - Symbols: 25-50 (current: 25)
+  - Articles: 100-500 per symbol
+  - Total nodes: < 1000 for smooth interaction
+
+- **For larger datasets**:
+  - Increase time to stabilize: `simulation.alphaTarget(0.01)`
+
+
+
+## Dependencies
+
+- **D3.js v7**: Loaded from CDN (https://d3js.org/d3.v7.min.js)
+- **jq** (optional): For JSON formatting in extraction script
+
+No build process required - it's pure HTML/JavaScript