|
| 1 | +""" |
| 2 | +🌙 Moon Dev - Cleanup Duplicate Winner Strategies |
| 3 | +Removes duplicate strategies, keeping only the 11 unique ones |
| 4 | +""" |
| 5 | + |
| 6 | +import json |
| 7 | +import os |
| 8 | +from collections import defaultdict |
| 9 | +import shutil |
| 10 | + |
| 11 | +WINNERS_DIR = 'src/data/rbi_auto/winners/' |
| 12 | +STRATEGIES_DIR = 'src/strategies/auto_generated/' |
| 13 | +BACKUP_DIR = 'src/data/rbi_auto/duplicates_backup/' |
| 14 | + |
| 15 | +# Create backup directory |
| 16 | +os.makedirs(BACKUP_DIR, exist_ok=True) |
| 17 | +os.makedirs(os.path.join(BACKUP_DIR, 'winners'), exist_ok=True) |
| 18 | +os.makedirs(os.path.join(BACKUP_DIR, 'strategies'), exist_ok=True) |
| 19 | + |
| 20 | +print("=" * 90) |
| 21 | +print("🧹 CLEANUP DUPLICATE WINNER STRATEGIES") |
| 22 | +print("=" * 90) |
| 23 | + |
| 24 | +# Load all winners |
| 25 | +winners = [] |
| 26 | +for filename in os.listdir(WINNERS_DIR): |
| 27 | + if filename.endswith('.json'): |
| 28 | + filepath = os.path.join(WINNERS_DIR, filename) |
| 29 | + with open(filepath, 'r') as f: |
| 30 | + winner = json.load(f) |
| 31 | + winner['filename'] = filename |
| 32 | + winner['filepath'] = filepath |
| 33 | + winners.append(winner) |
| 34 | + |
| 35 | +print(f"\n📊 Found {len(winners)} winner files") |
| 36 | + |
| 37 | +# Group by unique signature |
| 38 | +unique_groups = defaultdict(list) |
| 39 | + |
| 40 | +for winner in winners: |
| 41 | + strat = winner['strategy'] |
| 42 | + |
| 43 | + # Create signature from key parameters |
| 44 | + signature = ( |
| 45 | + strat.get('fast_ma'), |
| 46 | + strat.get('slow_ma'), |
| 47 | + strat.get('use_ema'), |
| 48 | + strat.get('leverage'), |
| 49 | + strat.get('stop_loss_pct'), |
| 50 | + strat.get('take_profit_pct'), |
| 51 | + strat.get('rsi_min'), |
| 52 | + strat.get('use_volume'), |
| 53 | + strat.get('volume_mult'), |
| 54 | + ) |
| 55 | + |
| 56 | + unique_groups[signature].append(winner) |
| 57 | + |
| 58 | +print(f"✅ Found {len(unique_groups)} unique configurations") |
| 59 | +print(f"❌ Duplicates to remove: {len(winners) - len(unique_groups)}") |
| 60 | + |
| 61 | +# Track what we'll keep and remove |
| 62 | +to_keep = [] |
| 63 | +to_remove = [] |
| 64 | + |
| 65 | +print(f"\n{'='*90}") |
| 66 | +print("PROCESSING EACH UNIQUE CONFIGURATION") |
| 67 | +print(f"{'='*90}\n") |
| 68 | + |
| 69 | +for signature, group in unique_groups.items(): |
| 70 | + if len(group) == 1: |
| 71 | + # No duplicates, keep it |
| 72 | + to_keep.append(group[0]) |
| 73 | + config = group[0]['strategy']['name'] |
| 74 | + print(f"✅ Unique: {config}") |
| 75 | + else: |
| 76 | + # Multiple duplicates - keep the best performing one |
| 77 | + # Sort by return (descending) |
| 78 | + group_sorted = sorted(group, key=lambda x: x['results']['return'], reverse=True) |
| 79 | + best = group_sorted[0] |
| 80 | + duplicates = group_sorted[1:] |
| 81 | + |
| 82 | + to_keep.append(best) |
| 83 | + to_remove.extend(duplicates) |
| 84 | + |
| 85 | + config = best['strategy']['name'] |
| 86 | + print(f"🔍 Found {len(group)} duplicates of: {config}") |
| 87 | + print(f" ✅ Keeping best: {best['filename']} ({best['results']['return']:.2f}%)") |
| 88 | + print(f" ❌ Removing {len(duplicates)} duplicates:") |
| 89 | + for dup in duplicates: |
| 90 | + print(f" - {dup['filename']}") |
| 91 | + |
| 92 | +# Perform cleanup |
| 93 | +print(f"\n{'='*90}") |
| 94 | +print("CLEANUP ACTIONS") |
| 95 | +print(f"{'='*90}\n") |
| 96 | + |
| 97 | +print(f"Backing up {len(to_remove)} duplicate files...") |
| 98 | + |
| 99 | +removed_count = 0 |
| 100 | +for winner in to_remove: |
| 101 | + # Backup winner JSON |
| 102 | + src_json = winner['filepath'] |
| 103 | + dst_json = os.path.join(BACKUP_DIR, 'winners', winner['filename']) |
| 104 | + |
| 105 | + if os.path.exists(src_json): |
| 106 | + shutil.move(src_json, dst_json) |
| 107 | + print(f" 📦 Backed up: {winner['filename']}") |
| 108 | + removed_count += 1 |
| 109 | + |
| 110 | + # Find and remove corresponding strategy file(s) |
| 111 | + strategy_name = winner['strategy']['name'].replace(' ', '').replace('-', '').replace('/', '_').replace('.', '_') |
| 112 | + |
| 113 | + # Strategy files might have different rank numbers and timestamps |
| 114 | + for strategy_file in os.listdir(STRATEGIES_DIR): |
| 115 | + if strategy_file.startswith(strategy_name): |
| 116 | + src_strategy = os.path.join(STRATEGIES_DIR, strategy_file) |
| 117 | + dst_strategy = os.path.join(BACKUP_DIR, 'strategies', strategy_file) |
| 118 | + |
| 119 | + if os.path.exists(src_strategy): |
| 120 | + shutil.move(src_strategy, dst_strategy) |
| 121 | + print(f" ↳ Removed strategy: {strategy_file}") |
| 122 | + |
| 123 | +print(f"\n✅ Removed {removed_count} duplicate winner files") |
| 124 | + |
| 125 | +# Verify final state |
| 126 | +remaining_winners = len([f for f in os.listdir(WINNERS_DIR) if f.endswith('.json')]) |
| 127 | +remaining_strategies = len([f for f in os.listdir(STRATEGIES_DIR) if f.endswith('.py')]) |
| 128 | + |
| 129 | +print(f"\n{'='*90}") |
| 130 | +print("FINAL STATE") |
| 131 | +print(f"{'='*90}\n") |
| 132 | + |
| 133 | +print(f"📁 Winners remaining: {remaining_winners}/{len(winners)}") |
| 134 | +print(f"📁 Strategies remaining: {remaining_strategies}") |
| 135 | +print(f"💾 Backups saved to: {BACKUP_DIR}") |
| 136 | + |
| 137 | +print(f"\n{'='*90}") |
| 138 | +print("UNIQUE WINNERS (sorted by return)") |
| 139 | +print(f"{'='*90}\n") |
| 140 | + |
| 141 | +# Display final unique list |
| 142 | +to_keep_sorted = sorted(to_keep, key=lambda x: x['results']['return'], reverse=True) |
| 143 | + |
| 144 | +print(f"{'Rank':<6} {'Strategy':<45} {'Return':<12} {'MaxDD':<10} {'Sharpe':<8}") |
| 145 | +print("-" * 90) |
| 146 | + |
| 147 | +for rank, winner in enumerate(to_keep_sorted, 1): |
| 148 | + strat = winner['strategy'] |
| 149 | + results = winner['results'] |
| 150 | + |
| 151 | + config = f"{strat.get('leverage')}x {'EMA' if strat.get('use_ema') else 'SMA'} {strat.get('fast_ma')}/{strat.get('slow_ma')} RSI>{strat.get('rsi_min')}" |
| 152 | + |
| 153 | + medal = "🥇" if rank == 1 else "🥈" if rank == 2 else "🥉" if rank == 3 else " " |
| 154 | + |
| 155 | + print(f"{medal} #{rank:<4} {config:<43} {results['return']:>9.2f}% {results['max_dd']:>8.2f}% {results['sharpe']:>6.2f}") |
| 156 | + |
| 157 | +print(f"\n{'='*90}") |
| 158 | +print("CLEANUP COMPLETE") |
| 159 | +print(f"{'='*90}\n") |
| 160 | + |
| 161 | +print(f"✅ {len(to_keep)} unique strategies remain") |
| 162 | +print(f"❌ {len(to_remove)} duplicates moved to backup") |
| 163 | +print(f"\n💡 To restore backups if needed:") |
| 164 | +print(f" cp {BACKUP_DIR}winners/*.json {WINNERS_DIR}") |
| 165 | +print(f" cp {BACKUP_DIR}strategies/*.py {STRATEGIES_DIR}") |
| 166 | + |
| 167 | +print(f"\n{'='*90}\n") |
0 commit comments