"""Data preprocessing and feature engineering""" import pandas as pd import numpy as np from miner_specs import MINER_SPECS, ELECTRICITY_RATES from fetch_blockchain_data import get_days_since_halving from electricity_prices import get_electricity_rate def engineer_features(blockchain_df): """Engineer features from blockchain data - keep it simple""" df = blockchain_df.copy().sort_values('date').reset_index(drop=True) df['date'] = pd.to_datetime(df['date']) # Just return as is, we'll select features later return df def prepare_miner_features(blockchain_df, miner_name, miner_price, region='texas'): """Add miner-specific features - EXACTLY 14 features""" df = blockchain_df.copy() specs = MINER_SPECS[miner_name] # Keep only these columns from blockchain data df = df[['date', 'bitcoin_price', 'difficulty', 'fees', 'hashrate', 'revenue', 'block_reward']].copy() df['date'] = pd.to_datetime(df['date']) # Add miner features df['machine_price'] = miner_price df['machine_hashrate'] = specs['hashrate'] df['power'] = specs['power'] df['efficiency'] = specs['efficiency'] # Calculate age_days (days since miner was released) release_date = pd.to_datetime(specs['release_date']) df['age_days'] = (df['date'] - release_date).dt.days # Days since halving df['days_since_halving'] = df['date'].apply(get_days_since_halving) # Revenue potential hashrate_hs = df['machine_hashrate'] * 1e12 btc_per_day = (hashrate_hs * 86400) / (df['difficulty'] * (2**32)) * (df['block_reward'] + (df['fees']/144)) df['Revenue_Potential'] = btc_per_day * df['bitcoin_price'] # Electricity rate # df['electricity_rate'] = ELECTRICITY_RATES.get(region, 0.10) df['electricity_rate'] = df['date'].dt.date.apply( lambda day: get_electricity_rate(region, day) ) return df def get_latest_sequence(blockchain_df, miner_name, miner_price, region='texas', window_size=30): """Get the most recent sequence for prediction - EXACTLY 14 features in CORRECT ORDER""" df_features = engineer_features(blockchain_df) df_miner = prepare_miner_features(df_features, miner_name, miner_price, region) # CRITICAL: This order MUST match your training data CSV exactly! # Your training CSV: bitcoin_price,difficulty,fees,hashrate,revenue,machine_price,machine_hashrate,power,efficiency,block_reward,age_days,days_since_halving,Revenue_Potential,electricity_rate feature_cols = [ 'bitcoin_price', # 1 'difficulty', # 2 'fees', # 3 'hashrate', # 4 'revenue', # 5 'machine_price', # 6 'machine_hashrate', # 7 'power', # 8 'efficiency', # 9 'block_reward', # 10 'age_days', # 11 'days_since_halving',# 12 'Revenue_Potential', # 13 'electricity_rate' # 14 ] df_miner = df_miner.dropna().reset_index(drop=True) if len(df_miner) < window_size: raise ValueError(f"Not enough data: need {window_size} days, have {len(df_miner)}") # Get last window_size days with exactly 14 features sequence = df_miner[feature_cols].values[-window_size:] latest_date = df_miner['date'].iloc[-1] # Verify shape if sequence.shape[1] != 14: raise ValueError(f"Expected 14 features, got {sequence.shape[1]}") return sequence, feature_cols, latest_date if __name__ == "__main__": from fetch_blockchain_data import get_latest_blockchain_data print("\n" + "="*80) print("TESTING PREPROCESSING PIPELINE") print("="*80 + "\n") # Fetch blockchain data print("📡 Fetching blockchain data...") blockchain_df = get_latest_blockchain_data(days=90) if blockchain_df is None: print("❌ Failed to fetch blockchain data") exit(1) print(f"✅ Fetched {len(blockchain_df)} days of data\n") # Test configuration miner_name = 's19pro' miner_price = 2500 region = 'texas' window_size = 30 print("⚙️ Test Configuration:") print(f" Miner: {MINER_SPECS[miner_name]['full_name']}") print(f" Price: ${miner_price:,}") print(f" Region: {region.title()}") print(f" Window: {window_size} days") print(f" Electricity: ${ELECTRICITY_RATES[region]}/kWh\n") # Step 1: Engineer features print("="*80) print("STEP 1: ENGINEER FEATURES") print("="*80) df_engineered = engineer_features(blockchain_df) print(f"✅ Engineered features") print(f" Shape: {df_engineered.shape}") print(f" Columns: {list(df_engineered.columns)}\n") print("First 3 rows:") print(df_engineered.head(3)) print("\nLast 3 rows:") print(df_engineered.tail(3)) # Step 2: Prepare miner features print("\n" + "="*80) print("STEP 2: PREPARE MINER FEATURES") print("="*80) df_miner = prepare_miner_features(df_engineered, miner_name, miner_price, region) print(f"✅ Added miner-specific features") print(f" Shape: {df_miner.shape}") print(f" Columns: {list(df_miner.columns)}\n") print("Miner-specific values (constant across all days):") print(f" machine_hashrate: {df_miner['machine_hashrate'].iloc[0]} TH/s") print(f" power: {df_miner['power'].iloc[0]} W") print(f" efficiency: {df_miner['efficiency'].iloc[0]} W/TH") print(f" machine_price: ${df_miner['machine_price'].iloc[0]:,.2f}") print(f" electricity_rate: ${df_miner['electricity_rate'].iloc[0]:.4f}/kWh") print("\nDynamic values (change over time):") print(f" age_days: {df_miner['age_days'].iloc[0]} → {df_miner['age_days'].iloc[-1]} days") print(f" days_since_halving: {df_miner['days_since_halving'].iloc[0]} → {df_miner['days_since_halving'].iloc[-1]} days") print(f" Revenue_Potential: ${df_miner['Revenue_Potential'].iloc[0]:.2f} → ${df_miner['Revenue_Potential'].iloc[-1]:.2f}/day") print("\nFirst 3 rows:") print(df_miner.head(3)) print("\nLast 3 rows:") print(df_miner.tail(3)) # Step 3: Get latest sequence print("\n" + "="*80) print("STEP 3: GET LATEST SEQUENCE") print("="*80) sequence, feature_cols, latest_date = get_latest_sequence(blockchain_df, miner_name, miner_price, region, window_size) print(f"✅ Created sequence for model") print(f" Shape: {sequence.shape}") print(f" Expected: ({window_size}, 14)") print(f" Latest date: {latest_date.strftime('%Y-%m-%d')}\n") print("14 Features (in order):") for i, col in enumerate(feature_cols, 1): print(f" {i:2d}. {col:25s} → First: {sequence[0, i-1]:>15.2f} Last: {sequence[-1, i-1]:>15.2f}") print("\n" + "="*80) print("SEQUENCE STATISTICS") print("="*80) print("\nFirst day in sequence:") for i, col in enumerate(feature_cols): print(f" {col:25s} = {sequence[0, i]:>15.2f}") print(f"\nLast day in sequence (for prediction on {latest_date.strftime('%Y-%m-%d')}):") for i, col in enumerate(feature_cols): print(f" {col:25s} = {sequence[-1, i]:>15.2f}") # Show some statistics print("\n" + "="*80) print("FEATURE RANGES") print("="*80) for i, col in enumerate(feature_cols): min_val = sequence[:, i].min() max_val = sequence[:, i].max() mean_val = sequence[:, i].mean() print(f"{col:25s} → Min: {min_val:>12.2f} Max: {max_val:>12.2f} Mean: {mean_val:>12.2f}") print("\n" + "="*80) print("✅ PREPROCESSING PIPELINE TEST COMPLETE") print("="*80 + "\n")