run_benchmarks.py

import pandas as pd
import dill
import numpy as np
import time
import os
from tensorflow.keras.models import load_model
from sklearn.preprocessing import MinMaxScaler
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='[ %(asctime)s ] - %(message)s')

# CONFIG
N_LATENCY_RUNS = 100            # Number of times to test single-sample latency
N_THROUGHPUT_SAMPLES = 10000    # Number of samples to test batch throughput


try:
    logging.info("Starting Model Benchmark...")

    # 1. Load the 55-feature PROCESSED test data
    logging.info("Loading data from artifacts/test_processed.csv...")
    processed_test_df = pd.read_csv("artifacts/test_processed.csv")
    X_test = processed_test_df.drop('label', axis=1)

    # Prepare data for NN (scaled)
    scaler = MinMaxScaler()
    X_test_scaled = scaler.fit_transform(X_test)

    logging.info(f"Test data shape: {X_test.shape}")

    # --- 2. XGBoost Benchmarking ---
    logging.info("\n--- Benchmarking XGBoost (model_trained.pkl) ---")
    xgb_model = dill.load(open("artifacts/model_trained.pkl", "rb"))

    # Get samples
    X_sample_xgb = X_test.iloc[0:1]
    X_batch_xgb = X_test.sample(n=N_THROUGHPUT_SAMPLES, random_state=42)

    # A) Latency
    latencies_xgb = []
    logging.info(f"Testing XGBoost latency ({N_LATENCY_RUNS} runs)...")
    for _ in range(N_LATENCY_RUNS):
        start = time.perf_counter()
        xgb_model.predict(X_sample_xgb)
        end= time.perf_counter()
        latencies_xgb.append((end - start) * 1000) # Convert to milliseconds

    avg_latency_xgb = np.mean(latencies_xgb)
    logging.info(f"XGBoost Avg. Latency: {avg_latency_xgb:.4f} ms/sample")

    # B) Throughput (samples/sec)
    logging.info(f"Testing XGBoost throughput ({N_THROUGHPUT_SAMPLES} samples)...")
    start = time.perf_counter()
    xgb_model.predict(X_batch_xgb)
    end = time.perf_counter()

    total_time_xgb = end - start
    throughput_xgb = N_THROUGHPUT_SAMPLES / total_time_xgb
    logging.info(f"XGBoost Throughput: {throughput_xgb:.2f} samples/sec")

    # C) Model Size
    size_xgb = os.path.getsize("artifacts/model_trained.pkl") / (1024**2) # Convert to MB
    logging.info(f"XGBoost Model Size: {size_xgb:.2f} MB")


    # --- 3. CNN+LSTM Benchmarking ---
    logging.info("\n--- Benchmarking CNN+LSTM (model_trained.keras) ---")
    nn_model = load_model("artifacts/model_trained.keras")

    # Get samples (scaled)
    X_sample_nn = X_test_scaled[0:1]
    X_batch_nn = pd.DataFrame(X_test_scaled).sample(n=N_THROUGHPUT_SAMPLES, random_state=42).values

    # A) Latency (ms/sample)
    latencies_nn = []
    logging.info(f"Testing NN latency ({N_LATENCY_RUNS} runs)...")
    for _ in range(N_LATENCY_RUNS):
        start = time.perf_counter()
        nn_model.predict(X_sample_nn, verbose=0)
        end = time.perf_counter()
        latencies_nn.append((end - start) * 1000)

    avg_latency_nn = np.mean(latencies_nn)
    logging.info(f"NN Avg. Latency: {avg_latency_nn:.4f} ms/sample")

    # B) Throughput (samples/sec)
    logging.info(f"Testing NN throughput ({N_THROUGHPUT_SAMPLES} samples)...")
    start = time.perf_counter()
    nn_model.predict(X_batch_nn, batch_size=32, verbose=0)
    end = time.perf_counter()

    total_time_nn = end - start
    throughput_nn = N_THROUGHPUT_SAMPLES / total_time_nn
    logging.info(f"NN Throughput: {throughput_nn:.2f} samples/sec")

    # C) Model Size
    size_nn = os.path.getsize("artifacts/model_trained.keras") / (1024**2) # Convert to MB
    logging.info(f"NN Model Size: {size_nn:.2f} MB")

    # --- 4. Final Report ---
    print("\n" + "="*80)
    print("FINAL BENCHMARK RESULTS (for Table 2)")
    print("="*80)
    print(f"{'Model':<15} | {'Avg. Latency (ms/sample)':<25} | {'Throughput (samples/sec)':<25} | {'Model Size (MB)':<15}")
    print("-" * 80)
    print(f"{'XGBoost':<15} | {avg_latency_xgb:<25.4f} | {throughput_xgb:<25.2f} | {size_xgb:<15.2f}")
    print(f"{'CNN+LSTM':<15} | {avg_latency_nn:<25.4f} | {throughput_nn:<25.2f} | {size_nn:<15.2f}")
    print("="*80)

except Exception as e:
    logging.error(f"An error occurred: {e}")
    import traceback
    traceback.print_exc()