Instead of Pre-loading of dataset, now server will use the lazy loading, i.e. load dataset only when a user sends request, overall it's loaded only once

ASR_Server.py

@@ -38,16 +38,32 @@ job_status = {
     "error_trace": None
 }
 
+
 csv_path = "test.csv"
-# csv_transcript = f'test_with_{ASR_model.replace("/", "_")}.csv'
-# csv_result = f'test_with_{ASR_model.replace("/","_")}_WER.csv'
 df = pd.read_csv(csv_path)
 print(f"CSV Loaded with {len(df)} rows")
 
-# Load dataset without decoding audio (required!)
-dataset = load_dataset("satyamr196/asr_fairness_audio", split="train")
-# dataset = dataset.with_format("python", decode_audio=False)
-dataset = dataset.cast_column("audio", Audio(decode=False))
+# lazy loading of dataset code:
+_dataset = None
+_dataset_lock = threading.Lock()
+
+def get_dataset():
+    """Lazily loads the dataset and ensures it's only loaded once."""
+    global _dataset
+    # Use a lock to prevent race conditions (two requests loading at once)
+    with _dataset_lock:
+        if _dataset is None:
+            print("Loading dataset for the first time...")
+            try:
+                ds = load_dataset("satyamr196/asr_fairness_audio", split="train")
+                _dataset = ds.cast_column("audio", Audio(decode=False))
+                print("Dataset loaded successfully.")
+            except Exception as e:
+                print(f"FATAL: Failed to load dataset: {e}")
+                # Propagate the error
+                raise
+        return _dataset
+# --- END LAZY LOADING ---
 
 def generateTranscript(ASR_model):
     import os
@@ -76,7 +92,23 @@ def generateTranscript(ASR_model):
     csv_result = f'test_with_{ASR_model.replace("/","_")}_WER.csv'
     
     try:
-    
+        
+        # --- BLOCK TO LOAD THE DATASET ---
+        try:
+            print("Attempting to get dataset...")
+            dataset = get_dataset()
+        except Exception as e:
+            tb = traceback.format_exc()
+            print(f"❌ Failed to load dataset: {e}")
+            job_status.update({
+                "running": False,
+                "message": f"Critical error: Failed to load dataset.",
+                "error": str(e),
+                "error_trace": tb[:1000],
+            })
+            return # Stop the function if dataset fails to load
+        # --- END OF dataset load BLOCK ---
+        
         # Check if transcript already exists
         df_transcript = download_csv(csv_transcript)
         if(df_transcript is None):