Spaces:
Running
Running
File size: 39,404 Bytes
092944c |
1 |
{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":115439,"databundleVersionId":13800781,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# 5cfcb5e8ef8458be6e85d57c45c7573477e2ad6a\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:16:22.802899Z","iopub.execute_input":"2025-11-29T09:16:22.803143Z","iopub.status.idle":"2025-11-29T09:16:22.808699Z","shell.execute_reply.started":"2025-11-29T09:16:22.803106Z","shell.execute_reply":"2025-11-29T09:16:22.808177Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/2025-sep-dl-gen-ai-project/sample_submission.csv\n/kaggle/input/2025-sep-dl-gen-ai-project/train.csv\n/kaggle/input/2025-sep-dl-gen-ai-project/test.csv\n","output_type":"stream"}],"execution_count":2},{"cell_type":"markdown","source":"# Initial work","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/train.csv\") #training set\ndt = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/test.csv\") #test set","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:16:27.537235Z","iopub.execute_input":"2025-11-29T09:16:27.537869Z","iopub.status.idle":"2025-11-29T09:16:27.579497Z","shell.execute_reply.started":"2025-11-29T09:16:27.537844Z","shell.execute_reply":"2025-11-29T09:16:27.578934Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nlabel_cols = ['anger','fear','joy','sadness','surprise']\n\nxtrain, xval, ytrain, yval = train_test_split(\n df['text'],\n df[label_cols].values,\n test_size=0.2,\n random_state=42,\n # shuffle=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:16:55.867984Z","iopub.execute_input":"2025-11-29T09:16:55.868758Z","iopub.status.idle":"2025-11-29T09:16:55.875658Z","shell.execute_reply.started":"2025-11-29T09:16:55.868724Z","shell.execute_reply":"2025-11-29T09:16:55.875052Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# wdb_t = user_secrets.get_secret(\"WB_TOKEN\")\n# import wandb\n# wandb.login(key=wdb_t)\n# # wandb.init(project=\"22f3001086-t32025\", name = \"BERT+Classifier head\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:16:46.454532Z","iopub.execute_input":"2025-11-29T09:16:46.455056Z","iopub.status.idle":"2025-11-29T09:16:46.935003Z","shell.execute_reply.started":"2025-11-29T09:16:46.455031Z","shell.execute_reply":"2025-11-29T09:16:46.934187Z"}},"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n","output_type":"stream"},{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}],"execution_count":6},{"cell_type":"markdown","source":"# Model n Definitions","metadata":{}},{"cell_type":"code","source":"import torch\nfrom tqdm import tqdm\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModel, get_linear_schedule_with_warmup\nfrom sklearn.metrics import f1_score\nimport numpy as np\nimport random\nimport wandb\nprint(\"imports done!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:17:13.658236Z","iopub.execute_input":"2025-11-29T09:17:13.658710Z","iopub.status.idle":"2025-11-29T09:17:13.663761Z","shell.execute_reply.started":"2025-11-29T09:17:13.658681Z","shell.execute_reply":"2025-11-29T09:17:13.662841Z"}},"outputs":[{"name":"stdout","text":"imports done!\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nMODEL_NAME = \"google/electra-base-discriminator\" # ← ELECTRA (was deberta-v3-base)\nMAX_LEN = 160\nBATCH_SIZE = 16 # ← ELECTRA is smaller → can use 16 (was 8)\nEPOCHS = 4\nLR = 2e-5\nlabel_cols = ['anger','fear','joy','sadness','surprise']\nnum_labels = len(label_cols)\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\ndef seed_everything(seed=42):\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\nseed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:17:18.160062Z","iopub.execute_input":"2025-11-29T09:17:18.160831Z","iopub.status.idle":"2025-11-29T09:17:18.545787Z","shell.execute_reply.started":"2025-11-29T09:17:18.160795Z","shell.execute_reply":"2025-11-29T09:17:18.544989Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"# ytrain, yval already exist from your earlier split in final-robert.ipynb\nytrain_arr = np.array(ytrain)\n\n# per-label pos_weight = (N_pos_reversed style) to upweight minority labels\nlabel_counts = ytrain_arr.sum(axis=0)\ntotal_counts = label_counts.sum()\npos_weight = (total_counts - label_counts) / (label_counts + 1e-6)\npos_weight = torch.tensor(pos_weight, dtype=torch.float32).to(DEVICE)\n\nprint(\"Label counts:\", dict(zip(label_cols, label_counts)))\nprint(\"pos_weight:\", pos_weight.cpu().numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:17:25.461721Z","iopub.execute_input":"2025-11-29T09:17:25.462292Z","iopub.status.idle":"2025-11-29T09:17:25.576818Z","shell.execute_reply.started":"2025-11-29T09:17:25.462264Z","shell.execute_reply":"2025-11-29T09:17:25.576178Z"}},"outputs":[{"name":"stdout","text":"Label counts: {'anger': 647, 'fear': 3102, 'joy': 1339, 'sadness': 1740, 'surprise': 1611}\npos_weight: [12.043277 1.7205029 5.3024645 3.85 4.2383614]\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"class EmotionDataset(Dataset):\n def __init__(self, texts, labels):\n self.texts = list(texts)\n self.labels = np.array(labels, dtype=np.float32)\n\n def __len__(self):\n return len(self.texts)\n\n def __getitem__(self, idx):\n text = str(self.texts[idx])\n enc = tokenizer(\n text,\n truncation=True,\n padding=\"max_length\",\n max_length=MAX_LEN,\n return_tensors=\"pt\"\n )\n item = {\n \"input_ids\": enc[\"input_ids\"].squeeze(0),\n \"attention_mask\": enc[\"attention_mask\"].squeeze(0),\n \"labels\": torch.tensor(self.labels[idx], dtype=torch.float32),\n }\n return item\n\ntrain_ds = EmotionDataset(xtrain, ytrain_arr)\nval_ds = EmotionDataset(xval, np.array(yval, dtype=np.float32))\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:17:42.748173Z","iopub.execute_input":"2025-11-29T09:17:42.748476Z","iopub.status.idle":"2025-11-29T09:17:42.756218Z","shell.execute_reply.started":"2025-11-29T09:17:42.748454Z","shell.execute_reply":"2025-11-29T09:17:42.755573Z"}},"outputs":[],"execution_count":14},{"cell_type":"code","source":"class RobertaClassifier(nn.Module):\n def __init__(self, model_name, num_labels, dropout=0.1):\n super().__init__()\n self.encoder = AutoModel.from_pretrained(model_name)\n hidden_size = self.encoder.config.hidden_size\n self.dropout = nn.Dropout(dropout)\n self.out = nn.Linear(hidden_size, num_labels)\n\n def forward(self, input_ids, attention_mask):\n outputs = self.encoder(\n input_ids=input_ids,\n attention_mask=attention_mask\n )\n pooled = outputs.last_hidden_state[:, 0] # CLS\n pooled = self.dropout(pooled)\n logits = self.out(pooled)\n return logits\n\nmodel = RobertaClassifier(MODEL_NAME, num_labels).to(DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:17:59.385842Z","iopub.execute_input":"2025-11-29T09:17:59.386401Z","iopub.status.idle":"2025-11-29T09:18:01.291517Z","shell.execute_reply.started":"2025-11-29T09:17:59.386381Z","shell.execute_reply":"2025-11-29T09:18:01.290634Z"}},"outputs":[],"execution_count":16},{"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR)\nnum_train_steps = len(train_loader) * EPOCHS\nnum_warmup_steps = int(0.1 * num_train_steps)\n\nscheduler = get_linear_schedule_with_warmup(\n optimizer,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_train_steps\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:18:05.682566Z","iopub.execute_input":"2025-11-29T09:18:05.683234Z","iopub.status.idle":"2025-11-29T09:18:05.688550Z","shell.execute_reply.started":"2025-11-29T09:18:05.683206Z","shell.execute_reply":"2025-11-29T09:18:05.687792Z"}},"outputs":[],"execution_count":17},{"cell_type":"code","source":"wandb.init(\n project=\"22f3001086-t32025\",\n name=\"final_electra_posweigh\",\n config={\n \"model_name\": MODEL_NAME,\n \"max_len\": MAX_LEN,\n \"batch_size\": BATCH_SIZE,\n \"epochs\": EPOCHS,\n \"lr\": LR,\n }\n)\nwandb.watch(model, log=\"all\", log_freq=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:18:11.365652Z","iopub.execute_input":"2025-11-29T09:18:11.365952Z","iopub.status.idle":"2025-11-29T09:18:18.077058Z","shell.execute_reply.started":"2025-11-29T09:18:11.365927Z","shell.execute_reply":"2025-11-29T09:18:18.076280Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.21.0"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20251129_091811-5nb8ycxa</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/5nb8ycxa' target=\"_blank\">final_electra_posweigh</a></strong> to <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/5nb8ycxa' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/5nb8ycxa</a>"},"metadata":{}}],"execution_count":18},{"cell_type":"code","source":"def train_one_epoch(epoch):\n model.train()\n total_loss = 0.0\n\n for step, batch in enumerate(tqdm(train_loader, desc=f\"Train epoch {epoch+1}\")):\n input_ids = batch[\"input_ids\"].to(DEVICE)\n attention_mask = batch[\"attention_mask\"].to(DEVICE)\n labels = batch[\"labels\"].to(DEVICE)\n\n optimizer.zero_grad()\n logits = model(input_ids=input_ids, attention_mask=attention_mask)\n loss = criterion(logits, labels)\n loss.backward()\n optimizer.step()\n scheduler.step()\n\n total_loss += loss.item()\n\n if (step + 1) % 50 == 0:\n avg_loss = total_loss / (step + 1)\n print(f\"Epoch {epoch+1} | Step {step+1}/{len(train_loader)} | Train loss: {avg_loss:.4f}\")\n wandb.log({\"train_step_loss\": avg_loss})\n\n avg_epoch_loss = total_loss / len(train_loader)\n return avg_epoch_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:18:44.406734Z","iopub.execute_input":"2025-11-29T09:18:44.407028Z","iopub.status.idle":"2025-11-29T09:18:44.414188Z","shell.execute_reply.started":"2025-11-29T09:18:44.407005Z","shell.execute_reply":"2025-11-29T09:18:44.413410Z"}},"outputs":[],"execution_count":19},{"cell_type":"code","source":"from sklearn.metrics import precision_score, recall_score\n\ndef validate(epoch, thresholds=None):\n model.eval()\n total_loss = 0.0\n all_labels = []\n all_probs = []\n\n with torch.no_grad():\n for batch in tqdm(val_loader, desc=f\"Valid epoch {epoch+1}\"):\n input_ids = batch[\"input_ids\"].to(DEVICE)\n attention_mask = batch[\"attention_mask\"].to(DEVICE)\n labels = batch[\"labels\"].to(DEVICE)\n\n logits = model(input_ids=input_ids, attention_mask=attention_mask)\n loss = criterion(logits, labels)\n\n probs = torch.sigmoid(logits).cpu().numpy()\n all_probs.append(probs)\n all_labels.append(labels.cpu().numpy())\n\n total_loss += loss.item()\n\n avg_loss = total_loss / len(val_loader)\n all_probs = np.concatenate(all_probs, axis=0)\n all_labels = np.concatenate(all_labels, axis=0)\n\n if thresholds is None:\n thresholds = np.full(num_labels, 0.5, dtype=float)\n\n preds = (all_probs >= thresholds[None, :]).astype(int)\n\n macro_f1 = f1_score(all_labels, preds, average=\"macro\")\n macro_precision = precision_score(all_labels, preds, average=\"macro\", zero_division=0)\n macro_recall = recall_score(all_labels, preds, average=\"macro\", zero_division=0)\n mean_support = all_labels.sum(axis=0).mean() # avg positives per label\n\n print(\n f\"Epoch {epoch+1} | Val loss: {avg_loss:.4f} | \"\n f\"Macro F1: {macro_f1:.4f} | Macro P: {macro_precision:.4f} | Macro R: {macro_recall:.4f}\"\n )\n\n wandb.log({\n \"val_loss\": avg_loss,\n \"val_macro_f1\": macro_f1,\n \"val_macro_precision\": macro_precision,\n \"val_macro_recall\": macro_recall,\n \"val_mean_support\": mean_support,\n })\n\n per_label_f1 = f1_score(all_labels, preds, average=None)\n for lbl, score in zip(label_cols, per_label_f1):\n wandb.log({f\"val_f1_{lbl}\": score})\n\n return avg_loss, macro_f1, all_probs, all_labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:18:53.638558Z","iopub.execute_input":"2025-11-29T09:18:53.639310Z","iopub.status.idle":"2025-11-29T09:18:53.652052Z","shell.execute_reply.started":"2025-11-29T09:18:53.639273Z","shell.execute_reply":"2025-11-29T09:18:53.651292Z"}},"outputs":[],"execution_count":20},{"cell_type":"code","source":"best_f1 = 0.0\nbest_state = None\n\nfor epoch in range(EPOCHS):\n print(f\"\\n=== Epoch {epoch+1}/{EPOCHS} ===\")\n train_loss = train_one_epoch(epoch)\n val_loss, val_f1, val_probs, val_labels = validate(epoch)\n \n wandb.log({\n \"train_epoch_loss\": train_loss,\n \"epoch\": epoch + 1\n })\n \n if val_f1 > best_f1:\n best_f1 = val_f1\n best_state = {\n \"model_state_dict\": model.state_dict(),\n \"model_name\": MODEL_NAME,\n \"label_cols\": label_cols\n }\n torch.save(best_state, \"electra_best.pt\")\n print(f\"✅ New best model saved! Macro F1 = {best_f1:.4f}\")\n \n # Save checkpoint every epoch to avoid losing progress\n torch.save({\n \"epoch\": epoch,\n \"model_state_dict\": model.state_dict(),\n \"optimizer_state_dict\": optimizer.state_dict(),\n \"best_f1\": best_f1\n }, \"checkpoint_e.pt\")\n \n print(f\"Best validation Macro F1 so far: {best_f1:.4f}\")\n\nprint(\"🏁 Training complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:19:30.705209Z","iopub.execute_input":"2025-11-29T09:19:30.705522Z","iopub.status.idle":"2025-11-29T09:26:39.289471Z","shell.execute_reply.started":"2025-11-29T09:19:30.705498Z","shell.execute_reply":"2025-11-29T09:26:39.288722Z"}},"outputs":[{"name":"stdout","text":"\n=== Epoch 1/4 ===\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 15%|█▍ | 50/342 [00:13<01:17, 3.77it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 50/342 | Train loss: 1.3079\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 29%|██▉ | 100/342 [00:29<04:07, 1.02s/it]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 100/342 | Train loss: 1.2819\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 44%|████▍ | 150/342 [00:42<00:50, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 150/342 | Train loss: 1.2530\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 58%|█████▊ | 200/342 [00:57<02:18, 1.03it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 200/342 | Train loss: 1.2100\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 73%|███████▎ | 250/342 [01:11<00:24, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 250/342 | Train loss: 1.1580\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 88%|████████▊ | 300/342 [01:25<00:32, 1.29it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Step 300/342 | Train loss: 1.1097\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 1: 100%|██████████| 342/342 [01:36<00:00, 3.53it/s]\nValid epoch 1: 100%|██████████| 86/86 [00:08<00:00, 10.20it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 1 | Val loss: 0.7273 | Macro F1: 0.6793 | Macro P: 0.5459 | Macro R: 0.9122\n✅ New best model saved! Macro F1 = 0.6793\nBest validation Macro F1 so far: 0.6793\n\n=== Epoch 2/4 ===\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 15%|█▍ | 50/342 [00:13<01:16, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 50/342 | Train loss: 0.6728\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 29%|██▉ | 100/342 [00:28<01:03, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 100/342 | Train loss: 0.6711\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 44%|████▍ | 150/342 [00:41<00:50, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 150/342 | Train loss: 0.6725\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 58%|█████▊ | 200/342 [00:55<00:37, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 200/342 | Train loss: 0.6576\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 73%|███████▎ | 250/342 [01:09<00:24, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 250/342 | Train loss: 0.6498\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 88%|████████▊ | 300/342 [01:23<00:11, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Step 300/342 | Train loss: 0.6427\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 2: 100%|██████████| 342/342 [01:34<00:00, 3.61it/s]\nValid epoch 2: 100%|██████████| 86/86 [00:08<00:00, 10.23it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 2 | Val loss: 0.5945 | Macro F1: 0.7577 | Macro P: 0.6549 | Macro R: 0.9010\n✅ New best model saved! Macro F1 = 0.7577\nBest validation Macro F1 so far: 0.7577\n\n=== Epoch 3/4 ===\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 15%|█▍ | 50/342 [00:14<01:29, 3.27it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 50/342 | Train loss: 0.4881\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 29%|██▉ | 100/342 [00:27<01:03, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 100/342 | Train loss: 0.4815\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 44%|████▍ | 150/342 [00:42<00:59, 3.25it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 150/342 | Train loss: 0.4691\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 58%|█████▊ | 200/342 [00:55<00:37, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 200/342 | Train loss: 0.4575\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 73%|███████▎ | 250/342 [01:10<00:28, 3.23it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 250/342 | Train loss: 0.4532\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 88%|████████▊ | 300/342 [01:24<00:11, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Step 300/342 | Train loss: 0.4436\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 3: 100%|██████████| 342/342 [01:35<00:00, 3.59it/s]\nValid epoch 3: 100%|██████████| 86/86 [00:08<00:00, 10.27it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 3 | Val loss: 0.5771 | Macro F1: 0.7633 | Macro P: 0.6575 | Macro R: 0.9190\n✅ New best model saved! Macro F1 = 0.7633\nBest validation Macro F1 so far: 0.7633\n\n=== Epoch 4/4 ===\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 15%|█▍ | 50/342 [00:14<01:16, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 50/342 | Train loss: 0.3770\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 29%|██▉ | 100/342 [00:27<01:03, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 100/342 | Train loss: 0.3695\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 44%|████▍ | 150/342 [00:42<00:50, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 150/342 | Train loss: 0.3648\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 58%|█████▊ | 200/342 [00:55<00:37, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 200/342 | Train loss: 0.3607\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 73%|███████▎ | 250/342 [01:10<00:24, 3.80it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 250/342 | Train loss: 0.3566\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 88%|████████▊ | 300/342 [01:24<00:11, 3.79it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Step 300/342 | Train loss: 0.3556\n","output_type":"stream"},{"name":"stderr","text":"Train epoch 4: 100%|██████████| 342/342 [01:36<00:00, 3.55it/s]\nValid epoch 4: 100%|██████████| 86/86 [00:08<00:00, 10.33it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 4 | Val loss: 0.5707 | Macro F1: 0.7909 | Macro P: 0.7057 | Macro R: 0.8997\n✅ New best model saved! Macro F1 = 0.7909\nBest validation Macro F1 so far: 0.7909\n🏁 Training complete!\n","output_type":"stream"}],"execution_count":21},{"cell_type":"markdown","source":"# Inferance","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import f1_score\n\n# Get validation predictions (run this after training completes)\ndef find_optimal_thresholds(val_probs, val_labels, label_cols, num_trials=100):\n \"\"\"Find per-label thresholds that maximize Macro F1\"\"\"\n best_f1 = 0.0\n best_thresholds = np.full(len(label_cols), 0.5)\n \n # Test thresholds from 0.1 to 0.9\n thresholds = np.linspace(0.1, 0.9, num_trials)\n \n for thresh_comb in thresholds:\n # Test same threshold for all labels first\n preds = (val_probs >= thresh_comb).astype(int)\n macro_f1 = f1_score(val_labels, preds, average='macro')\n \n if macro_f1 > best_f1:\n best_f1 = macro_f1\n best_thresholds = np.full(len(label_cols), thresh_comb)\n \n # Per-label optimization (more precise)\n for i, label in enumerate(label_cols):\n label_probs = val_probs[:, i]\n label_true = val_labels[:, i]\n \n label_f1s = []\n for thresh in thresholds:\n pred = (label_probs >= thresh).astype(int)\n f1 = f1_score(label_true, pred)\n label_f1s.append(f1)\n \n best_thresh_idx = np.argmax(label_f1s)\n best_thresholds[i] = thresholds[best_thresh_idx]\n \n final_preds = (val_probs >= best_thresholds).astype(int)\n final_macro_f1 = f1_score(val_labels, final_preds, average='macro')\n \n print(\"🎯 Optimal Thresholds Found:\")\n for i, (label, thresh) in enumerate(zip(label_cols, best_thresholds)):\n print(f\" {label}: {thresh:.3f}\")\n \n print(f\"\\n📊 Results Comparison:\")\n print(f\"Default (0.5): {f1_score(val_labels, (val_probs>=0.5).astype(int), average='macro'):.4f}\")\n print(f\"Optimized: {final_macro_f1:.4f} (+{final_macro_f1 - 0.8085:+.4f})\")\n \n return best_thresholds, final_macro_f1\n\n# Run optimization (uses val_probs, val_labels from your last epoch)\noptimal_thresholds, opt_f1 = find_optimal_thresholds(val_probs, val_labels, label_cols)\nwandb.log({\"val_macro_f1_optimized\": opt_f1})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:27:21.905010Z","iopub.execute_input":"2025-11-29T09:27:21.905653Z","iopub.status.idle":"2025-11-29T09:27:22.863877Z","shell.execute_reply.started":"2025-11-29T09:27:21.905627Z","shell.execute_reply":"2025-11-29T09:27:22.863018Z"}},"outputs":[{"name":"stdout","text":"🎯 Optimal Thresholds Found:\n anger: 0.852\n fear: 0.722\n joy: 0.884\n sadness: 0.682\n surprise: 0.730\n\n📊 Results Comparison:\nDefault (0.5): 0.7909\nOptimized: 0.8181 (++0.0096)\n","output_type":"stream"}],"execution_count":22},{"cell_type":"code","source":"berta_thresholds = [0.9, 0.561, 0.625, 0.795, 0.771]\nelectra_thresholds = [0.852, 0.722, 0.884, 0.682, 0.730]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test predictions with optimal thresholds\ntest_ds = EmotionDataset(dt['text'], np.zeros((len(dt), num_labels)))\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)\n\nall_test_probs = []\nmodel.eval()\nwith torch.no_grad():\n for batch in tqdm(test_loader, desc=\"Test inference\"):\n input_ids = batch[\"input_ids\"].to(DEVICE)\n attention_mask = batch[\"attention_mask\"].to(DEVICE)\n logits = model(input_ids, attention_mask)\n probs = torch.sigmoid(logits).cpu().numpy()\n all_test_probs.append(probs)\n\ntest_probs = np.concatenate(all_test_probs)\n\n# Apply optimal thresholds\nsubmission = dt[['id']].copy()\nfor i, col in enumerate(label_cols):\n submission[col] = (test_probs[:, i] >= optimal_thresholds[i]).astype(int)\n\nsubmission.to_csv(\"submission_electra_optimized.csv\", index=False)\nprint(\"✅ Optimized submission saved!\")\nprint(\"\\nFirst 5 predictions:\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:28:51.427565Z","iopub.execute_input":"2025-11-29T09:28:51.427828Z","iopub.status.idle":"2025-11-29T09:29:01.539738Z","shell.execute_reply.started":"2025-11-29T09:28:51.427796Z","shell.execute_reply":"2025-11-29T09:29:01.539182Z"}},"outputs":[{"name":"stderr","text":"Test inference: 100%|██████████| 107/107 [00:10<00:00, 10.61it/s]","output_type":"stream"},{"name":"stdout","text":"✅ Optimized submission saved!\n\nFirst 5 predictions:\n","output_type":"stream"},{"name":"stderr","text":"\n","output_type":"stream"},{"execution_count":24,"output_type":"execute_result","data":{"text/plain":" id anger fear joy sadness surprise\n0 0 1 1 0 0 0\n1 1 0 0 0 0 0\n2 2 1 1 0 0 0\n3 3 0 1 0 0 0\n4 4 0 1 0 0 1","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>id</th>\n <th>anger</th>\n <th>fear</th>\n <th>joy</th>\n <th>sadness</th>\n <th>surprise</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>0</td>\n <td>1</td>\n <td>1</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n </tr>\n <tr>\n <th>1</th>\n <td>1</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n </tr>\n <tr>\n <th>2</th>\n <td>2</td>\n <td>1</td>\n <td>1</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n </tr>\n <tr>\n <th>3</th>\n <td>3</td>\n <td>0</td>\n <td>1</td>\n <td>0</td>\n <td>0</td>\n <td>0</td>\n </tr>\n <tr>\n <th>4</th>\n <td>4</td>\n <td>0</td>\n <td>1</td>\n <td>0</td>\n <td>0</td>\n <td>1</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":24},{"cell_type":"code","source":"# 1. Delete current model completely\ndel model\ntorch.cuda.empty_cache()\n\n# 2. Recreate FRESH model with EXACT training architecture \nMODEL_NAME = \"google/electra-base-discriminator\" # Your DeBERTa training model or roberta-base\nmodel = RobertaClassifier(MODEL_NAME, num_labels=num_labels).to(DEVICE)\n\n# 3. Load with strict=False (ignores pooler mismatches)\nbest_state = torch.load(\"electra_best.pt\") # Your DeBERTa checkpoint\nmodel.load_state_dict(best_state[\"model_state_dict\"], strict=False)\nmodel.eval()\n\nprint(\"✅ Model loaded successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:39:19.984823Z","iopub.execute_input":"2025-11-29T09:39:19.985465Z","iopub.status.idle":"2025-11-29T09:39:21.363876Z","shell.execute_reply.started":"2025-11-29T09:39:19.985437Z","shell.execute_reply":"2025-11-29T09:39:21.363171Z"}},"outputs":[{"name":"stdout","text":"✅ Model loaded successfully!\n","output_type":"stream"}],"execution_count":25},{"cell_type":"code","source":"# DeBERTa test inference (same pattern)\nMODEL_NAME = \"google/electra-base-discriminator\"\nmodel = RobertaClassifier(MODEL_NAME, num_labels).to(DEVICE)\nbest_state = torch.load(\"electra_best.pt\")\nmodel.load_state_dict(best_state[\"model_state_dict\"], strict=False)\nmodel.eval()\n\ntest_ds = EmotionDataset(dt['text'], np.zeros((len(dt), num_labels)))\ntest_loader = DataLoader(test_ds, batch_size=8, shuffle=False) # DeBERTa batch=8\n\nall_test_probs = []\nwith torch.no_grad():\n for batch in tqdm(test_loader, desc=\"DeBERTa test\"):\n input_ids = batch[\"input_ids\"].to(DEVICE)\n attention_mask = batch[\"attention_mask\"].to(DEVICE)\n logits = model(input_ids, attention_mask)\n probs = torch.sigmoid(logits).cpu().numpy()\n all_test_probs.append(probs)\n\nnp.save('electra_test_probs.npy', np.concatenate(all_test_probs))\nprint(\"✅ ELECTRA probs saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:39:25.227253Z","iopub.execute_input":"2025-11-29T09:39:25.227793Z","iopub.status.idle":"2025-11-29T09:39:36.065199Z","shell.execute_reply.started":"2025-11-29T09:39:25.227765Z","shell.execute_reply":"2025-11-29T09:39:36.064487Z"}},"outputs":[{"name":"stderr","text":"DeBERTa test: 100%|██████████| 214/214 [00:09<00:00, 22.36it/s]","output_type":"stream"},{"name":"stdout","text":"✅ ELECTRA probs saved!\n","output_type":"stream"},{"name":"stderr","text":"\n","output_type":"stream"}],"execution_count":26},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n\n# # Load your submission\n# submission = pd.read_csv(\"/kaggle/working/submission_roberta_optimized.csv\")\n# label_cols = ['anger', 'fear', 'joy', 'sadness', 'surprise']\n\n# # Convert binary predictions to probability estimates\n# # High confidence for predicted labels, low for others\n# binary_preds = submission[label_cols].values\n# roberta_test_probs = np.zeros_like(binary_preds, dtype=np.float32)\n\n# # Map 0→0.25, 1→0.75 (good for ensemble weighting)\n# roberta_test_probs[binary_preds == 1] = 0.75\n# roberta_test_probs[binary_preds == 0] = 0.25\n\n# # Save for ensemble\n# np.save('roberta_test_probs.npy', roberta_test_probs)\n# print(\"✅ roberta_test_probs.npy created from submission.csv!\")\n# print(f\"Shape: {roberta_test_probs.shape}\")\n# print(f\"Sample:\\n{roberta_test_probs[:3]}\")\n# print(\"\\nFirst 5 rows match submission:\")\n# print(submission[label_cols].head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wandb.finish()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T09:39:43.327222Z","iopub.execute_input":"2025-11-29T09:39:43.327844Z","iopub.status.idle":"2025-11-29T09:39:45.235588Z","shell.execute_reply.started":"2025-11-29T09:39:43.327819Z","shell.execute_reply":"2025-11-29T09:39:45.234897Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<br> <style><br> .wandb-row {<br> display: flex;<br> flex-direction: row;<br> flex-wrap: wrap;<br> justify-content: flex-start;<br> width: 100%;<br> }<br> .wandb-col {<br> display: flex;<br> flex-direction: column;<br> flex-basis: 100%;<br> flex: 1;<br> padding: 10px;<br> }<br> </style><br><div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▃▆█</td></tr><tr><td>train_epoch_loss</td><td>█▄▂▁</td></tr><tr><td>train_step_loss</td><td>███▇▇▇▃▃▃▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁</td></tr><tr><td>val_f1_anger</td><td>▁▇▅█</td></tr><tr><td>val_f1_fear</td><td>▁▆▆█</td></tr><tr><td>val_f1_joy</td><td>▁▅██</td></tr><tr><td>val_f1_sadness</td><td>▁▆▇█</td></tr><tr><td>val_f1_surprise</td><td>▁▅▅█</td></tr><tr><td>val_loss</td><td>█▂▁▁</td></tr><tr><td>val_macro_f1</td><td>▁▆▆█</td></tr><tr><td>val_macro_f1_optimized</td><td>▁</td></tr><tr><td>val_macro_precision</td><td>▁▆▆█</td></tr><tr><td>val_macro_recall</td><td>▆▁█▁</td></tr><tr><td>val_mean_support</td><td>▁▁▁▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>4</td></tr><tr><td>train_epoch_loss</td><td>0.35532</td></tr><tr><td>train_step_loss</td><td>0.35556</td></tr><tr><td>val_f1_anger</td><td>0.71545</td></tr><tr><td>val_f1_fear</td><td>0.83245</td></tr><tr><td>val_f1_joy</td><td>0.80166</td></tr><tr><td>val_f1_sadness</td><td>0.78884</td></tr><tr><td>val_f1_surprise</td><td>0.81591</td></tr><tr><td>val_loss</td><td>0.5707</td></tr><tr><td>val_macro_f1</td><td>0.79086</td></tr><tr><td>val_macro_f1_optimized</td><td>0.8181</td></tr><tr><td>val_macro_precision</td><td>0.70574</td></tr><tr><td>val_macro_recall</td><td>0.89966</td></tr><tr><td>val_mean_support</td><td>411.79999</td></tr></table><br/></div></div>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run <strong style=\"color:#cdcd00\">final_electra_posweigh</strong> at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/5nb8ycxa' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/5nb8ycxa</a><br> View project at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Find logs at: <code>./wandb/run-20251129_091811-5nb8ycxa/logs</code>"},"metadata":{}}],"execution_count":27},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# 1. Load some CSV from the input directory (example path)\ninput_path = \"/kaggle/input/sub-files-dlgenai/submission_bbl_tuned.csv\"\ndf_sub = pd.read_csv(input_path)\n\n# Optional: quick sanity check\nprint(df_sub.head())\nprint(\"\\nColumns :\")\nprint(df_sub.columns)\n\n# 2. Save it as the competition submission file in the working/output dir\ndf_sub.to_csv(\"submission.csv\", index=False)\n\nprint(\"Done!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} |