Spaces:
Sleeping
Sleeping
File size: 48,106 Bytes
092944c |
1 |
{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":115439,"databundleVersionId":13800781,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# 5cfcb5e8ef8458be6e85d57c45c7573477e2ad6a\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:19:09.852220Z","iopub.execute_input":"2025-11-30T09:19:09.853064Z","iopub.status.idle":"2025-11-30T09:19:09.857013Z","shell.execute_reply.started":"2025-11-30T09:19:09.853036Z","shell.execute_reply":"2025-11-30T09:19:09.856302Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/train.csv\") #training set\ndt = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/test.csv\") #test set","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:19:14.477003Z","iopub.execute_input":"2025-11-30T09:19:14.477287Z","iopub.status.idle":"2025-11-30T09:19:14.521481Z","shell.execute_reply.started":"2025-11-30T09:19:14.477265Z","shell.execute_reply":"2025-11-30T09:19:14.520946Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# wdb_t = user_secrets.get_secret(\"WB_TOKEN\")\n# import wandb\n# wandb.login(key=wdb_t)\n# # wandb.init(project=\"22f3001086-t32025\", name = \"BERT+Classifier head\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:19:17.653917Z","iopub.execute_input":"2025-11-30T09:19:17.654456Z","iopub.status.idle":"2025-11-30T09:19:27.824215Z","shell.execute_reply.started":"2025-11-30T09:19:17.654432Z","shell.execute_reply":"2025-11-30T09:19:27.823579Z"}},"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: No netrc file found, creating one.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mvaishnavib\u001b[0m (\u001b[33mvaishnavib-iitm-jntuh-\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n","output_type":"stream"},{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}],"execution_count":4},{"cell_type":"code","source":"# Imports and device setup\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.optim import AdamW\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForMaskedLM, get_linear_schedule_with_warmup\nfrom sklearn.metrics import f1_score\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import train_test_split # Used for splitting later\nimport gc\nfrom torch.cuda.amp import autocast, GradScaler\nfrom nltk.corpus import wordnet\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:20:31.398860Z","iopub.execute_input":"2025-11-30T09:20:31.399496Z","iopub.status.idle":"2025-11-30T09:20:31.910556Z","shell.execute_reply.started":"2025-11-30T09:20:31.399473Z","shell.execute_reply":"2025-11-30T09:20:31.909993Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"## Data Loading, Preparation, and Configuration (Combined from Original Cells 2 & 12)\n\n# Load data\ndf = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/train.csv\")\ndt = pd.read_csv(\"/kaggle/input/2025-sep-dl-gen-ai-project/test.csv\")\n\n# Ensure 'disgust' exists and is 0 (Important for 6-label backbone)\ndf = df.drop(columns=[\"emotions\"])\ndf = df.sample(frac=1, random_state=42).reset_index(drop=True)\nif 'disgust' not in df.columns:\n df['disgust'] = 0\n\nbase_label_cols = ['anger','fear','joy','sadness','surprise']\nlabel_cols = base_label_cols + ['disgust'] # 6 for backbone\n\nX = df['text'].values\ny = df[label_cols].values\n\n# Splitting data (Replaced missing iterative_train_test_split with standard train_test_split)\n# Using 80/20 split as implied by the original code's intent (test_size=0.2)\nX_train, X_val, y_train, y_val = train_test_split(\n X, y, test_size=0.2, random_state=42, shuffle=True\n) \n\ntrain_texts = X_train\nval_texts = X_val\ntrain_labels = y_train\nval_labels = y_val\ntest_texts = dt['text'].values\n\n# Configuration Constants (adjusted MAX_LEN_TAPT/FT to match usage later)\nMODEL_NAME = \"AnkitAI/deberta-v3-small-base-emotions-classifier\"\nMAX_LEN_TAPT = 64 # Short for TAPT to save memory\nMAX_LEN_FT = 128 # Used for Fine-Tuning\nBATCH_SIZE = 4 # Used for Fine-Tuning","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:22:09.870011Z","iopub.execute_input":"2025-11-30T09:22:09.870323Z","iopub.status.idle":"2025-11-30T09:22:09.910105Z","shell.execute_reply.started":"2025-11-30T09:22:09.870303Z","shell.execute_reply":"2025-11-30T09:22:09.909359Z"}},"outputs":[],"execution_count":10},{"cell_type":"code","source":"## Data Augmentation Functions (Original Cell 13)\n\ndef synonym_replacement(text, n=1):\n words = text.split()\n new_words = words.copy()\n # Find words with synonyms\n random_word_list = list(set([word for word in words if wordnet.synsets(word)]))\n random.shuffle(random_word_list)\n num_replaced = 0\n for random_word in random_word_list:\n synonyms = wordnet.synsets(random_word)\n if len(synonyms) >= 1:\n # Replace with the first lemma/synonym\n synonym = synonyms[0].lemmas()[0].name()\n new_words = [synonym if word == random_word else word for word in new_words]\n num_replaced += 1\n if num_replaced >= n:\n break\n return ' '.join(new_words)\n\ndef random_deletion(text, p=0.1):\n words = text.split()\n if len(words) == 1:\n return text\n new_words = [word for word in words if random.uniform(0,1) > p]\n if len(new_words) == 0:\n new_words = [random.choice(words)] # ensure not empty\n return ' '.join(new_words)\n\ndef augment(text):\n if random.random() < 0.3:\n text = synonym_replacement(text, n=1)\n if random.random() < 0.3:\n text = random_deletion(text, p=0.1)\n return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:22:23.029996Z","iopub.execute_input":"2025-11-30T09:22:23.030847Z","iopub.status.idle":"2025-11-30T09:22:23.037935Z","shell.execute_reply.started":"2025-11-30T09:22:23.030819Z","shell.execute_reply":"2025-11-30T09:22:23.037232Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"## TAPT - Text-Adaptive Pre-training (Original Cell 15)\n\n# ---------- ULTRA-LIGHT TAPT (MLM, SAFE) ----------\nmlm_epochs = 1\nmlm_lr = 5e-5\nBATCH_SIZE_TAPT = 4\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\nclass MLMDataset(Dataset):\n def __init__(self, texts, tokenizer, max_len):\n self.texts = texts\n self.tok = tokenizer\n self.max_len = max_len\n def __len__(self):\n return len(self.texts)\n def __getitem__(self, idx):\n enc = self.tok(\n self.texts[idx],\n truncation=True,\n padding='max_length',\n max_length=self.max_len,\n return_tensors='pt'\n )\n item = {k: v.squeeze(0) for k, v in enc.items()}\n return item\n\nmlm_dataset = MLMDataset(train_texts, tokenizer, MAX_LEN_TAPT)\nmlm_loader = DataLoader(mlm_dataset, batch_size=BATCH_SIZE_TAPT, shuffle=True)\n\nmlm_model = AutoModelForMaskedLM.from_pretrained(MODEL_NAME).to(DEVICE)\nmlm_model.config.use_cache = False # small memory help\n\nmlm_optimizer = AdamW(mlm_model.parameters(), lr=mlm_lr)\nmlm_total_steps = len(mlm_loader) * mlm_epochs\nmlm_scheduler = get_linear_schedule_with_warmup(\n mlm_optimizer, num_warmup_steps=100, num_training_steps=mlm_total_steps\n)\n\n# WANDB setup for TAPT (reinit=True to start a new run)\nimport wandb\nwandb.init(project=\"22f3001086-t32025\", name=\"TAPT-MLM-safe\", reinit=True)\nwandb.config.update({\n \"phase\": \"TAPT\",\n \"epochs\": mlm_epochs,\n \"batch_size\": BATCH_SIZE_TAPT,\n \"lr\": mlm_lr,\n \"max_len\": MAX_LEN_TAPT\n})\n\nmlm_model.train()\nfor epoch in range(mlm_epochs):\n total_loss = 0.0\n for batch in mlm_loader:\n batch = {k: v.to(DEVICE) for k, v in batch.items()}\n mlm_optimizer.zero_grad()\n # Use input_ids as labels for MLM\n outputs = mlm_model(**batch, labels=batch[\"input_ids\"]) \n loss = outputs.loss\n loss.backward()\n mlm_optimizer.step()\n mlm_scheduler.step()\n total_loss += loss.item()\n wandb.log({\"tapt/train_loss\": loss.item()})\n print(f\"TAPT Epoch {epoch+1}, Loss {total_loss/len(mlm_loader):.4f}\")\n\nmlm_model.save_pretrained(\"tapt_deberta_emotions\")\ntokenizer.save_pretrained(\"tapt_deberta_emotions\")\nwandb.finish()\n\ndel mlm_model, mlm_dataset, mlm_loader\ntorch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:22:36.319716Z","iopub.execute_input":"2025-11-30T09:22:36.320034Z","iopub.status.idle":"2025-11-30T09:25:12.312595Z","shell.execute_reply.started":"2025-11-30T09:22:36.320011Z","shell.execute_reply":"2025-11-30T09:25:12.311999Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"20fe5ec0086240568b90e95f44681927"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"spm.model: 0%| | 0.00/2.46M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"512830ea52b74f1ba37d61b37354d1f5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7a624b1765064757af31940d4c204ec8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"added_tokens.json: 0%| | 0.00/23.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b9a0e183f7544f6db7f34552ee06831a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json: 0%| | 0.00/286 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e0d88df614bb4728b4b9e734dce62221"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8e9421b7e7ea4019b73d07079ba35a97"}},"metadata":{}},{"name":"stderr","text":"2025-11-30 09:22:43.136711: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\nWARNING: All log messages before absl::InitializeLog() is called are written to STDERR\nE0000 00:00:1764494563.376405 47 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\nE0000 00:00:1764494563.441990 47 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;31mAttributeError\u001b[0m: 'MessageFactory' object has no attribute 'GetPrototype'"],"ename":"AttributeError","evalue":"'MessageFactory' object has no attribute 'GetPrototype'","output_type":"error"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;31mAttributeError\u001b[0m: 'MessageFactory' object has no attribute 'GetPrototype'"],"ename":"AttributeError","evalue":"'MessageFactory' object has no attribute 'GetPrototype'","output_type":"error"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;31mAttributeError\u001b[0m: 'MessageFactory' object has no attribute 'GetPrototype'"],"ename":"AttributeError","evalue":"'MessageFactory' object has no attribute 'GetPrototype'","output_type":"error"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;31mAttributeError\u001b[0m: 'MessageFactory' object has no attribute 'GetPrototype'"],"ename":"AttributeError","evalue":"'MessageFactory' object has no attribute 'GetPrototype'","output_type":"error"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;31mAttributeError\u001b[0m: 'MessageFactory' object has no attribute 'GetPrototype'"],"ename":"AttributeError","evalue":"'MessageFactory' object has no attribute 'GetPrototype'","output_type":"error"},{"output_type":"display_data","data":{"text/plain":"model.safetensors: 0%| | 0.00/568M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"885eb4d830fd43d19acef5e213927c60"}},"metadata":{}},{"name":"stderr","text":"Some weights of DebertaV2ForMaskedLM were not initialized from the model checkpoint at AnkitAI/deberta-v3-small-base-emotions-classifier and are newly initialized: ['cls.predictions.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.21.0"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20251130_092310-d6p0d7fu</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/d6p0d7fu' target=\"_blank\">TAPT-MLM-safe</a></strong> to <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/d6p0d7fu' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/d6p0d7fu</a>"},"metadata":{}},{"name":"stdout","text":"TAPT Epoch 1, Loss 1.1689\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<br> <style><br> .wandb-row {<br> display: flex;<br> flex-direction: row;<br> flex-wrap: wrap;<br> justify-content: flex-start;<br> width: 100%;<br> }<br> .wandb-col {<br> display: flex;<br> flex-direction: column;<br> flex-basis: 100%;<br> flex: 1;<br> padding: 10px;<br> }<br> </style><br><div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>tapt/train_loss</td><td>█▇▅▅▅▂▂▂▂▂▁▁▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>tapt/train_loss</td><td>0.02918</td></tr></table><br/></div></div>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run <strong style=\"color:#cdcd00\">TAPT-MLM-safe</strong> at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/d6p0d7fu' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/d6p0d7fu</a><br> View project at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Find logs at: <code>./wandb/run-20251130_092310-d6p0d7fu/logs</code>"},"metadata":{}}],"execution_count":12},{"cell_type":"code","source":"## Define Weighted BCE + Exclusivity Loss (Original Cell 20)\n\n# Weights are already calculated in the combined data setup cell, but recalculated here for safety/modularity\npos_weights = []\nfor i, col in enumerate(label_cols):\n pos = train_labels[:, i].sum()\n neg = len(train_labels) - pos\n w = 1.0 if pos == 0 else neg / pos\n pos_weights.append(torch.tensor(w))\npos_weights = torch.stack(pos_weights).float().to(DEVICE)\n\nclass WeightedBCEWithExclusivityLoss(nn.Module):\n def __init__(self, class_weights, lambda_excl=0.1):\n super().__init__()\n # nn.BCEWithLogitsLoss expects pos_weight for class imbalance\n self.bce = nn.BCEWithLogitsLoss(pos_weight=class_weights) \n self.lambda_excl = lambda_excl\n\n def forward(self, outputs, targets):\n logits = outputs\n probs = torch.sigmoid(logits)\n bce_loss = self.bce(logits, targets)\n # Exclusivity penalty: difference between the sum of predicted probabilities\n # and the sum of true labels (encourages predicting only the necessary number of labels)\n penalty = ((probs.sum(1) - targets.sum(1)).abs()).mean()\n loss = bce_loss + self.lambda_excl * penalty\n return loss\n\ncriterion = WeightedBCEWithExclusivityLoss(pos_weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:25:26.143038Z","iopub.execute_input":"2025-11-30T09:25:26.143757Z","iopub.status.idle":"2025-11-30T09:25:26.219047Z","shell.execute_reply.started":"2025-11-30T09:25:26.143733Z","shell.execute_reply":"2025-11-30T09:25:26.218416Z"}},"outputs":[],"execution_count":13},{"cell_type":"code","source":"## Load TAPT Checkpoint and Setup DataLoaders for Fine-Tuning (Combined from Original Cells 16, 19)\n\n# Reload tokenizer and classification model starting from TAPT weights\ntokenizer = AutoTokenizer.from_pretrained(\"tapt_deberta_emotions\")\nmodel = AutoModelForSequenceClassification.from_pretrained(\n \"tapt_deberta_emotions\",\n num_labels=len(label_cols)\n).to(DEVICE)\n\nMAX_LEN = MAX_LEN_FT # Using 128 for Fine-Tuning\n\nclass EmotionDataset(Dataset):\n def __init__(self, texts, labels=None, tokenizer=None, max_len=MAX_LEN, augment=False):\n self.texts = texts\n self.labels = labels\n self.tokenizer = tokenizer\n self.max_len = max_len\n self.augment = augment\n\n def __len__(self):\n return len(self.texts)\n\n def __getitem__(self, idx):\n text = self.texts[idx]\n if self.augment:\n text = augment(text)\n enc = self.tokenizer(\n text,\n truncation=True,\n padding=\"max_length\",\n max_length=self.max_len,\n return_tensors=\"pt\",\n )\n item = {k: v.squeeze(0) for k, v in enc.items()}\n if self.labels is not None:\n label = torch.tensor(self.labels[idx]).float()\n return item, label\n else:\n return item\n\ntrain_dataset = EmotionDataset(train_texts, train_labels, tokenizer, MAX_LEN, augment=True)\nval_dataset = EmotionDataset(val_texts, val_labels, tokenizer, MAX_LEN, augment=False)\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:25:35.994729Z","iopub.execute_input":"2025-11-30T09:25:35.995408Z","iopub.status.idle":"2025-11-30T09:25:36.758238Z","shell.execute_reply.started":"2025-11-30T09:25:35.995382Z","shell.execute_reply":"2025-11-30T09:25:36.757649Z"}},"outputs":[{"name":"stderr","text":"Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at tapt_deberta_emotions and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"}],"execution_count":14},{"cell_type":"code","source":"import gc\nfrom torch.cuda.amp import autocast, GradScaler\n\n# Free up memory before training\ngc.collect()\ntorch.cuda.empty_cache()\n\ndef train_one_epoch(model, loader, optimizer, scheduler, scaler, accumulation_steps):\n model.train()\n total_loss = 0\n optimizer.zero_grad()\n for step, (inputs, labels) in enumerate(loader):\n inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n labels = labels.to(DEVICE)\n\n with autocast():\n logits = model(**inputs).logits\n loss = criterion(logits, labels)\n loss = loss / accumulation_steps\n\n scaler.scale(loss).backward()\n\n # Gradient accumulation step\n if (step + 1) % accumulation_steps == 0:\n scaler.step(optimizer)\n scaler.update()\n optimizer.zero_grad()\n scheduler.step()\n\n total_loss += loss.item() * accumulation_steps\n\n # Handle remaining gradients if batches not divisible by accumulation_steps\n if len(loader) % accumulation_steps != 0:\n # Only step if gradients exist\n if torch.is_grad_enabled():\n scaler.step(optimizer)\n scaler.update()\n optimizer.zero_grad()\n scheduler.step()\n\n avg_loss = total_loss / len(loader)\n return avg_loss\n\ndef eval_model(model, loader):\n model.eval()\n all_probs = []\n all_labels = []\n with torch.no_grad():\n for inputs, labels in loader:\n inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n outputs = model(**inputs).logits\n probs = torch.sigmoid(outputs).cpu().numpy()\n all_probs.append(probs)\n all_labels.append(labels.numpy())\n\n all_probs = np.vstack(all_probs)\n all_labels = np.vstack(all_labels)\n preds_bin = (all_probs > 0.5).astype(int)\n macro_f1 = f1_score(all_labels[:, :5], preds_bin[:, :5], average=\"macro\")\n return macro_f1, all_probs, all_labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T09:37:12.353883Z","iopub.execute_input":"2025-11-30T09:37:12.354336Z","iopub.status.idle":"2025-11-30T09:37:13.008716Z","shell.execute_reply.started":"2025-11-30T09:37:12.354307Z","shell.execute_reply":"2025-11-30T09:37:13.008118Z"}},"outputs":[],"execution_count":19},{"cell_type":"code","source":"## Full Training Loop for Multiple Seeds (Original Cell 25)\n\nEPOCHS = 8\nLEARNING_RATE = 3e-5\nGRAD_ACCUM_STEPS = 4\nseeds = [42, 7, 2026]\n\ndef train_loop(seed):\n # Set seed for reproducibility\n torch.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n \n # Reload model from TAPT checkpoint\n model = AutoModelForSequenceClassification.from_pretrained(\n \"tapt_deberta_emotions\", num_labels=len(label_cols)\n ).to(DEVICE)\n \n optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)\n # Calculate total steps for scheduler\n total_steps = len(train_loader) // GRAD_ACCUM_STEPS * EPOCHS\n scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=200, num_training_steps=total_steps)\n scaler = GradScaler()\n \n best_f1 = 0\n no_improve = 0\n patience = 1\n \n # WANDB setup for Fine-Tuning\n import wandb\n wandb.init(project=\"22f3001086-t32025\", name=f\"FT-Deberta-Seed{seed}\", reinit=True)\n wandb.config.update({\n \"phase\": \"Fine-Tuning\",\n \"seed\": seed,\n \"epochs\": EPOCHS,\n \"lr\": LEARNING_RATE,\n \"accum_steps\": GRAD_ACCUM_STEPS,\n \"max_len\": MAX_LEN\n })\n \n for epoch in range(EPOCHS):\n train_loss = train_one_epoch(model, train_loader, optimizer, scheduler, scaler, GRAD_ACCUM_STEPS)\n val_f1, val_probs, val_labels_curr = eval_model(model, val_loader)\n \n wandb.log({\"train/loss\": train_loss, \"val/macro_f1\": val_f1, \"epoch\": epoch + 1})\n \n print(f\"Seed {seed} Epoch {epoch + 1} - Train Loss: {train_loss:.4f}, Val F1: {val_f1:.4f}\")\n \n if val_f1 > best_f1:\n best_f1 = val_f1\n no_improve = 0\n # Save the best model state dict for ensembling\n torch.save(model.state_dict(), f\"best_model_seed{seed}.pt\") \n else:\n no_improve += 1\n if no_improve > patience:\n print(\"Early stopping.\")\n break\n \n # Load the best model weights for the current seed\n model.load_state_dict(torch.load(f\"best_model_seed{seed}.pt\", map_location=DEVICE))\n model.eval()\n wandb.finish()\n \n # Recalculate validation probabilities using the best weights\n val_f1, val_probs, val_labels_curr = eval_model(model, val_loader) \n \n return model, val_probs, val_labels_curr\n\nmodels = []\nval_probs_list = []\nval_labels_list = []\n\nfor sd in seeds:\n model, val_probs, val_labels_curr = train_loop(sd)\n models.append(model)\n val_probs_list.append(val_probs)\n # The labels should be the same across runs, but keeping the check for safety\n val_labels_list.append(val_labels_curr) \n\n# Take the labels from the first run (they should all be the same)\nval_labels = val_labels_list[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T10:33:38.540284Z","iopub.execute_input":"2025-11-30T10:33:38.540624Z"}},"outputs":[{"name":"stderr","text":"Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at tapt_deberta_emotions and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Finishing previous runs because reinit is set to True."},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run <strong style=\"color:#cdcd00\">FT-Deberta-Seed7</strong> at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/dw6clzle' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/dw6clzle</a><br> View project at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Find logs at: <code>./wandb/run-20251130_103229-dw6clzle/logs</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.21.0"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20251130_103338-krqjyc0r</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/krqjyc0r' target=\"_blank\">FT-Deberta-Seed42</a></strong> to <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/krqjyc0r' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/krqjyc0r</a>"},"metadata":{}},{"name":"stdout","text":"Seed 42 Epoch 1 - Train Loss: 0.8718, Val F1: 0.5713\nSeed 42 Epoch 2 - Train Loss: 0.6149, Val F1: 0.7145\nSeed 42 Epoch 3 - Train Loss: 0.4542, Val F1: 0.7586\nSeed 42 Epoch 4 - Train Loss: 0.3543, Val F1: 0.7818\nSeed 42 Epoch 5 - Train Loss: 0.2786, Val F1: 0.7990\nSeed 42 Epoch 6 - Train Loss: 0.2296, Val F1: 0.8010\nSeed 42 Epoch 7 - Train Loss: 0.1995, Val F1: 0.8087\nSeed 42 Epoch 8 - Train Loss: 0.1822, Val F1: 0.8116\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<br> <style><br> .wandb-row {<br> display: flex;<br> flex-direction: row;<br> flex-wrap: wrap;<br> justify-content: flex-start;<br> width: 100%;<br> }<br> .wandb-col {<br> display: flex;<br> flex-direction: column;<br> flex-basis: 100%;<br> flex: 1;<br> padding: 10px;<br> }<br> </style><br><div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▂▃▄▅▆▇█</td></tr><tr><td>train/loss</td><td>█▅▄▃▂▁▁▁</td></tr><tr><td>val/macro_f1</td><td>▁▅▆▇████</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>8</td></tr><tr><td>train/loss</td><td>0.1822</td></tr><tr><td>val/macro_f1</td><td>0.81157</td></tr></table><br/></div></div>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run <strong style=\"color:#cdcd00\">FT-Deberta-Seed42</strong> at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/krqjyc0r' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/krqjyc0r</a><br> View project at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Find logs at: <code>./wandb/run-20251130_103338-krqjyc0r/logs</code>"},"metadata":{}},{"name":"stderr","text":"Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at tapt_deberta_emotions and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.21.0"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20251130_104431-86qf6y9n</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/86qf6y9n' target=\"_blank\">FT-Deberta-Seed7</a></strong> to <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/86qf6y9n' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/86qf6y9n</a>"},"metadata":{}},{"name":"stdout","text":"Seed 7 Epoch 1 - Train Loss: 0.8642, Val F1: 0.6557\nSeed 7 Epoch 2 - Train Loss: 0.6061, Val F1: 0.7098\nSeed 7 Epoch 3 - Train Loss: 0.4608, Val F1: 0.7463\nSeed 7 Epoch 4 - Train Loss: 0.3539, Val F1: 0.7778\nSeed 7 Epoch 5 - Train Loss: 0.2834, Val F1: 0.7997\nSeed 7 Epoch 6 - Train Loss: 0.2320, Val F1: 0.8036\nSeed 7 Epoch 7 - Train Loss: 0.2057, Val F1: 0.8101\nSeed 7 Epoch 8 - Train Loss: 0.1835, Val F1: 0.8084\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<br> <style><br> .wandb-row {<br> display: flex;<br> flex-direction: row;<br> flex-wrap: wrap;<br> justify-content: flex-start;<br> width: 100%;<br> }<br> .wandb-col {<br> display: flex;<br> flex-direction: column;<br> flex-basis: 100%;<br> flex: 1;<br> padding: 10px;<br> }<br> </style><br><div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▂▃▄▅▆▇█</td></tr><tr><td>train/loss</td><td>█▅▄▃▂▁▁▁</td></tr><tr><td>val/macro_f1</td><td>▁▃▅▇████</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>8</td></tr><tr><td>train/loss</td><td>0.18348</td></tr><tr><td>val/macro_f1</td><td>0.80841</td></tr></table><br/></div></div>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run <strong style=\"color:#cdcd00\">FT-Deberta-Seed7</strong> at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/86qf6y9n' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/86qf6y9n</a><br> View project at: <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Find logs at: <code>./wandb/run-20251130_104431-86qf6y9n/logs</code>"},"metadata":{}},{"name":"stderr","text":"Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at tapt_deberta_emotions and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.21.0"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20251130_105522-971m1n1n</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/971m1n1n' target=\"_blank\">FT-Deberta-Seed2026</a></strong> to <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/971m1n1n' target=\"_blank\">https://wandb.ai/vaishnavib-iitm-jntuh-/22f3001086-t32025/runs/971m1n1n</a>"},"metadata":{}},{"name":"stdout","text":"Seed 2026 Epoch 1 - Train Loss: 0.8685, Val F1: 0.6247\nSeed 2026 Epoch 2 - Train Loss: 0.6143, Val F1: 0.7144\nSeed 2026 Epoch 3 - Train Loss: 0.4554, Val F1: 0.7619\nSeed 2026 Epoch 4 - Train Loss: 0.3600, Val F1: 0.7841\nSeed 2026 Epoch 5 - Train Loss: 0.2907, Val F1: 0.7973\n","output_type":"stream"}],"execution_count":null},{"cell_type":"code","source":"## Ensembling and Temperature Scaling (Combined from Original Cells 26, 27)\n\n# --- Collect Logits for Ensembling ---\n\ndef collect_logits(model, loader):\n logits_list = []\n with torch.no_grad():\n # Iterate over validation loader with both inputs and labels\n for inputs, _ in loader: \n inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n logits = model(**inputs).logits\n logits_list.append(logits.cpu())\n return torch.cat(logits_list, dim=0)\n\n# Ensemble logits for validation set (average across seeds)\nval_logits_ensemble = torch.zeros(len(val_texts), len(label_cols))\nfor m in models:\n val_logits_ensemble += collect_logits(m, val_loader).cpu()\nval_logits_ensemble /= len(models)\nval_logits_ensemble = val_logits_ensemble.to(DEVICE)\nval_labels_tensor = torch.tensor(val_labels).to(DEVICE).float()\n\n# --- Temperature Scaling ---\n\nclass ModelWithTemperature(nn.Module):\n def __init__(self):\n super().__init__()\n self.temperature = nn.Parameter(torch.ones(1) * 1.0)\n def forward(self, logits):\n return logits / self.temperature\n\ntemp_model = ModelWithTemperature().to(DEVICE)\noptimizer_t = AdamW(temp_model.parameters(), lr=0.01)\n\n# Optimize temperature\nfor _ in range(200):\n optimizer_t.zero_grad()\n scaled_logits = temp_model(val_logits_ensemble)\n # Use BCEWithLogitsLoss (or equivalent) for calibration\n loss_t = nn.functional.binary_cross_entropy_with_logits(scaled_logits, val_labels_tensor) \n loss_t.backward()\n optimizer_t.step()\n\nT = temp_model.temperature.item()\nprint(f\"Optimal temperature: {T:.4f}\")\n\n# Function to convert logits to calibrated probabilities\ndef logits_to_probs(logits, T):\n return torch.sigmoid(logits / T)\n\n# Apply temperature to ensemble logits\nval_probs_cal = logits_to_probs(val_logits_ensemble, T).cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T11:22:56.533312Z","iopub.execute_input":"2025-11-30T11:22:56.534087Z","iopub.status.idle":"2025-11-30T11:23:12.682210Z","shell.execute_reply.started":"2025-11-30T11:22:56.534060Z","shell.execute_reply":"2025-11-30T11:23:12.681392Z"}},"outputs":[{"name":"stdout","text":"Optimal temperature: 1.3561\n","output_type":"stream"}],"execution_count":23},{"cell_type":"code","source":"## Per-label Threshold Optimization (Original Cell 29)\n\ndef optimize_thresholds(y_true, y_probs, n_labels=5):\n # Search space for thresholds\n thresholds = np.linspace(0.05, 0.89, 85) \n best_t = np.full(n_labels, 0.5)\n for i in range(n_labels):\n best_f1 = 0.0\n # Iterate through possible thresholds for label i\n for t in thresholds: \n preds = (y_probs[:, i] > t).astype(int)\n f1 = f1_score(y_true[:, i], preds)\n if f1 > best_f1:\n best_f1 = f1\n best_t[i] = t\n return best_t\n\n# Optimize on the first 5 labels (excluding 'disgust')\noptimal_thresholds = optimize_thresholds(val_labels[:, :5], val_probs_cal[:, :5], n_labels=5) \nprint(f\"Optimal thresholds (anger, fear, joy, sadness, surprise): {optimal_thresholds}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T11:23:23.557243Z","iopub.execute_input":"2025-11-30T11:23:23.558080Z","iopub.status.idle":"2025-11-30T11:23:24.220262Z","shell.execute_reply.started":"2025-11-30T11:23:23.558038Z","shell.execute_reply":"2025-11-30T11:23:24.219426Z"}},"outputs":[{"name":"stdout","text":"Optimal thresholds (anger, fear, joy, sadness, surprise): [0.87 0.43 0.57 0.64 0.3 ]\n","output_type":"stream"}],"execution_count":24},{"cell_type":"markdown","source":"Optimal temperature: 1.6917624473571777\nOptimal thresholds: [0.46 0.44 0.74 0.6 0.6 ]\n\nOptimal thresholds (anger, fear, joy, sadness, surprise): [0.87 0.43 0.57 0.64 0.3 ]","metadata":{}},{"cell_type":"code","source":"## Test Data Preparation (Original Cell 30)\n\ntest_df = dt # Use dt (loaded test data)\ntest_texts = test_df[\"text\"].tolist()\n\nclass TestDataset(Dataset):\n def __init__(self, texts, tokenizer, max_len=MAX_LEN):\n self.texts = texts\n self.tokenizer = tokenizer\n self.max_len = max_len\n def __len__(self):\n return len(self.texts)\n def __getitem__(self, idx):\n enc = self.tokenizer(\n self.texts[idx],\n truncation=True,\n padding=\"max_length\",\n max_length=self.max_len,\n return_tensors=\"pt\",\n )\n return {k: v.squeeze(0) for k, v in enc.items()}\n\ntest_dataset = TestDataset(test_texts, tokenizer)\n# Use the same batch size as training/validation\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T11:28:59.042895Z","iopub.execute_input":"2025-11-30T11:28:59.043633Z","iopub.status.idle":"2025-11-30T11:28:59.049606Z","shell.execute_reply.started":"2025-11-30T11:28:59.043609Z","shell.execute_reply":"2025-11-30T11:28:59.048882Z"}},"outputs":[],"execution_count":28},{"cell_type":"code","source":"# --- Inference on test set ---\ntest_logits_ensemble = []\n\nwith torch.no_grad():\n for inputs in test_loader:\n inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n logits_sum = 0\n for m in models:\n logits_sum += m(**inputs).logits\n logits_mean = logits_sum / len(models)\n test_logits_ensemble.append(logits_mean.cpu())\n\ntest_logits_ensemble = torch.cat(test_logits_ensemble, dim=0)\n\n# --- Apply Calibration and Thresholds ---\ntest_probs_cal = logits_to_probs(test_logits_ensemble.to(DEVICE), T).cpu().numpy()\n\n# Initialize binary predictions\ntest_preds_bin = np.zeros_like(test_probs_cal, dtype=int)\n# The optimal_thresholds array only has 5 values, so we use a default for 'disgust' (index 5)\nthresholds_for_inference = np.append(optimal_thresholds, 0.5)\n\n# Apply thresholds to all 6 columns\nfor i in range(len(label_cols)):\n test_preds_bin[:, i] = (test_probs_cal[:, i] > thresholds_for_inference[i]).astype(int)\n\n# --- Prepare and save submission ---\nsubmission_data = {\n \"id\": test_df[\"id\"],\n # Only include the 5 required emotions in the submission\n **{label: test_preds_bin[:, i] for i, label in enumerate(base_label_cols)}\n}\n\nsubmission = pd.DataFrame(submission_data)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T11:29:47.692295Z","iopub.execute_input":"2025-11-30T11:29:47.692870Z","iopub.status.idle":"2025-11-30T11:30:05.525324Z","shell.execute_reply.started":"2025-11-30T11:29:47.692844Z","shell.execute_reply":"2025-11-30T11:30:05.524544Z"}},"outputs":[],"execution_count":29}]} |