Domain-Adaptation-NN(DANN).ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import statements"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import BertConfig, BertTokenizer, BertTokenizerFast, BertModel\nimport numpy as np\nimport math\nimport os\nimport pandas as pd\nimport torch.nn as nn","execution_count":1,"outputs":[{"output_type":"stream","text":"\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n","name":"stderr"}]},{"metadata":{},"cell_type":"markdown","source":"# Select device"},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":2,"outputs":[{"output_type":"stream","text":"cuda\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Parameters "},{"metadata":{"trusted":true},"cell_type":"code","source":"config = {\n    \"num_labels\": 2,\n    \"hidden_dropout_prob\": 0.15,\n    \"hidden_size\": 768,\n    \"max_length\": 512,\n}\n\ntraining_parameters = {\n    \"batch_size\": 2,\n    \"epochs\": 10,\n    \"output_folder\": \"../working/\",\n    \"output_file\": \"model.bin\",\n    \"learning_rate\": 1e-5,\n    \"print_after_steps\": 100,\n    \"save_steps\": 5000,\n\n}","execution_count":18,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Dataset and dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ReviewDataset(Dataset):\n    def __init__(self, df, model_name = \"bert-base-uncased\"):\n        self.df = df\n        self.config = BertConfig.from_pretrained(\"bert-base-uncased\", output_hidden_states=True)\n        self.tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\", do_lower_case=True)\n\n    def __getitem__(self, index):\n        review = self.df.iloc[index][\"review_text\"]\n        sentiment = self.df.iloc[index][\"sentiment\"]\n\n        encoded_input = self.tokenizer.encode_plus(\n                review,\n                add_special_tokens=True,\n                max_length= config[\"max_length\"],\n                pad_to_max_length=True,\n                return_overflowing_tokens=True,\n                truncation=True,\n                padding='max_length'\n            )\n\n        input_ids = encoded_input[\"input_ids\"]\n        attention_mask = encoded_input[\"attention_mask\"] if \"attention_mask\" in encoded_input else None\n        token_type_ids = encoded_input[\"token_type_ids\"] if \"token_type_ids\" in encoded_input else None\n\n\n\n        data_input = {\n            \"input_ids\": torch.tensor(input_ids),\n            \"attention_mask\": torch.tensor(attention_mask),\n            \"token_type_ids\": torch.tensor(token_type_ids),\n            \"label\": torch.tensor(sentiment, dtype=torch.long),\n        }\n\n        return data_input[\"input_ids\"], data_input[\"attention_mask\"], data_input[\"token_type_ids\"], data_input[\"label\"]\n\n\n\n    def __len__(self):\n        return self.df.shape[0]\n\n","execution_count":19,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root = \"../input/amazonproductsreview/amazon-review/\"\n# domains = [\"books\", \"dvd\", \"electronics\", \"kitchen_housewares\"]\nsource = \"books\"\ntarget = \"dvd\"","execution_count":20,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"source_df = pd.read_csv(root+source+\".csv\", nrows=1500, usecols=['review_text', 'sentiment'])\nsource_dataset = ReviewDataset(source_df)\nsource_dataloader = DataLoader(dataset = source_dataset, batch_size = training_parameters[\"batch_size\"], num_workers=4, shuffle = True)\n\ntarget_df = pd.read_csv(root+target+\".csv\", nrows=1500, usecols=['review_text', 'sentiment'])\ntarget_dataset = ReviewDataset(target_df)\ntarget_dataloader = DataLoader(dataset = target_dataset, batch_size = training_parameters[\"batch_size\"], num_workers=4, shuffle = False)","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(source_df.shape, target_df.shape)","execution_count":22,"outputs":[{"output_type":"stream","text":"(1500, 2) (1500, 2)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# source_df.head(20)","execution_count":23,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Gradient Reversal Function "},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.autograd import Function\n\n\nclass GradientReversalFn(Function):\n    @staticmethod\n    def forward(ctx, x, alpha):\n        ctx.alpha = alpha\n        \n        return x.view_as(x)\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        output = grad_output.neg() * ctx.alpha\n\n        return output, None\n","execution_count":24,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass DomainAdaptationModel(nn.Module):\n    def __init__(self):\n        super(DomainAdaptationModel, self).__init__()\n        \n        num_labels = config[\"num_labels\"]\n        self.bert = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)\n        self.dropout = nn.Dropout(config[\"hidden_dropout_prob\"])\n        self.sentiment_classifier = nn.Sequential(\n            nn.Linear(config[\"hidden_size\"], num_labels),\n            nn.LogSoftmax(dim=1),\n        )\n        self.domain_classifier = nn.Sequential(\n            nn.Linear(config[\"hidden_size\"], 2),\n            nn.LogSoftmax(dim=1),\n        )\n\n\n    def forward(\n          self,\n          input_ids=None,\n          attention_mask=None,\n          token_type_ids=None,\n          labels=None,\n          grl_lambda = 1.0, \n          ):\n\n        top_layer, pooled_output, layers = self.bert(\n                input_ids,\n                attention_mask=attention_mask,\n                token_type_ids=token_type_ids,\n            )\n\n#         top_layer, pooled, layers = self.base(input_ids, attention_mask)\n#         pooled_output = outputs[1]\n        pooled_output = self.dropout(pooled_output)\n\n        reversed_pooled_output = GradientReversalFn.apply(pooled_output, grl_lambda)\n\n        sentiment_pred = self.sentiment_classifier(pooled_output)\n        domain_pred = self.domain_classifier(reversed_pooled_output)\n        \n        return layers, pooled_output, sentiment_pred.to(device), domain_pred.to(device)","execution_count":25,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Computer Accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_accuracy(logits, labels):\n    \n    predicted_labels_dict = {\n      0: 0,\n      1: 0,\n    }\n    \n    predicted_label = logits.max(dim = 1)[1]\n    \n    for pred in predicted_label:\n        predicted_labels_dict[pred.item()] += 1\n    acc = (predicted_label == labels).float().mean()\n    \n    return acc, predicted_labels_dict","execution_count":26,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(model, dataset = \"electronics\", percentage = 100):\n    with torch.no_grad():\n        predicted_labels_dict = {                                                   \n          0: 0,                                                                     \n          1: 0,                                                                     \n        }\n        \n        dev_df = pd.read_csv(root+dataset+\".csv\")\n        dev_df = dev_df[1500:]\n        dev_df = dev_df.reset_index(drop=True)\n#         data_size = dev_df.shape[0]\n#         selected_for_evaluation = int(data_size*percentage/100)\n#         dev_df = dev_df.head(selected_for_evaluation)\n                             \n#         dev_df = dev_df.iloc[1500:, :]\n        dataset = ReviewDataset(dev_df)\n\n        dataloader = DataLoader(dataset = dataset, batch_size = training_parameters[\"batch_size\"], shuffle = False, num_workers=4)\n\n        mean_accuracy = 0.0\n        total_batches = len(dataloader)\n        \n        for input_ids, attention_mask, token_type_ids, labels in dataloader:\n            inputs = {\n                \"input_ids\": input_ids,\n                \"attention_mask\": attention_mask,\n                \"token_type_ids\" : token_type_ids,\n                \"labels\": labels,\n            }\n            for k, v in inputs.items():\n                inputs[k] = v.to(device)\n\n\n            _, _, sentiment_pred, _ = model(**inputs)\n            accuracy, predicted_labels = compute_accuracy(sentiment_pred, inputs[\"labels\"])\n            mean_accuracy += accuracy\n            predicted_labels_dict[0] += predicted_labels[0]\n            predicted_labels_dict[1] += predicted_labels[1]  \n#         print(predicted_labels_dict)\n    return mean_accuracy/total_batches","execution_count":27,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training "},{"metadata":{"trusted":true},"cell_type":"code","source":"target_acc = []\nmodel = DomainAdaptationModel().to(device)","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = training_parameters[\"learning_rate\"]\nn_epochs = training_parameters[\"epochs\"]\n\n\noptimizer = optim.Adam(model.parameters(), lr)\n\nloss_fn_sentiment_classifier = torch.nn.NLLLoss()\nloss_fn_domain_classifier = torch.nn.NLLLoss()\n'''\nIn one training step we will update the model using both the source labeled data and target unlabeled data\nWe will run it till the batches last for any of these datasets\n\nIn our case target dataset has more data. Hence, we will leverage the entire source dataset for training\n\nIf we use the same approach in a case where the source dataset has more data then the target dataset then we will\nunder-utilize the labeled source dataset. In such a scenario it is better to reload the target dataset when it finishes\nThis will ensure that we are utilizing the entire source dataset to train our model.\n'''\n\nmax_batches = min(len(source_dataloader), len(target_dataloader))\n\nfor epoch_idx in range(n_epochs):\n    \n    source_iterator = iter(source_dataloader)\n    target_iterator = iter(target_dataloader)\n\n    for batch_idx in range(max_batches):\n        \n        p = float(batch_idx + epoch_idx * max_batches) / (training_parameters[\"epochs\"] * max_batches)\n        grl_lambda = 2. / (1. + np.exp(-10 * p)) - 1\n        grl_lambda = torch.tensor(grl_lambda)\n        \n        model.train()\n        \n        if(batch_idx%training_parameters[\"print_after_steps\"] == 0 ):\n            print(\"Training Step:\", batch_idx)\n        \n        optimizer.zero_grad()\n        \n        # Souce dataset training update\n        input_ids, attention_mask, token_type_ids, labels = next(source_iterator)\n        inputs = {\n            \"input_ids\": input_ids,\n            \"attention_mask\": attention_mask,\n            \"token_type_ids\" : token_type_ids,\n            \"labels\" : labels,\n            \"grl_lambda\" : grl_lambda,\n        }\n\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        _, _, sentiment_pred, domain_pred = model(**inputs)\n        loss_s_sentiment = loss_fn_sentiment_classifier(sentiment_pred, inputs[\"labels\"])\n        y_s_domain = torch.zeros(training_parameters[\"batch_size\"], dtype=torch.long).to(device)\n        loss_s_domain = loss_fn_domain_classifier(domain_pred, y_s_domain)\n\n\n        # Target dataset training update \n        input_ids, attention_mask, token_type_ids, labels = next(target_iterator)\n        inputs = {\n            \"input_ids\": input_ids,\n            \"attention_mask\": attention_mask,\n            \"token_type_ids\" : token_type_ids,\n            \"labels\" : labels,\n            \"grl_lambda\" : grl_lambda,\n        }\n\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n    \n        _, _, _, domain_pred = model(**inputs)\n        \n        # Note that we are not using the sentiment predictions here for updating the weights\n        y_t_domain = torch.ones(training_parameters[\"batch_size\"], dtype=torch.long).to(device)\n        loss_t_domain = loss_fn_domain_classifier(domain_pred, y_t_domain)\n\n        # Combining the loss \n\n        loss = loss_s_sentiment + loss_s_domain + loss_t_domain\n        loss.backward()\n        optimizer.step()\n\n    # Evaluate the model after every epoch\n    \n    \n    accuracy = evaluate(model, dataset = source, percentage = 1).item()\n    print(f'Accuracy on {source} domain after epoch {epoch_idx} = {accuracy}')\n#     print(\"Accuracy on amazon after epoch \" + str(epoch_idx) + \" is \" + str(accuracy))\n\n    accuracy = evaluate(model, dataset = target, percentage = 100).item()\n    if len(target_acc)>0 and accuracy + 0.001 > target_acc[-1]:\n        torch.save(model.state_dict(), os.path.join(training_parameters[\"output_folder\"], \"_\"+str(accuracy)[:4]+\"_\"+training_parameters[\"output_file\"]))\n        target_acc.append(accuracy)\n    elif len(target_acc)<=0:\n        torch.save(model.state_dict(), os.path.join(training_parameters[\"output_folder\"], \"_\"+str(accuracy)[:4]+\"_\"+training_parameters[\"output_file\"] ))\n        target_acc.append(accuracy)\n    print(f'Accuracy on {target} domain after epoch {epoch_idx} = {accuracy}')\n  ","execution_count":null,"outputs":[{"output_type":"stream","text":"Training Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\nTraining Step: 700\nAccuracy on books domain after epoch 0 = 0.8360655307769775\nAccuracy on dvd domain after epoch 0 = 0.7573221325874329\nTraining Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\nTraining Step: 700\nAccuracy on books domain after epoch 2 = 0.7827868461608887\nAccuracy on dvd domain after epoch 2 = 0.6652719378471375\nTraining Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\nTraining Step: 700\nAccuracy on books domain after epoch 3 = 0.8442622423171997\nAccuracy on dvd domain after epoch 3 = 0.6297070980072021\nTraining Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\nTraining Step: 700\nAccuracy on books domain after epoch 4 = 0.8442622423171997\nAccuracy on dvd domain after epoch 4 = 0.5292887091636658\nTraining Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\nTraining Step: 700\nAccuracy on books domain after epoch 5 = 0.8442622423171997\nAccuracy on dvd domain after epoch 5 = 0.8075313568115234\nTraining Step: 0\nTraining Step: 100\nTraining Step: 200\nTraining Step: 300\nTraining Step: 400\nTraining Step: 500\nTraining Step: 600\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate the model on the entire dev set at the end"},{"metadata":{"trusted":true},"cell_type":"code","source":"domain = \"books\"\naccuracy = evaluate(model, dataset = domain, percentage = 100).item()  \nprint(f'Accuracy on {domain} domain = {accuracy}')\n\ndomain = \"dvd\"\naccuracy = evaluate(model, dataset = domain, percentage = 100).item()  \nprint(f'Accuracy on {domain} domain = {accuracy}')\n\n# domain = \"electronics\"\n# accuracy = evaluate(model, dataset = domain, percentage = 50).item()  \n# print(f'Accuracy on {domain} domain = {accuracy}')\n\n# domain = \"kitchen_housewares\"\n# accuracy = evaluate(model, dataset = domain, percentage = 50).item()  \n# print(f'Accuracy on {domain} domain = {accuracy}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# target_acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = DomainAdaptationModel()\nmodel.to(device)\nmodel.load_state_dict(torch.load(\"../working/_1.0_model.bin\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"domain = \"books\"\naccuracy = evaluate(model, dataset = domain, percentage = 100).item()  \nprint(f'Accuracy on {domain} domain = {accuracy}')\n\ndomain = \"dvd\"\naccuracy = evaluate(model, dataset = domain, percentage = 100).item()  \nprint(f'Accuracy on {domain} domain = {accuracy}')\n\n# domain = \"electronics\"\n# accuracy = evaluate(model, dataset = domain, percentage = 50).item()  \n# print(f'Accuracy on {domain} domain = {accuracy}')\n\n# domain = \"kitchen_housewares\"\n# accuracy = evaluate(model, dataset = domain, percentage = 50).item()  \n# print(f'Accuracy on {domain} domain = {accuracy}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}