Low GPU Utilization

[Sakurakdx]

When I use the pytorch-lightning framework, the GPU utilization is very low.
I want to know what is the reason for this.

[aniketmaurya]

Hi @Sakurakdx, could you provide a reproducible code and which version of PL are you using right now? This will help us in finding the issue. Also there is a similar discussion that you can refer to - #10670

[Sakurakdx]

Sure.

I use DataModule to process data.

class DInterface(pl.LightningDataModule):
def __init__(self, config):
        super().__init__()
        self.train_batch_size = config.train_batch_size
        self.dev_batch_size = config.dev_batch_size
        self.num_works = config.num_works if config.num_works else 8

        self.token_helper = BertTokenHelper(config.bert_dir)
        config.bert_embedding_size = len(self.token_helper.tokenizer)
        self.config = config
        self.prepare_data()

    def prepare_data(self):
        logger.info("prepare_data")
        self.train_instances = read_corpus(self.config.train_file)
        logger.info(f"load train data from {self.config.train_file} sucessfully")

        self.dev_instances = read_corpus(self.config.dev_file)
        logger.info(f"load dev data from {self.config.dev_file} sucessfully")

        if self.config.test:
            self.test_instances = read_corpus(self.config.test_file)
            logger.info(f"load test data from {self.config.test_file} sucessfully")

    def setup(self, stage=None):
        if stage == "fit" or stage is None:
            self.train_set = MyDataset(self.config, self.train_instances)
            self.train_set.convert_one_to_features(self.token_helper)
            self.train_set.label2id(self.vocab)

            self.dev_set = MyDataset(self.config, self.dev_instances)
            self.dev_set.convert_one_to_features(self.token_helper)
            self.dev_set.label2id(self.vocab)

        if stage == "test" or stage is None:
            self.test_set = MyDataset(self.config, self.test_instances)
            self.test_set.convert_one_to_features(self.token_helper)
            self.test_set.label2id(self.vocab)

    def train_dataloader(self):
        return DataLoader(
            self.train_set,
            batch_size=self.train_batch_size,
            shuffle=True,
            collate_fn=train_wrapper,
            num_workers=self.num_works,
        ) 
    ...

My model as follows:

class MInterface(pl.LightningModule):
    def __init__(self, config, vocab) -> None:
        super().__init__()
        self.vocab = vocab
        self.save_hyperparameters(config.__dict__)
        self.init_model()


    def configure_optimizers(self):
        plm_param = list(self.enc.bert_extractor.parameters())

        seg_param = \
        list(self.enc.mlp_words.parameters()) + \
        list(self.enc.rescale.parameters()) + \
        list(self.dec.parameters())
        bert_opt = Adam(self.parameters(), lr=1e-3)
        model_param = [{
            'params': plm_param,
            'lr': self.hparams.plm_learning_rate
        }, {
            'params': seg_param,
            'lr': self.hparams.learning_rate
        }]
        optim = torch.optim.Adam(model_param,
                                 lr=self.hparams.learning_rate,
                                 betas=(self.hparams.beta_1,
                                        self.hparams.beta_2),
                                 eps=self.hparams.epsilon,
                                 weight_decay=self.hparams.l2_reg)
        decay, decay_step = self.hparams.decay, self.hparams.decay_steps
        l = lambda epoch: decay**(epoch // decay_step)
        scheduler = torch.optim.lr_scheduler.LambdaLR(optim, lr_lambda=l)

        return [optim], [scheduler]

    def training_step(self, batch, batch_idx):
        _, inputs, token_indexs, denominators, gold_labels = batch
        self.encode(inputs, token_indexs, denominators)  ## bert encoder

        loss = self.compute_loss(gold_labels)

        acc  = self.compute_accuracy()

        self.log("train loss", loss)
        self.log("train acc", acc)

        return {"loss": loss, "acc": acc}

    def training_epoch_end(self, outputs):
        outputs_num = len(outputs)
        epoch_loss = 0
        epoch_acc = 0
        for idx, item in enumerate(outputs):
            epoch_loss += item["loss"]
            epoch_acc += item['acc']

        logger.info(
            f"epoch {self.trainer.current_epoch:03}, train epoch loss: {epoch_loss / outputs_num:5.4f}, train epoch acc: {epoch_acc / outputs_num:5.4f}"
        )

    def validation_step(self, batch, batch_idx):
        batch_data, inputs, token_indexs, denominators = batch
        self.encode(inputs, token_indexs, denominators)   ## bert encoder
        self.decode(batch_data, self.vocab)  ## sequence labeling decoder

        return batch_data

    def validation_epoch_end(self, outputs):
        if outputs == []:
            logger.error("outputs is None")
            return None
        instances = [inst for insts in outputs for inst in insts]
        outputFile = os.path.join(
            self.hparams.save_dev_dir,
            self.hparams.dev_file.split("/")[-1] + "_" +
            str(self.trainer.current_epoch))

        with open(outputFile, mode="w", encoding="utf-8") as outf:
            for i, inst in enumerate(instances):
                if i > 0 and inst.first_line != "":
                    outf.write("\n")
                outf.write(inst.first_line + "\n")

                for idx, line in enumerate(inst.sentence_info):
                    if inst.pred_labels[idx] == "b" or idx == 0:
                        pred_label = "BeginSeg=Yes"
                    elif inst.pred_labels[idx] == "i":
                        pred_label = "_"
                    info = line.strip().split("\t")
                    info[9] = pred_label
                    predict_line = "\t".join(info)
                    outf.write(predict_line + "\n")

        score_dict = get_scores(self.hparams.dev_file, outputFile)
        dev_p = score_dict["prec"]
        dev_r = score_dict["rec"]
        dev_f1 = score_dict["f_score"]

        self.log("dev P", dev_p)
        self.log("dev R", dev_r)
        self.log("dev F1 score", dev_f1)

        logger.info(
            f"Dev: P: {dev_p:5.4f}, R: {dev_r:5.4f}, F1 score: {dev_f1: 5.4f}")

    def init_model(self):
        enc_model = BertExtractor(self.hparams)
        # logger.info(enc_model)
        logger.info('Load pretrained encoder ok')

        self.enc = GlobalEncoder(self.vocab, self.hparams, enc_model)
        self.dec = Decoder(self.vocab, self.hparams)

        dump_pkl(self.vocab, self.hparams.save_vocab_path)

Main function:

data_module = DInterface(config)
model = MInterface(config, data_module.vocab)

trainer = pl.Trainer.from_argparse_args(config)
trainer.fit(model, data_module, ckpt_path=config.ckpt_path)

[aniketmaurya]

Thanks for the code @Sakurakdx. Here are a few things that you can do to find the reason for low GPU usage-

• Try increasing batch size.
• Check if num_workers in torch Dataloaders are properly set
• Find the bottleneck in your code using Profiler

[TimaGitHub]

Hi! Have you solved this problem?