Can't parse arguments (deep learning tutorial using pytorch)

Question

I'm following this tutorial.

The first scripts run fine and I have a "data" folder in the folder of my scripts containing the MRI data downloaded from MRnet.

However when it comes to the "train" script I get an error. Here's the full script and the error (using jupyter notebook):

import shutil
import os
import time
from datetime import datetime
import argparse
import numpy as np
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from torchsample.transforms import RandomRotate, RandomTranslate, RandomFlip, ToTensor, Compose, RandomAffine
from torchvision import transforms
import torch.nn.functional as F
from tensorboardX import SummaryWriter
import nbimporter

from dataloader import MRDataset
import model

from sklearn import metrics


def train_model(model, train_loader, epoch, num_epochs, optimizer, writer, current_lr, log_every=100):
    _ = model.train()

    if torch.cuda.is_available():
        model.cuda()

    y_preds = []
    y_trues = []
    losses = []

    for i, (image, label, weight) in enumerate(train_loader):
        optimizer.zero_grad()

        if torch.cuda.is_available():
            image = image.cuda()
            label = label.cuda()
            weight = weight.cuda()

        label = label[0]
        weight = weight[0]

        prediction = model.forward(image.float())

        loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label)
        loss.backward()
        optimizer.step()

        loss_value = loss.item()
        losses.append(loss_value)

        probas = torch.sigmoid(prediction)

        y_trues.append(int(label[0][1]))
        y_preds.append(probas[0][1].item())

        try:
            auc = metrics.roc_auc_score(y_trues, y_preds)
        except:
            auc = 0.5

        writer.add_scalar('Train/Loss', loss_value,
                          epoch * len(train_loader) + i)
        writer.add_scalar('Train/AUC', auc, epoch * len(train_loader) + i)

        if (i % log_every == 0) & (i > 0):
            print('''[Epoch: {0} / {1} |Single batch number : {2} / {3} ]| avg train loss {4} | train auc : {5} | lr : {6}'''.
                  format(
                      epoch + 1,
                      num_epochs,
                      i,
                      len(train_loader),
                      np.round(np.mean(losses), 4),
                      np.round(auc, 4),
                      current_lr
                  )
                  )

    writer.add_scalar('Train/AUC_epoch', auc, epoch + i)

    train_loss_epoch = np.round(np.mean(losses), 4)
    train_auc_epoch = np.round(auc, 4)
    return train_loss_epoch, train_auc_epoch


def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr, log_every=20):
    _ = model.eval()

    if torch.cuda.is_available():
        model.cuda()

    y_trues = []
    y_preds = []
    losses = []

    for i, (image, label, weight) in enumerate(val_loader):

        if torch.cuda.is_available():
            image = image.cuda()
            label = label.cuda()
            weight = weight.cuda()

        label = label[0]
        weight = weight[0]

        prediction = model.forward(image.float())

        loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label)

        loss_value = loss.item()
        losses.append(loss_value)

        probas = torch.sigmoid(prediction)

        y_trues.append(int(label[0][1]))
        y_preds.append(probas[0][1].item())

        try:
            auc = metrics.roc_auc_score(y_trues, y_preds)
        except:
            auc = 0.5

        writer.add_scalar('Val/Loss', loss_value, epoch * len(val_loader) + i)
        writer.add_scalar('Val/AUC', auc, epoch * len(val_loader) + i)

        if (i % log_every == 0) & (i > 0):
            print('''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | avg val loss {4} | val auc : {5} | lr : {6}'''.
                  format(
                      epoch + 1,
                      num_epochs,
                      i,
                      len(val_loader),
                      np.round(np.mean(losses), 4),
                      np.round(auc, 4),
                      current_lr
                  )
                  )

    writer.add_scalar('Val/AUC_epoch', auc, epoch + i)

    val_loss_epoch = np.round(np.mean(losses), 4)
    val_auc_epoch = np.round(auc, 4)
    return val_loss_epoch, val_auc_epoch

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']


def run(args):
    log_root_folder = "./logs/{0}/{1}/".format(args.task, args.plane)
    if args.flush_history == 1:
        objects = os.listdir(log_root_folder)
        for f in objects:
            if os.path.isdir(log_root_folder + f):
                shutil.rmtree(log_root_folder + f)

    now = datetime.now()
    logdir = log_root_folder + now.strftime("%Y%m%d-%H%M%S") + "/"
    os.makedirs(logdir)

    writer = SummaryWriter(logdir)

    augmentor = Compose([
        transforms.Lambda(lambda x: torch.Tensor(x)),
        RandomRotate(25),
        RandomTranslate([0.11, 0.11]),
        RandomFlip(),
        transforms.Lambda(lambda x: x.repeat(3, 1, 1, 1).permute(1, 0, 2, 3)),
    ])

    train_dataset = MRDataset('./data/', args.task,
                              args.plane, transform=augmentor, train=True)
    train_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=1, shuffle=True, num_workers=11, drop_last=False)

    validation_dataset = MRDataset(
        './data/', args.task, args.plane, train=False)
    validation_loader = torch.utils.data.DataLoader(
        validation_dataset, batch_size=1, shuffle=-True, num_workers=11, drop_last=False)

    mrnet = model.MRNet()

    if torch.cuda.is_available():
        mrnet = mrnet.cuda()

    optimizer = optim.Adam(mrnet.parameters(), lr=args.lr, weight_decay=0.1)

    if args.lr_scheduler == "plateau":
        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            optimizer, patience=3, factor=.3, threshold=1e-4, verbose=True)
    elif args.lr_scheduler == "step":
        scheduler = torch.optim.lr_scheduler.StepLR(
            optimizer, step_size=3, gamma=args.gamma)

    best_val_loss = float('inf')
    best_val_auc = float(0)

    num_epochs = args.epochs
    iteration_change_loss = 0
    patience = args.patience
    log_every = args.log_every

    t_start_training = time.time()

    for epoch in range(num_epochs):
        current_lr = get_lr(optimizer)

        t_start = time.time()
        
        train_loss, train_auc = train_model(
            mrnet, train_loader, epoch, num_epochs, optimizer, writer, current_lr, log_every)
        val_loss, val_auc = evaluate_model(
            mrnet, validation_loader, epoch, num_epochs, writer, current_lr)

        if args.lr_scheduler == 'plateau':
            scheduler.step(val_loss)
        elif args.lr_scheduler == 'step':
            scheduler.step()

        t_end = time.time()
        delta = t_end - t_start

        print("train loss : {0} | train auc {1} | val loss {2} | val auc {3} | elapsed time {4} s".format(
            train_loss, train_auc, val_loss, val_auc, delta))

        iteration_change_loss += 1
        print('-' * 30)

        if val_auc > best_val_auc:
            best_val_auc = val_auc
            if bool(args.save_model):
                file_name = f'model_{args.prefix_name}_{args.task}_{args.plane}_val_auc_{val_auc:0.4f}_train_auc_{train_auc:0.4f}_epoch_{epoch+1}.pth'
                for f in os.listdir('./models/'):
                    if (args.task in f) and (args.plane in f) and (args.prefix_name in f):
                        os.remove(f'./models/{f}')
                torch.save(mrnet, f'./models/{file_name}')

        if val_loss < best_val_loss:
            best_val_loss = val_loss
            iteration_change_loss = 0

        if iteration_change_loss == patience:
            print('Early stopping after {0} iterations without the decrease of the val loss'.
                  format(iteration_change_loss))
            break

    t_end_training = time.time()
    print(f'training took {t_end_training - t_start_training} s')


def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-t', '--task', type=str, required=True,
                        choices=['abnormal', 'acl', 'meniscus'])
    parser.add_argument('-p', '--plane', type=str, required=True,
                        choices=['sagittal', 'coronal', 'axial'])
    parser.add_argument('--prefix_name', type=str, required=True)
    parser.add_argument('--augment', type=int, choices=[0, 1], default=1)
    parser.add_argument('--lr_scheduler', type=str,
                        default='plateau', choices=['plateau', 'step'])
    parser.add_argument('--gamma', type=float, default=0.5)
    parser.add_argument('--epochs', type=int, default=50)
    parser.add_argument('--lr', type=float, default=1e-5)
    parser.add_argument('--flush_history', type=int, choices=[0, 1], default=0)
    parser.add_argument('--save_model', type=int, choices=[0, 1], default=1)
    parser.add_argument('--patience', type=int, default=5)
    parser.add_argument('--log_every', type=int, default=100)
    args = parser.parse_args()
    return args


if __name__ == "__main__":
    args = parse_arguments()
    run(args)

Error:

usage: ipykernel_launcher.py [-h] -t {abnormal,acl,meniscus} -p
                             {sagittal,coronal,axial} --prefix_name
                             PREFIX_NAME [--augment {0,1}]
                             [--lr_scheduler {plateau,step}] [--gamma GAMMA]
                             [--epochs EPOCHS] [--lr LR]
                             [--flush_history {0,1}] [--save_model {0,1}]
                             [--patience PATIENCE] [--log_every LOG_EVERY]
ipykernel_launcher.py: error: the following arguments are required: -t/--task, -p/--plane, --prefix_name

%tb:

    ---------------------------------------------------------------------------
SystemExit                                Traceback (most recent call last)
<ipython-input-3-e6a34ab63dc0> in <module>
    275 
    276 if __name__ == "__main__":
--> 277     args = parse_arguments()
    278     run(args)

<ipython-input-3-e6a34ab63dc0> in parse_arguments()
    270     parser.add_argument('--patience', type=int, default=5)
    271     parser.add_argument('--log_every', type=int, default=100)
--> 272     args = parser.parse_args()
    273     return args
    274 

~\anaconda3\envs\Pytorch\lib\argparse.py in parse_args(self, args, namespace)
   1753     # =====================================
   1754     def parse_args(self, args=None, namespace=None):
-> 1755         args, argv = self.parse_known_args(args, namespace)
   1756         if argv:
   1757             msg = _('unrecognized arguments: %s')

~\anaconda3\envs\Pytorch\lib\argparse.py in parse_known_args(self, args, namespace)
   1785         # parse the arguments and exit if there are any errors
   1786         try:
-> 1787             namespace, args = self._parse_known_args(args, namespace)
   1788             if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
   1789                 args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))

~\anaconda3\envs\Pytorch\lib\argparse.py in _parse_known_args(self, arg_strings, namespace)
   2020         if required_actions:
   2021             self.error(_('the following arguments are required: %s') %
-> 2022                        ', '.join(required_actions))
   2023 
   2024         # make sure all required groups had one option present

~\anaconda3\envs\Pytorch\lib\argparse.py in error(self, message)
   2506         self.print_usage(_sys.stderr)
   2507         args = {'prog': self.prog, 'message': message}
-> 2508         self.exit(2, _('%(prog)s: error: %(message)s\n') % args)

~\anaconda3\envs\Pytorch\lib\argparse.py in exit(self, status, message)
   2493         if message:
   2494             self._print_message(message, _sys.stderr)
-> 2495         _sys.exit(status)
   2496 
   2497     def error(self, message):

SystemExit: 2

I have no clue on how to move forwards. I'm stranded here. Anyone know where to go from here?

Answer 1

I will guess.

ArgumentParser was created to get arguments when you run it in console/terminal not Juputer

python script.py -t abnormal -p axial --prefix_name abc

and Python puts these arguments as list in sys.argv and ArgumentParser uses automatically values from sys.argv in parser.parse_args()

If you want to run it in juputer then you have to send arguments manually as list

args = parser.parse_args( ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"] )

or you have to append argument to sys.argv

sys.argv.append("-t")
sys.argv.append("abnormal")
sys.argv.append("-p")
sys.argv.append("axial")
sys.argv.append("--prefix_name")
sys.argv.append("abc")

or using .extend( list )

sys.argv.extend( ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"] )

or using string with split(" ")

sys.argv.extend( "-t abnormal -p axial --prefix_name abc".split(' ') )

but if you run it in Jupyter many times with different arguments then it will remeber all arguments and you will need to remove previous arguments

sys.argv.clear()
sys.argv.extend( ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"] )

or replace all elements (except first which sometimes can be useful)

sys.argv[1:] = ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"]

---

if __name__ == "__main__":

    sys.argv[1:] = ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"]

    args = parse_arguments()
    run(args)

---

Eventually you can change function parse_arguments() to make it more universal.

You can set

def parse_arguments(arguments=None):

    # ... code ...

    args = parser.parse_args(arguments)

and then you can use it in script which you run in console/terminal

args = parse_arguments()

or in Jupyter

args = parse_arguments( ["-t", "abnormal", "-p", "axial", "--prefix_name", "abc"] )

---

BTW:

sys.argv.append() and sys.argv.extend() can be also useful to add some options to all executions when you run in console/terminal

There is module shlex to split arguments with spaces inside -msg "Hello World"

Normal text.split(" ") will create incorrect list ['-msg', '"Hello', 'World"']

shlex.split(text) will create correct list ['-msg', 'Hello World']