--model MODEL         Name of the model to train (default: "countception"
  --redundant-bias      Override model config for redundant bias
  --no-redundant-bias   Override model config for redundant bias
  --pretrained          Start with the pretrained version of a specified network (if avail)
  --pretrained-backbone-path PATH
                        Start from pre-trained backbone weights.
  --initial-checkpoint PATH
                        Initialize model from this checkpoint (default: none)
  --resume              Resume full model and optimizer state from checkpoint (default: False)
  --no-resume-opt       Prevent resume of optimizer state when resuming model
  --interpolation NAME  Image resize interpolation type (overrides model)
  --fill-color NAME     Image augmentation fill (background) color ("mean" or int)
  -b N, --batch-size N  input batch size for training (default: 32)
  -vb N, --validation-batch-size-multiplier N
                        ratio of validation batch size to training batch size (default: 1)
  --input_size PCT      Image size (default: None) if this is not set default model image size is taken
  --drop PCT            Dropout rate (default: 0.)
  --clip-grad NORM      Clip gradient norm (default: 10.0)
  --opt OPTIMIZER       Optimizer (default: "momentum"
  --opt-eps EPSILON     Optimizer Epsilon (default: 1e-3)
  --momentum M          SGD momentum (default: 0.9)
  --weight-decay WEIGHT_DECAY
                        weight decay (default: 0.00004)
  --sched SCHEDULER     LR scheduler (default: "step"
  --lr LR               learning rate (default: 0.01)
  --lr-noise pct, pct [pct, pct ...]
                        learning rate noise on/off epoch percentages
  --lr-noise-pct PERCENT
                        learning rate noise limit percent (default: 0.67)
  --lr-noise-std STDDEV
                        learning rate noise std-dev (default: 1.0)
  --lr-cycle-mul MULT   learning rate cycle len multiplier (default: 1.0)
  --lr-cycle-limit N    learning rate cycle limit
  --warmup-lr LR        warmup learning rate (default: 0.0001)
  --min-lr LR           lower lr bound for cyclic schedulers that hit 0 (1e-5)
  --epochs N            number of epochs to train (default: 2)
  --start-epoch N       manual epoch number (useful on restarts)
  --decay-epochs N      epoch interval to decay LR
  --warmup-epochs N     epochs to warmup LR, if scheduler supports
  --cooldown-epochs N   epochs to cooldown LR at min_lr, after cyclic schedule ends
  --patience-epochs N   patience epochs for Plateau LR scheduler (default: 10
  --decay-rate RATE, --dr RATE
                        LR decay rate (default: 0.1)
  --mixup MIXUP         mixup alpha, mixup enabled if > 0. (default: 0.)
  --mixup-off-epoch N   turn off mixup after this epoch, disabled if 0 (default: 0)
  --smoothing SMOOTHING
                        label smoothing (default: 0.0)
  --train-interpolation TRAIN_INTERPOLATION
                        Training interpolation (random, bilinear, bicubic default: "random")
  --sync-bn             Enable NVIDIA Apex or Torch synchronized BatchNorm.
  --dist-bn DIST_BN     Distribute BatchNorm stats between nodes after each epoch ("broadcast", "reduce", or "")
  --model-ema           Enable tracking moving average of model weights
  --model-ema-decay MODEL_EMA_DECAY
                        decay factor for model weights moving average (default: 0.9998)
  --dist-group-size DIST_GROUP_SIZE
                        Group size for sync-bn
  --seed S              random seed (default: 42)
  --log-interval N      how many batches to wait before logging training status
  --eval-after N        Start evaluating after eval-after epochs
  --benchmark           Turn this on when measuring performance
  --benchmark-steps N   Run training for this number of steps for performance measurement
  --dllogger-file PATH  File name of dllogger json file (default: log.json, current dir)
  --save-checkpoint-interval N
                        Save checkpoints after so many epochs
  -j N, --workers N     how many training processes to use (default: 1)
  --amp                 use NVIDIA amp for mixed precision training
  --no-pin-mem          Disable pin CPU memory in DataLoader.
  --no-prefetcher       disable fast prefetcher
  --output PATH         path to the output folder (default: none, current dir)
  --eval-metric EVAL_METRIC
                        Best metric (default: "map"
  --local_rank LOCAL_RANK
  --memory-format {nchw,nhwc}
                        memory layout, nchw or nhwc
  --fused-focal-loss    Use fused focal loss for better performance.
  --waymo               Train on Waymo dataset or COCO dataset. Default: False (COCO dataset)
  --num_classes PCT     Number of classes the model needs to be trained for (default: None)
  --remove-weights [REMOVE_WEIGHTS [REMOVE_WEIGHTS ...]]
                        Remove these weights from the state dict before loading checkpoint (use case can be not loading heads)
  --freeze-layers [FREEZE_LAYERS [FREEZE_LAYERS ...]]
                        Freeze these layers
  --waymo-train-annotation WAYMO_TRAIN_ANNOTATION
                        Absolute Path to waymo training annotation (default: "None")
  --waymo-val-annotation WAYMO_VAL_ANNOTATION
                        Absolute Path to waymo validation annotation (default: "None")
  --waymo-train WAYMO_TRAIN
                        Path to waymo training relative to waymo data (default: "None")
  --waymo-val WAYMO_VAL
                        Path to waymo validation relative to waymo data (default: "None")

sbatch N 4 --ntasks-per-node=8 ./scripts/D0/train_{AMP, FP32}_32xV100-32G.sub