🏃‍♀️ Quickstart

Use Weights & Biases for machine learning experiment tracking, model checkpointing, and collaboration with your team. See the full Weights & Biases Documentation here

🤩 A shared dashboard for your experiments

With just a few lines of code, you'll get rich, interactive, shareable dashboards which you can see yourself here.

🔒 Data & Privacy

We take security very seriously, and our cloud-hosted dashboard uses industry standard best practices for encryption. If you're working with models and datasets that cannot leave your enterprise cluster, we have on-prem installations available.

It's also easy to download all your data and export it to other tools — like custom analysis in a Jupyter notebook. Here's more on our API.

🪄 Install wandb library and login

Start by installing the library and logging in to your free account.

!pip install wandb -qU

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 10.3 MB/s eta 0:00:00
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 188.6/188.6 kB 14.6 MB/s eta 0:00:00
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 218.8/218.8 kB 14.0 MB/s eta 0:00:00
[?25h  Preparing metadata (setup.py) ... [?25l[?25hdone
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 7.3 MB/s eta 0:00:00
[?25h  Building wheel for pathtools (setup.py) ... [?25l[?25hdone

# Log in to your W&B account
import wandb
wandb.login()

<IPython.core.display.Javascript object>

wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc

True

👟 Run an experiment

1️⃣. Start a new run and pass in hyperparameters to track

2️⃣. Log metrics from training or evaluation

3️⃣. Visualize results in the dashboard

import random

# Launch 5 simulated experiments
total_runs = 5
for run in range(total_runs):
  # 🐝 1️⃣ Start a new run to track this script
  wandb.init(
      # Set the project where this run will be logged
      project="basic-intro",
      # We pass a run name (otherwise it’ll be randomly assigned, like sunshine-lollypop-10)
      name=f"experiment_{run}",
      # Track hyperparameters and run metadata
      config={
      "learning_rate": 0.02,
      "architecture": "CNN",
      "dataset": "CIFAR-100",
      "epochs": 10,
      })

  # This simple block simulates a training loop logging metrics
  epochs = 10
  offset = random.random() / 5
  for epoch in range(2, epochs):
      acc = 1 - 2 ** -epoch - random.random() / epoch - offset
      loss = 2 ** -epoch + random.random() / epoch + offset

      # 🐝 2️⃣ Log metrics from your script to W&B
      wandb.log({"acc": acc, "loss": loss})

  # Mark the run as finished
  wandb.finish()

wandb: Currently logged in as: watanabe3tipapa. Use `wandb login --relogin` to force relogin

3️⃣ You can find your interactive dashboard by clicking any of the 👆 wandb links above.

🔥 Simple Pytorch Neural Network

💪 Run this model to train a simple MNIST classifier, and click on the project page link to see your results stream in live to a W&B project.

Any run in wandb automatically logs metrics, system information, hyperparameters, terminal output and you'll see an interactive table with model inputs and outputs.

Set up Dataloader

#@title
import wandb
import math
import random
import torch, torchvision
import torch.nn as nn
import torchvision.transforms as T

device = "cuda:0" if torch.cuda.is_available() else "cpu"

def get_dataloader(is_train, batch_size, slice=5):
    "Get a training dataloader"
    full_dataset = torchvision.datasets.MNIST(root=".", train=is_train, transform=T.ToTensor(), download=True)
    sub_dataset = torch.utils.data.Subset(full_dataset, indices=range(0, len(full_dataset), slice))
    loader = torch.utils.data.DataLoader(dataset=sub_dataset,
                                         batch_size=batch_size,
                                         shuffle=True if is_train else False,
                                         pin_memory=True, num_workers=2)
    return loader

def get_model(dropout):
    "A simple model"
    model = nn.Sequential(nn.Flatten(),
                         nn.Linear(28*28, 256),
                         nn.BatchNorm1d(256),
                         nn.ReLU(),
                         nn.Dropout(dropout),
                         nn.Linear(256,10)).to(device)
    return model

def validate_model(model, valid_dl, loss_func, log_images=False, batch_idx=0):
    "Compute performance of the model on the validation dataset and log a wandb.Table"
    model.eval()
    val_loss = 0.
    with torch.inference_mode():
        correct = 0
        for i, (images, labels) in enumerate(valid_dl):
            images, labels = images.to(device), labels.to(device)

            # Forward pass ➡
            outputs = model(images)
            val_loss += loss_func(outputs, labels)*labels.size(0)

            # Compute accuracy and accumulate
            _, predicted = torch.max(outputs.data, 1)
            correct += (predicted == labels).sum().item()

            # Log one batch of images to the dashboard, always same batch_idx.
            if i==batch_idx and log_images:
                log_image_table(images, predicted, labels, outputs.softmax(dim=1))
    return val_loss / len(valid_dl.dataset), correct / len(valid_dl.dataset)

def log_image_table(images, predicted, labels, probs):
    "Log a wandb.Table with (img, pred, target, scores)"
    # 🐝 Create a wandb Table to log images, labels and predictions to
    table = wandb.Table(columns=["image", "pred", "target"]+[f"score_{i}" for i in range(10)])
    for img, pred, targ, prob in zip(images.to("cpu"), predicted.to("cpu"), labels.to("cpu"), probs.to("cpu")):
        table.add_data(wandb.Image(img[0].numpy()*255), pred, targ, *prob.numpy())
    wandb.log({"predictions_table":table}, commit=False)

Train Your Model

# Launch 5 experiments, trying different dropout rates
for _ in range(5):
    # 🐝 initialise a wandb run
    wandb.init(
        project="pytorch-intro",
        config={
            "epochs": 10,
            "batch_size": 128,
            "lr": 1e-3,
            "dropout": random.uniform(0.01, 0.80),
            })

    # Copy your config
    config = wandb.config

    # Get the data
    train_dl = get_dataloader(is_train=True, batch_size=config.batch_size)
    valid_dl = get_dataloader(is_train=False, batch_size=2*config.batch_size)
    n_steps_per_epoch = math.ceil(len(train_dl.dataset) / config.batch_size)

    # A simple MLP model
    model = get_model(config.dropout)

    # Make the loss and optimizer
    loss_func = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)

   # Training
    example_ct = 0
    step_ct = 0
    for epoch in range(config.epochs):
        model.train()
        for step, (images, labels) in enumerate(train_dl):
            images, labels = images.to(device), labels.to(device)

            outputs = model(images)
            train_loss = loss_func(outputs, labels)
            optimizer.zero_grad()
            train_loss.backward()
            optimizer.step()

            example_ct += len(images)
            metrics = {"train/train_loss": train_loss,
                       "train/epoch": (step + 1 + (n_steps_per_epoch * epoch)) / n_steps_per_epoch,
                       "train/example_ct": example_ct}

            if step + 1 < n_steps_per_epoch:
                # 🐝 Log train metrics to wandb
                wandb.log(metrics)

            step_ct += 1

        val_loss, accuracy = validate_model(model, valid_dl, loss_func, log_images=(epoch==(config.epochs-1)))

        # 🐝 Log train and validation metrics to wandb
        val_metrics = {"val/val_loss": val_loss,
                       "val/val_accuracy": accuracy}
        wandb.log({**metrics, **val_metrics})

        print(f"Train Loss: {train_loss:.3f}, Valid Loss: {val_loss:3f}, Accuracy: {accuracy:.2f}")

    # If you had a test set, this is how you could log it as a Summary metric
    wandb.summary['test_accuracy'] = 0.8

    # 🐝 Close your wandb run
    wandb.finish()

Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./MNIST/raw/train-images-idx3-ubyte.gz

100%|██████████| 9912422/9912422 [00:00<00:00, 36992851.80it/s]

Extracting ./MNIST/raw/train-images-idx3-ubyte.gz to ./MNIST/raw

Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz
Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./MNIST/raw/train-labels-idx1-ubyte.gz

100%|██████████| 28881/28881 [00:00<00:00, 9455600.17it/s]

Extracting ./MNIST/raw/train-labels-idx1-ubyte.gz to ./MNIST/raw

Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz
Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./MNIST/raw/t10k-images-idx3-ubyte.gz

100%|██████████| 1648877/1648877 [00:00<00:00, 31215798.60it/s]

Extracting ./MNIST/raw/t10k-images-idx3-ubyte.gz to ./MNIST/raw

Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz
Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./MNIST/raw/t10k-labels-idx1-ubyte.gz

100%|██████████| 4542/4542 [00:00<00:00, 4807097.85it/s]

Extracting ./MNIST/raw/t10k-labels-idx1-ubyte.gz to ./MNIST/raw

Train Loss: 0.220, Valid Loss: 0.283760, Accuracy: 0.92
Train Loss: 0.167, Valid Loss: 0.227055, Accuracy: 0.93
Train Loss: 0.136, Valid Loss: 0.195836, Accuracy: 0.94
Train Loss: 0.226, Valid Loss: 0.177382, Accuracy: 0.94
Train Loss: 0.037, Valid Loss: 0.168279, Accuracy: 0.95
Train Loss: 0.071, Valid Loss: 0.155267, Accuracy: 0.95
Train Loss: 0.042, Valid Loss: 0.163625, Accuracy: 0.95
Train Loss: 0.054, Valid Loss: 0.157482, Accuracy: 0.95
Train Loss: 0.031, Valid Loss: 0.148917, Accuracy: 0.95
Train Loss: 0.016, Valid Loss: 0.163321, Accuracy: 0.95

Train Loss: 0.279, Valid Loss: 0.285735, Accuracy: 0.92
Train Loss: 0.159, Valid Loss: 0.230620, Accuracy: 0.93
Train Loss: 0.203, Valid Loss: 0.200515, Accuracy: 0.93
Train Loss: 0.102, Valid Loss: 0.189465, Accuracy: 0.94
Train Loss: 0.076, Valid Loss: 0.182065, Accuracy: 0.94
Train Loss: 0.100, Valid Loss: 0.155006, Accuracy: 0.95
Train Loss: 0.074, Valid Loss: 0.152520, Accuracy: 0.96
Train Loss: 0.072, Valid Loss: 0.160035, Accuracy: 0.95
Train Loss: 0.041, Valid Loss: 0.152959, Accuracy: 0.95
Train Loss: 0.041, Valid Loss: 0.147800, Accuracy: 0.95

Train Loss: 0.614, Valid Loss: 0.348776, Accuracy: 0.90
Train Loss: 0.465, Valid Loss: 0.279722, Accuracy: 0.92
Train Loss: 0.520, Valid Loss: 0.259074, Accuracy: 0.92
Train Loss: 0.365, Valid Loss: 0.242544, Accuracy: 0.92
Train Loss: 0.270, Valid Loss: 0.229999, Accuracy: 0.93
Train Loss: 0.322, Valid Loss: 0.219416, Accuracy: 0.94
Train Loss: 0.261, Valid Loss: 0.209230, Accuracy: 0.93
Train Loss: 0.340, Valid Loss: 0.202012, Accuracy: 0.94
Train Loss: 0.297, Valid Loss: 0.202392, Accuracy: 0.94
Train Loss: 0.151, Valid Loss: 0.191447, Accuracy: 0.94

Train Loss: 0.275, Valid Loss: 0.288655, Accuracy: 0.92
Train Loss: 0.327, Valid Loss: 0.226741, Accuracy: 0.93
Train Loss: 0.209, Valid Loss: 0.199396, Accuracy: 0.94
Train Loss: 0.188, Valid Loss: 0.176084, Accuracy: 0.94
Train Loss: 0.125, Valid Loss: 0.164420, Accuracy: 0.95
Train Loss: 0.071, Valid Loss: 0.157538, Accuracy: 0.94
Train Loss: 0.124, Valid Loss: 0.156629, Accuracy: 0.95
Train Loss: 0.034, Valid Loss: 0.149043, Accuracy: 0.95
Train Loss: 0.041, Valid Loss: 0.146938, Accuracy: 0.95
Train Loss: 0.025, Valid Loss: 0.157761, Accuracy: 0.95

Train Loss: 0.288, Valid Loss: 0.275766, Accuracy: 0.92
Train Loss: 0.260, Valid Loss: 0.217162, Accuracy: 0.93
Train Loss: 0.150, Valid Loss: 0.194545, Accuracy: 0.94
Train Loss: 0.074, Valid Loss: 0.175806, Accuracy: 0.94
Train Loss: 0.050, Valid Loss: 0.173306, Accuracy: 0.94
Train Loss: 0.066, Valid Loss: 0.159733, Accuracy: 0.95
Train Loss: 0.049, Valid Loss: 0.154360, Accuracy: 0.95
Train Loss: 0.036, Valid Loss: 0.153197, Accuracy: 0.95
Train Loss: 0.033, Valid Loss: 0.153414, Accuracy: 0.96
Train Loss: 0.010, Valid Loss: 0.151329, Accuracy: 0.95

You have now trained your first model using wandb! 👆 Click on the wandb link above to see your metrics

🔔 Try W&B Alerts

W&B Alerts allows you to send alerts, triggered from your Python code, to your Slack or email. There are 2 steps to follow the first time you'd like to send a Slack or email alert, triggered from your code:

1) Turn on Alerts in your W&B User Settings

2) Add wandb.alert() to your code:

wandb.alert(
    title="Low accuracy",
    text=f"Accuracy is below the acceptable threshold"
)

See the minimal example below to see how to use wandb.alert. You can find the full docs for W&B Alerts here

# Start a wandb run
wandb.init(project="pytorch-intro")

# Simulating a model training loop
acc_threshold = 0.3
for training_step in range(1000):

    # Generate a random number for accuracy
    accuracy = round(random.random() + random.random(), 3)
    print(f'Accuracy is: {accuracy}, {acc_threshold}')

    # 🐝 Log accuracy to wandb
    wandb.log({"Accuracy": accuracy})

    # 🔔 If the accuracy is below the threshold, fire a W&B Alert and stop the run
    if accuracy <= acc_threshold:
        # 🐝 Send the wandb Alert
        wandb.alert(
            title='Low Accuracy',
            text=f'Accuracy {accuracy} at step {training_step} is below the acceptable theshold, {acc_threshold}',
        )
        print('Alert triggered')
        break

# Mark the run as finished (useful in Jupyter notebooks)
wandb.finish()

Accuracy is: 0.45, 0.3
Accuracy is: 1.281, 0.3
Accuracy is: 1.398, 0.3
Accuracy is: 1.635, 0.3
Accuracy is: 0.77, 0.3
Accuracy is: 1.819, 0.3
Accuracy is: 0.933, 0.3
Accuracy is: 1.124, 0.3
Accuracy is: 1.071, 0.3
Accuracy is: 0.899, 0.3
Accuracy is: 0.945, 0.3
Accuracy is: 0.96, 0.3
Accuracy is: 1.259, 0.3
Accuracy is: 0.468, 0.3
Accuracy is: 1.057, 0.3
Accuracy is: 1.086, 0.3
Accuracy is: 1.22, 0.3
Accuracy is: 1.578, 0.3
Accuracy is: 0.793, 0.3
Accuracy is: 0.501, 0.3
Accuracy is: 1.171, 0.3
Accuracy is: 1.355, 0.3
Accuracy is: 1.31, 0.3
Accuracy is: 1.198, 0.3
Accuracy is: 1.316, 0.3
Accuracy is: 1.068, 0.3
Accuracy is: 1.536, 0.3
Accuracy is: 0.536, 0.3
Accuracy is: 0.815, 0.3
Accuracy is: 0.501, 0.3
Accuracy is: 0.642, 0.3
Accuracy is: 1.214, 0.3
Accuracy is: 1.404, 0.3
Accuracy is: 1.354, 0.3
Accuracy is: 1.523, 0.3
Accuracy is: 1.244, 0.3
Accuracy is: 0.566, 0.3
Accuracy is: 0.084, 0.3
Alert triggered

VBox(children=(Label(value='0.001 MB of 0.001 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…

What's next 🚀 ?

The next tutorial you will learn how to do hyperparameter optimization using W&B Sweeps:

👉 Hyperparameters sweeps using PyTorch