Note
Click here to download the full example code
Deep CCAΒΆ
This example demonstrates how to easily train Deep CCA models and variants
import numpy as np
import pytorch_lightning as pl
from matplotlib import pyplot as plt
from torch.utils.data import Subset
from cca_zoo.data import Split_MNIST_Dataset
from cca_zoo.deepmodels import (
DCCA,
CCALightning,
get_dataloaders,
architectures,
DCCA_NOI,
DCCA_SDL,
BarlowTwins,
)
def plot_latent_label(model, dataloader, num_batches=100):
fig, ax = plt.subplots(ncols=model.latent_dims)
for j in range(model.latent_dims):
ax[j].set_title(f"Dimension {j}")
ax[j].set_xlabel("View 1")
ax[j].set_ylabel("View 2")
for i, (data, label) in enumerate(dataloader):
z = model(*data)
zx, zy = z
zx = zx.to("cpu").detach().numpy()
zy = zy.to("cpu").detach().numpy()
for j in range(model.latent_dims):
ax[j].scatter(zx[:, j], zy[:, j], c=label.numpy(), cmap="tab10")
if i > num_batches:
plt.colorbar()
break
n_train = 500
n_val = 100
train_dataset = Split_MNIST_Dataset(mnist_type="MNIST", train=True)
val_dataset = Subset(train_dataset, np.arange(n_train, n_train + n_val))
train_dataset = Subset(train_dataset, np.arange(n_train))
train_loader, val_loader = get_dataloaders(train_dataset, val_dataset, batch_size=128)
# The number of latent dimensions across models
latent_dims = 2
# number of epochs for deep models
epochs = 20
encoder_1 = architectures.Encoder(latent_dims=latent_dims, feature_size=392)
encoder_2 = architectures.Encoder(latent_dims=latent_dims, feature_size=392)
Deep CCA
dcca = DCCA(latent_dims=latent_dims, encoders=[encoder_1, encoder_2])
dcca = CCALightning(dcca)
trainer = pl.Trainer(max_epochs=epochs, enable_checkpointing=False)
trainer.fit(dcca, train_loader, val_loader)
plot_latent_label(dcca.model, train_loader)
plt.suptitle("DCCA")
plt.show()
Out:
Validation sanity check: 0it [00:00, ?it/s]
Validation sanity check: 0%| | 0/1 [00:00<?, ?it/s]
/home/docs/checkouts/readthedocs.org/user_builds/cca-zoo/envs/v1.10.5/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:408: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
f"The number of training samples ({self.num_training_batches}) is smaller than the logging interval"
Training: 0it [00:00, ?it/s]
Training: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 25%|##5 | 1/4 [00:00<00:00, 68.95it/s, loss=-0.318, v_num=4]
Epoch 0: 50%|##### | 2/4 [00:00<00:00, 81.11it/s, loss=-0.692, v_num=4]
Epoch 0: 75%|#######5 | 3/4 [00:00<00:00, 98.43it/s, loss=-0.83, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 0: 100%|##########| 4/4 [00:00<00:00, 92.39it/s, loss=-0.83, v_num=4]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s, loss=-0.83, v_num=4]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=-0.83, v_num=4]
Epoch 1: 25%|##5 | 1/4 [00:00<00:00, 68.34it/s, loss=-1, v_num=4]
Epoch 1: 50%|##### | 2/4 [00:00<00:00, 80.89it/s, loss=-1.1, v_num=4]
Epoch 1: 75%|#######5 | 3/4 [00:00<00:00, 98.18it/s, loss=-1.18, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 1: 100%|##########| 4/4 [00:00<00:00, 92.23it/s, loss=-1.18, v_num=4]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.18, v_num=4]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.18, v_num=4]
Epoch 2: 25%|##5 | 1/4 [00:00<00:00, 67.77it/s, loss=-1.24, v_num=4]
Epoch 2: 50%|##### | 2/4 [00:00<00:00, 80.47it/s, loss=-1.3, v_num=4]
Epoch 2: 75%|#######5 | 3/4 [00:00<00:00, 97.41it/s, loss=-1.35, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 2: 100%|##########| 4/4 [00:00<00:00, 91.23it/s, loss=-1.35, v_num=4]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.35, v_num=4]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.35, v_num=4]
Epoch 3: 25%|##5 | 1/4 [00:00<00:00, 67.35it/s, loss=-1.39, v_num=4]
Epoch 3: 50%|##### | 2/4 [00:00<00:00, 80.27it/s, loss=-1.42, v_num=4]
Epoch 3: 75%|#######5 | 3/4 [00:00<00:00, 97.55it/s, loss=-1.45, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 3: 100%|##########| 4/4 [00:00<00:00, 91.93it/s, loss=-1.45, v_num=4]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.45, v_num=4]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.45, v_num=4]
Epoch 4: 25%|##5 | 1/4 [00:00<00:00, 68.08it/s, loss=-1.48, v_num=4]
Epoch 4: 50%|##### | 2/4 [00:00<00:00, 81.16it/s, loss=-1.5, v_num=4]
Epoch 4: 75%|#######5 | 3/4 [00:00<00:00, 98.53it/s, loss=-1.52, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 4: 100%|##########| 4/4 [00:00<00:00, 92.64it/s, loss=-1.52, v_num=4]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.52, v_num=4]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.52, v_num=4]
Epoch 5: 25%|##5 | 1/4 [00:00<00:00, 67.55it/s, loss=-1.54, v_num=4]
Epoch 5: 50%|##### | 2/4 [00:00<00:00, 80.21it/s, loss=-1.56, v_num=4]
Epoch 5: 75%|#######5 | 3/4 [00:00<00:00, 97.30it/s, loss=-1.57, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 5: 100%|##########| 4/4 [00:00<00:00, 91.60it/s, loss=-1.57, v_num=4]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.57, v_num=4]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.57, v_num=4]
Epoch 6: 25%|##5 | 1/4 [00:00<00:00, 67.30it/s, loss=-1.58, v_num=4]
Epoch 6: 50%|##### | 2/4 [00:00<00:00, 79.04it/s, loss=-1.6, v_num=4]
Epoch 6: 75%|#######5 | 3/4 [00:00<00:00, 95.77it/s, loss=-1.67, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 6: 100%|##########| 4/4 [00:00<00:00, 90.47it/s, loss=-1.67, v_num=4]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.67, v_num=4]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.67, v_num=4]
Epoch 7: 25%|##5 | 1/4 [00:00<00:00, 67.15it/s, loss=-1.71, v_num=4]
Epoch 7: 50%|##### | 2/4 [00:00<00:00, 79.23it/s, loss=-1.75, v_num=4]
Epoch 7: 75%|#######5 | 3/4 [00:00<00:00, 94.59it/s, loss=-1.77, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 7: 100%|##########| 4/4 [00:00<00:00, 88.54it/s, loss=-1.77, v_num=4]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.77, v_num=4]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.77, v_num=4]
Epoch 8: 25%|##5 | 1/4 [00:00<00:00, 66.67it/s, loss=-1.79, v_num=4]
Epoch 8: 50%|##### | 2/4 [00:00<00:00, 79.69it/s, loss=-1.81, v_num=4]
Epoch 8: 75%|#######5 | 3/4 [00:00<00:00, 96.68it/s, loss=-1.82, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 8: 100%|##########| 4/4 [00:00<00:00, 91.20it/s, loss=-1.82, v_num=4]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.82, v_num=4]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.82, v_num=4]
Epoch 9: 25%|##5 | 1/4 [00:00<00:00, 67.24it/s, loss=-1.83, v_num=4]
Epoch 9: 50%|##### | 2/4 [00:00<00:00, 80.08it/s, loss=-1.84, v_num=4]
Epoch 9: 75%|#######5 | 3/4 [00:00<00:00, 97.06it/s, loss=-1.85, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 9: 100%|##########| 4/4 [00:00<00:00, 90.37it/s, loss=-1.85, v_num=4]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.85, v_num=4]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.85, v_num=4]
Epoch 10: 25%|##5 | 1/4 [00:00<00:00, 68.44it/s, loss=-1.85, v_num=4]
Epoch 10: 50%|##### | 2/4 [00:00<00:00, 81.39it/s, loss=-1.86, v_num=4]
Epoch 10: 75%|#######5 | 3/4 [00:00<00:00, 98.86it/s, loss=-1.87, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 10: 100%|##########| 4/4 [00:00<00:00, 93.09it/s, loss=-1.87, v_num=4]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.87, v_num=4]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.87, v_num=4]
Epoch 11: 25%|##5 | 1/4 [00:00<00:00, 68.25it/s, loss=-1.88, v_num=4]
Epoch 11: 50%|##### | 2/4 [00:00<00:00, 79.91it/s, loss=-1.88, v_num=4]
Epoch 11: 75%|#######5 | 3/4 [00:00<00:00, 97.13it/s, loss=-1.89, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 11: 100%|##########| 4/4 [00:00<00:00, 91.51it/s, loss=-1.89, v_num=4]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.89, v_num=4]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.89, v_num=4]
Epoch 12: 25%|##5 | 1/4 [00:00<00:00, 67.91it/s, loss=-1.89, v_num=4]
Epoch 12: 50%|##### | 2/4 [00:00<00:00, 81.26it/s, loss=-1.9, v_num=4]
Epoch 12: 75%|#######5 | 3/4 [00:00<00:00, 98.78it/s, loss=-1.9, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 12: 100%|##########| 4/4 [00:00<00:00, 91.92it/s, loss=-1.9, v_num=4]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.9, v_num=4]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.9, v_num=4]
Epoch 13: 25%|##5 | 1/4 [00:00<00:00, 68.20it/s, loss=-1.91, v_num=4]
Epoch 13: 50%|##### | 2/4 [00:00<00:00, 81.20it/s, loss=-1.91, v_num=4]
Epoch 13: 75%|#######5 | 3/4 [00:00<00:00, 98.01it/s, loss=-1.92, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 13: 100%|##########| 4/4 [00:00<00:00, 91.30it/s, loss=-1.92, v_num=4]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.92, v_num=4]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.92, v_num=4]
Epoch 14: 25%|##5 | 1/4 [00:00<00:00, 67.73it/s, loss=-1.92, v_num=4]
Epoch 14: 50%|##### | 2/4 [00:00<00:00, 80.23it/s, loss=-1.92, v_num=4]
Epoch 14: 75%|#######5 | 3/4 [00:00<00:00, 97.23it/s, loss=-1.93, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 14: 100%|##########| 4/4 [00:00<00:00, 91.44it/s, loss=-1.93, v_num=4]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=4]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=4]
Epoch 15: 25%|##5 | 1/4 [00:00<00:00, 65.13it/s, loss=-1.93, v_num=4]
Epoch 15: 50%|##### | 2/4 [00:00<00:00, 77.39it/s, loss=-1.93, v_num=4]
Epoch 15: 75%|#######5 | 3/4 [00:00<00:00, 93.92it/s, loss=-1.93, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 15: 100%|##########| 4/4 [00:00<00:00, 88.91it/s, loss=-1.93, v_num=4]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=4]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=4]
Epoch 16: 25%|##5 | 1/4 [00:00<00:00, 65.58it/s, loss=-1.94, v_num=4]
Epoch 16: 50%|##### | 2/4 [00:00<00:00, 78.85it/s, loss=-1.94, v_num=4]
Epoch 16: 75%|#######5 | 3/4 [00:00<00:00, 95.19it/s, loss=-1.94, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 16: 100%|##########| 4/4 [00:00<00:00, 89.83it/s, loss=-1.94, v_num=4]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=4]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=4]
Epoch 17: 25%|##5 | 1/4 [00:00<00:00, 69.10it/s, loss=-1.94, v_num=4]
Epoch 17: 50%|##### | 2/4 [00:00<00:00, 82.12it/s, loss=-1.94, v_num=4]
Epoch 17: 75%|#######5 | 3/4 [00:00<00:00, 99.29it/s, loss=-1.94, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 17: 100%|##########| 4/4 [00:00<00:00, 92.97it/s, loss=-1.94, v_num=4]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=4]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=4]
Epoch 18: 25%|##5 | 1/4 [00:00<00:00, 67.05it/s, loss=-1.95, v_num=4]
Epoch 18: 50%|##### | 2/4 [00:00<00:00, 79.63it/s, loss=-1.95, v_num=4]
Epoch 18: 75%|#######5 | 3/4 [00:00<00:00, 96.02it/s, loss=-1.95, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 18: 100%|##########| 4/4 [00:00<00:00, 90.32it/s, loss=-1.95, v_num=4]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=4]
Epoch 19: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=4]
Epoch 19: 25%|##5 | 1/4 [00:00<00:00, 65.94it/s, loss=-1.95, v_num=4]
Epoch 19: 50%|##### | 2/4 [00:00<00:00, 78.34it/s, loss=-1.95, v_num=4]
Epoch 19: 75%|#######5 | 3/4 [00:00<00:00, 94.81it/s, loss=-1.95, v_num=4]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 89.16it/s, loss=-1.95, v_num=4]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 63.75it/s, loss=-1.95, v_num=4]
Deep CCA by Non-Linear Orthogonal Iterations
dcca_noi = DCCA_NOI(
latent_dims=latent_dims, N=len(train_dataset), encoders=[encoder_1, encoder_2]
)
dcca_noi = CCALightning(dcca_noi)
trainer = pl.Trainer(max_epochs=epochs, enable_checkpointing=False)
trainer.fit(dcca_noi, train_loader, val_loader)
plot_latent_label(dcca_noi.model, train_loader)
plt.title("DCCA by Non-Linear Orthogonal Iterations")
plt.show()
Out:
Validation sanity check: 0it [00:00, ?it/s]
Validation sanity check: 0%| | 0/1 [00:00<?, ?it/s]
/home/docs/checkouts/readthedocs.org/user_builds/cca-zoo/envs/v1.10.5/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:408: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
f"The number of training samples ({self.num_training_batches}) is smaller than the logging interval"
Training: 0it [00:00, ?it/s]
Training: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 25%|##5 | 1/4 [00:00<00:00, 71.12it/s, loss=63.6, v_num=5]
Epoch 0: 50%|##### | 2/4 [00:00<00:00, 86.99it/s, loss=46.7, v_num=5]
Epoch 0: 75%|#######5 | 3/4 [00:00<00:00, 107.76it/s, loss=35.7, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 0: 100%|##########| 4/4 [00:00<00:00, 97.65it/s, loss=35.7, v_num=5]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s, loss=35.7, v_num=5]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=35.7, v_num=5]
Epoch 1: 25%|##5 | 1/4 [00:00<00:00, 70.93it/s, loss=27.8, v_num=5]
Epoch 1: 50%|##### | 2/4 [00:00<00:00, 85.03it/s, loss=23.3, v_num=5]
Epoch 1: 75%|#######5 | 3/4 [00:00<00:00, 105.26it/s, loss=20.9, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 1: 100%|##########| 4/4 [00:00<00:00, 96.27it/s, loss=20.9, v_num=5]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=20.9, v_num=5]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=20.9, v_num=5]
Epoch 2: 25%|##5 | 1/4 [00:00<00:00, 70.97it/s, loss=19.6, v_num=5]
Epoch 2: 50%|##### | 2/4 [00:00<00:00, 85.22it/s, loss=18.6, v_num=5]
Epoch 2: 75%|#######5 | 3/4 [00:00<00:00, 105.21it/s, loss=17.5, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 2: 100%|##########| 4/4 [00:00<00:00, 96.46it/s, loss=17.5, v_num=5]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=17.5, v_num=5]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=17.5, v_num=5]
Epoch 3: 25%|##5 | 1/4 [00:00<00:00, 72.72it/s, loss=16.4, v_num=5]
Epoch 3: 50%|##### | 2/4 [00:00<00:00, 86.66it/s, loss=15.3, v_num=5]
Epoch 3: 75%|#######5 | 3/4 [00:00<00:00, 106.89it/s, loss=14.2, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 3: 100%|##########| 4/4 [00:00<00:00, 97.62it/s, loss=14.2, v_num=5]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=14.2, v_num=5]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=14.2, v_num=5]
Epoch 4: 25%|##5 | 1/4 [00:00<00:00, 72.00it/s, loss=13.2, v_num=5]
Epoch 4: 50%|##### | 2/4 [00:00<00:00, 86.35it/s, loss=12.3, v_num=5]
Epoch 4: 75%|#######5 | 3/4 [00:00<00:00, 106.43it/s, loss=11.5, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 4: 100%|##########| 4/4 [00:00<00:00, 97.17it/s, loss=11.5, v_num=5]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=11.5, v_num=5]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=11.5, v_num=5]
Epoch 5: 25%|##5 | 1/4 [00:00<00:00, 70.79it/s, loss=10.9, v_num=5]
Epoch 5: 50%|##### | 2/4 [00:00<00:00, 84.42it/s, loss=10.3, v_num=5]
Epoch 5: 75%|#######5 | 3/4 [00:00<00:00, 103.63it/s, loss=9.78, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 5: 100%|##########| 4/4 [00:00<00:00, 95.57it/s, loss=9.78, v_num=5]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=9.78, v_num=5]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=9.78, v_num=5]
Epoch 6: 25%|##5 | 1/4 [00:00<00:00, 72.41it/s, loss=9.35, v_num=5]
Epoch 6: 50%|##### | 2/4 [00:00<00:00, 86.31it/s, loss=8.96, v_num=5]
Epoch 6: 75%|#######5 | 3/4 [00:00<00:00, 106.49it/s, loss=5.87, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 6: 100%|##########| 4/4 [00:00<00:00, 96.35it/s, loss=5.87, v_num=5]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=5.87, v_num=5]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=5.87, v_num=5]
Epoch 7: 25%|##5 | 1/4 [00:00<00:00, 72.70it/s, loss=4.46, v_num=5]
Epoch 7: 50%|##### | 2/4 [00:00<00:00, 87.94it/s, loss=3.87, v_num=5]
Epoch 7: 75%|#######5 | 3/4 [00:00<00:00, 108.31it/s, loss=3.74, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 7: 100%|##########| 4/4 [00:00<00:00, 98.62it/s, loss=3.74, v_num=5]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=3.74, v_num=5]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=3.74, v_num=5]
Epoch 8: 25%|##5 | 1/4 [00:00<00:00, 72.29it/s, loss=3.55, v_num=5]
Epoch 8: 50%|##### | 2/4 [00:00<00:00, 86.87it/s, loss=3.15, v_num=5]
Epoch 8: 75%|#######5 | 3/4 [00:00<00:00, 106.53it/s, loss=2.59, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 8: 100%|##########| 4/4 [00:00<00:00, 96.76it/s, loss=2.59, v_num=5]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=2.59, v_num=5]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=2.59, v_num=5]
Epoch 9: 25%|##5 | 1/4 [00:00<00:00, 72.95it/s, loss=2.09, v_num=5]
Epoch 9: 50%|##### | 2/4 [00:00<00:00, 87.86it/s, loss=1.67, v_num=5]
Epoch 9: 75%|#######5 | 3/4 [00:00<00:00, 108.25it/s, loss=1.39, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 9: 100%|##########| 4/4 [00:00<00:00, 98.65it/s, loss=1.39, v_num=5]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=1.39, v_num=5]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=1.39, v_num=5]
Epoch 10: 25%|##5 | 1/4 [00:00<00:00, 71.95it/s, loss=1.23, v_num=5]
Epoch 10: 50%|##### | 2/4 [00:00<00:00, 84.92it/s, loss=1.15, v_num=5]
Epoch 10: 75%|#######5 | 3/4 [00:00<00:00, 104.24it/s, loss=1.13, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 10: 100%|##########| 4/4 [00:00<00:00, 95.89it/s, loss=1.13, v_num=5]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=1.13, v_num=5]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=1.13, v_num=5]
Epoch 11: 25%|##5 | 1/4 [00:00<00:00, 73.55it/s, loss=1.12, v_num=5]
Epoch 11: 50%|##### | 2/4 [00:00<00:00, 88.21it/s, loss=1.12, v_num=5]
Epoch 11: 75%|#######5 | 3/4 [00:00<00:00, 108.64it/s, loss=1.11, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 11: 100%|##########| 4/4 [00:00<00:00, 98.90it/s, loss=1.11, v_num=5]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=1.11, v_num=5]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=1.11, v_num=5]
Epoch 12: 25%|##5 | 1/4 [00:00<00:00, 73.11it/s, loss=1.1, v_num=5]
Epoch 12: 50%|##### | 2/4 [00:00<00:00, 88.23it/s, loss=1.06, v_num=5]
Epoch 12: 75%|#######5 | 3/4 [00:00<00:00, 108.77it/s, loss=1.01, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 12: 100%|##########| 4/4 [00:00<00:00, 99.00it/s, loss=1.01, v_num=5]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=1.01, v_num=5]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=1.01, v_num=5]
Epoch 13: 25%|##5 | 1/4 [00:00<00:00, 73.58it/s, loss=0.957, v_num=5]
Epoch 13: 50%|##### | 2/4 [00:00<00:00, 87.94it/s, loss=0.894, v_num=5]
Epoch 13: 75%|#######5 | 3/4 [00:00<00:00, 108.23it/s, loss=0.83, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 13: 100%|##########| 4/4 [00:00<00:00, 99.08it/s, loss=0.83, v_num=5]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=0.83, v_num=5]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=0.83, v_num=5]
Epoch 14: 25%|##5 | 1/4 [00:00<00:00, 72.76it/s, loss=0.779, v_num=5]
Epoch 14: 50%|##### | 2/4 [00:00<00:00, 87.87it/s, loss=0.72, v_num=5]
Epoch 14: 75%|#######5 | 3/4 [00:00<00:00, 108.68it/s, loss=0.67, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 14: 100%|##########| 4/4 [00:00<00:00, 99.66it/s, loss=0.67, v_num=5]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=0.67, v_num=5]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=0.67, v_num=5]
Epoch 15: 25%|##5 | 1/4 [00:00<00:00, 72.99it/s, loss=0.639, v_num=5]
Epoch 15: 50%|##### | 2/4 [00:00<00:00, 87.87it/s, loss=0.616, v_num=5]
Epoch 15: 75%|#######5 | 3/4 [00:00<00:00, 108.24it/s, loss=0.589, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 15: 100%|##########| 4/4 [00:00<00:00, 98.78it/s, loss=0.589, v_num=5]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=0.589, v_num=5]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=0.589, v_num=5]
Epoch 16: 25%|##5 | 1/4 [00:00<00:00, 73.93it/s, loss=0.573, v_num=5]
Epoch 16: 50%|##### | 2/4 [00:00<00:00, 88.57it/s, loss=0.555, v_num=5]
Epoch 16: 75%|#######5 | 3/4 [00:00<00:00, 109.02it/s, loss=0.53, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 16: 100%|##########| 4/4 [00:00<00:00, 99.50it/s, loss=0.53, v_num=5]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=0.53, v_num=5]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=0.53, v_num=5]
Epoch 17: 25%|##5 | 1/4 [00:00<00:00, 73.73it/s, loss=0.512, v_num=5]
Epoch 17: 50%|##### | 2/4 [00:00<00:00, 88.82it/s, loss=0.494, v_num=5]
Epoch 17: 75%|#######5 | 3/4 [00:00<00:00, 109.44it/s, loss=0.47, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 17: 100%|##########| 4/4 [00:00<00:00, 99.68it/s, loss=0.47, v_num=5]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=0.47, v_num=5]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=0.47, v_num=5]
Epoch 18: 25%|##5 | 1/4 [00:00<00:00, 73.36it/s, loss=0.445, v_num=5]
Epoch 18: 50%|##### | 2/4 [00:00<00:00, 88.20it/s, loss=0.426, v_num=5]
Epoch 18: 75%|#######5 | 3/4 [00:00<00:00, 108.77it/s, loss=0.408, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 18: 100%|##########| 4/4 [00:00<00:00, 99.60it/s, loss=0.408, v_num=5]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=0.408, v_num=5]
Epoch 19: 0%| | 0/4 [00:00<?, ?it/s, loss=0.408, v_num=5]
Epoch 19: 25%|##5 | 1/4 [00:00<00:00, 74.38it/s, loss=0.397, v_num=5]
Epoch 19: 50%|##### | 2/4 [00:00<00:00, 88.93it/s, loss=0.385, v_num=5]
Epoch 19: 75%|#######5 | 3/4 [00:00<00:00, 109.44it/s, loss=0.374, v_num=5]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 98.96it/s, loss=0.374, v_num=5]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 68.89it/s, loss=0.374, v_num=5]
Deep CCA by Stochastic Decorrelation Loss
dcca_sdl = DCCA_SDL(
latent_dims=latent_dims, N=len(train_dataset), encoders=[encoder_1, encoder_2]
)
dcca_sdl = CCALightning(dcca_sdl)
trainer = pl.Trainer(max_epochs=epochs, enable_checkpointing=False)
trainer.fit(dcca_sdl, train_loader, val_loader)
plot_latent_label(dcca_sdl.model, train_loader)
plt.title("DCCA by Stochastic Decorrelation")
plt.show()
Out:
Validation sanity check: 0it [00:00, ?it/s]
Validation sanity check: 0%| | 0/1 [00:00<?, ?it/s]
/home/docs/checkouts/readthedocs.org/user_builds/cca-zoo/envs/v1.10.5/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:408: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
f"The number of training samples ({self.num_training_batches}) is smaller than the logging interval"
Training: 0it [00:00, ?it/s]
Training: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 25%|##5 | 1/4 [00:00<00:00, 72.04it/s, loss=58.4, v_num=6]
Epoch 0: 50%|##### | 2/4 [00:00<00:00, 86.99it/s, loss=50.8, v_num=6]
Epoch 0: 75%|#######5 | 3/4 [00:00<00:00, 107.90it/s, loss=42.9, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 0: 100%|##########| 4/4 [00:00<00:00, 99.73it/s, loss=42.9, v_num=6]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s, loss=42.9, v_num=6]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=42.9, v_num=6]
Epoch 1: 25%|##5 | 1/4 [00:00<00:00, 73.77it/s, loss=37.9, v_num=6]
Epoch 1: 50%|##### | 2/4 [00:00<00:00, 88.29it/s, loss=31.7, v_num=6]
Epoch 1: 75%|#######5 | 3/4 [00:00<00:00, 109.19it/s, loss=29, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 1: 100%|##########| 4/4 [00:00<00:00, 100.42it/s, loss=29, v_num=6]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=29, v_num=6]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=29, v_num=6]
Epoch 2: 25%|##5 | 1/4 [00:00<00:00, 75.14it/s, loss=27.9, v_num=6]
Epoch 2: 50%|##### | 2/4 [00:00<00:00, 90.20it/s, loss=28.1, v_num=6]
Epoch 2: 75%|#######5 | 3/4 [00:00<00:00, 111.23it/s, loss=28.2, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 2: 100%|##########| 4/4 [00:00<00:00, 101.88it/s, loss=28.2, v_num=6]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=28.2, v_num=6]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=28.2, v_num=6]
Epoch 3: 25%|##5 | 1/4 [00:00<00:00, 72.87it/s, loss=28.2, v_num=6]
Epoch 3: 50%|##### | 2/4 [00:00<00:00, 88.51it/s, loss=27.8, v_num=6]
Epoch 3: 75%|#######5 | 3/4 [00:00<00:00, 109.52it/s, loss=27.1, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 3: 100%|##########| 4/4 [00:00<00:00, 99.99it/s, loss=27.1, v_num=6]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=27.1, v_num=6]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=27.1, v_num=6]
Epoch 4: 25%|##5 | 1/4 [00:00<00:00, 74.13it/s, loss=26.4, v_num=6]
Epoch 4: 50%|##### | 2/4 [00:00<00:00, 88.48it/s, loss=25.3, v_num=6]
Epoch 4: 75%|#######5 | 3/4 [00:00<00:00, 109.49it/s, loss=24.7, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 4: 100%|##########| 4/4 [00:00<00:00, 100.57it/s, loss=24.7, v_num=6]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=24.7, v_num=6]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=24.7, v_num=6]
Epoch 5: 25%|##5 | 1/4 [00:00<00:00, 76.00it/s, loss=24.1, v_num=6]
Epoch 5: 50%|##### | 2/4 [00:00<00:00, 91.16it/s, loss=23.6, v_num=6]
Epoch 5: 75%|#######5 | 3/4 [00:00<00:00, 112.62it/s, loss=23.5, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 5: 100%|##########| 4/4 [00:00<00:00, 103.34it/s, loss=23.5, v_num=6]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=23.5, v_num=6]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=23.5, v_num=6]
Epoch 6: 25%|##5 | 1/4 [00:00<00:00, 74.12it/s, loss=23.4, v_num=6]
Epoch 6: 50%|##### | 2/4 [00:00<00:00, 89.73it/s, loss=23.3, v_num=6]
Epoch 6: 75%|#######5 | 3/4 [00:00<00:00, 111.02it/s, loss=21.2, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 6: 100%|##########| 4/4 [00:00<00:00, 101.98it/s, loss=21.2, v_num=6]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=21.2, v_num=6]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=21.2, v_num=6]
Epoch 7: 25%|##5 | 1/4 [00:00<00:00, 75.23it/s, loss=19.7, v_num=6]
Epoch 7: 50%|##### | 2/4 [00:00<00:00, 90.47it/s, loss=18.9, v_num=6]
Epoch 7: 75%|#######5 | 3/4 [00:00<00:00, 111.63it/s, loss=18.3, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 7: 100%|##########| 4/4 [00:00<00:00, 102.41it/s, loss=18.3, v_num=6]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=18.3, v_num=6]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=18.3, v_num=6]
Epoch 8: 25%|##5 | 1/4 [00:00<00:00, 72.82it/s, loss=18.4, v_num=6]
Epoch 8: 50%|##### | 2/4 [00:00<00:00, 88.45it/s, loss=17.9, v_num=6]
Epoch 8: 75%|#######5 | 3/4 [00:00<00:00, 109.28it/s, loss=17.1, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 8: 100%|##########| 4/4 [00:00<00:00, 100.43it/s, loss=17.1, v_num=6]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=17.1, v_num=6]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=17.1, v_num=6]
Epoch 9: 25%|##5 | 1/4 [00:00<00:00, 75.40it/s, loss=15.9, v_num=6]
Epoch 9: 50%|##### | 2/4 [00:00<00:00, 90.49it/s, loss=14.9, v_num=6]
Epoch 9: 75%|#######5 | 3/4 [00:00<00:00, 111.55it/s, loss=14.2, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 9: 100%|##########| 4/4 [00:00<00:00, 101.93it/s, loss=14.2, v_num=6]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=14.2, v_num=6]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=14.2, v_num=6]
Epoch 10: 25%|##5 | 1/4 [00:00<00:00, 75.42it/s, loss=13.9, v_num=6]
Epoch 10: 50%|##### | 2/4 [00:00<00:00, 89.30it/s, loss=13.6, v_num=6]
Epoch 10: 75%|#######5 | 3/4 [00:00<00:00, 110.39it/s, loss=13.3, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 10: 100%|##########| 4/4 [00:00<00:00, 101.25it/s, loss=13.3, v_num=6]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=13.3, v_num=6]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=13.3, v_num=6]
Epoch 11: 25%|##5 | 1/4 [00:00<00:00, 75.60it/s, loss=13.2, v_num=6]
Epoch 11: 50%|##### | 2/4 [00:00<00:00, 90.77it/s, loss=12.8, v_num=6]
Epoch 11: 75%|#######5 | 3/4 [00:00<00:00, 112.20it/s, loss=12.4, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 11: 100%|##########| 4/4 [00:00<00:00, 102.71it/s, loss=12.4, v_num=6]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=12.4, v_num=6]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=12.4, v_num=6]
Epoch 12: 25%|##5 | 1/4 [00:00<00:00, 76.44it/s, loss=11.8, v_num=6]
Epoch 12: 50%|##### | 2/4 [00:00<00:00, 91.60it/s, loss=11.2, v_num=6]
Epoch 12: 75%|#######5 | 3/4 [00:00<00:00, 113.03it/s, loss=10.5, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 12: 100%|##########| 4/4 [00:00<00:00, 103.34it/s, loss=10.5, v_num=6]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=10.5, v_num=6]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=10.5, v_num=6]
Epoch 13: 25%|##5 | 1/4 [00:00<00:00, 73.05it/s, loss=9.85, v_num=6]
Epoch 13: 50%|##### | 2/4 [00:00<00:00, 87.27it/s, loss=9.39, v_num=6]
Epoch 13: 75%|#######5 | 3/4 [00:00<00:00, 107.54it/s, loss=9.08, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 13: 100%|##########| 4/4 [00:00<00:00, 98.88it/s, loss=9.08, v_num=6]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=9.08, v_num=6]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=9.08, v_num=6]
Epoch 14: 25%|##5 | 1/4 [00:00<00:00, 73.91it/s, loss=8.81, v_num=6]
Epoch 14: 50%|##### | 2/4 [00:00<00:00, 88.67it/s, loss=8.39, v_num=6]
Epoch 14: 75%|#######5 | 3/4 [00:00<00:00, 109.22it/s, loss=8.08, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 14: 100%|##########| 4/4 [00:00<00:00, 100.20it/s, loss=8.08, v_num=6]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=8.08, v_num=6]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=8.08, v_num=6]
Epoch 15: 25%|##5 | 1/4 [00:00<00:00, 74.01it/s, loss=8.08, v_num=6]
Epoch 15: 50%|##### | 2/4 [00:00<00:00, 88.50it/s, loss=8.25, v_num=6]
Epoch 15: 75%|#######5 | 3/4 [00:00<00:00, 109.21it/s, loss=8.28, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 15: 100%|##########| 4/4 [00:00<00:00, 100.33it/s, loss=8.28, v_num=6]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=8.28, v_num=6]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=8.28, v_num=6]
Epoch 16: 25%|##5 | 1/4 [00:00<00:00, 73.89it/s, loss=8.3, v_num=6]
Epoch 16: 50%|##### | 2/4 [00:00<00:00, 88.90it/s, loss=7.75, v_num=6]
Epoch 16: 75%|#######5 | 3/4 [00:00<00:00, 109.60it/s, loss=7.22, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 16: 100%|##########| 4/4 [00:00<00:00, 100.46it/s, loss=7.22, v_num=6]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=7.22, v_num=6]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=7.22, v_num=6]
Epoch 17: 25%|##5 | 1/4 [00:00<00:00, 74.84it/s, loss=6.68, v_num=6]
Epoch 17: 50%|##### | 2/4 [00:00<00:00, 89.97it/s, loss=6.33, v_num=6]
Epoch 17: 75%|#######5 | 3/4 [00:00<00:00, 111.19it/s, loss=6.07, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 17: 100%|##########| 4/4 [00:00<00:00, 102.05it/s, loss=6.07, v_num=6]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=6.07, v_num=6]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=6.07, v_num=6]
Epoch 18: 25%|##5 | 1/4 [00:00<00:00, 73.83it/s, loss=5.86, v_num=6]
Epoch 18: 50%|##### | 2/4 [00:00<00:00, 89.14it/s, loss=5.63, v_num=6]
Epoch 18: 75%|#######5 | 3/4 [00:00<00:00, 110.17it/s, loss=5.81, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 18: 100%|##########| 4/4 [00:00<00:00, 100.73it/s, loss=5.81, v_num=6]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=5.81, v_num=6]
Epoch 19: 0%| | 0/4 [00:00<?, ?it/s, loss=5.81, v_num=6]
Epoch 19: 25%|##5 | 1/4 [00:00<00:00, 75.80it/s, loss=5.71, v_num=6]
Epoch 19: 50%|##### | 2/4 [00:00<00:00, 91.21it/s, loss=5.61, v_num=6]
Epoch 19: 75%|#######5 | 3/4 [00:00<00:00, 110.82it/s, loss=5.43, v_num=6]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 101.29it/s, loss=5.43, v_num=6]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 69.47it/s, loss=5.43, v_num=6]
Deep CCA by Barlow Twins
barlowtwins = BarlowTwins(latent_dims=latent_dims, encoders=[encoder_1, encoder_2])
barlowtwins = CCALightning(barlowtwins)
trainer = pl.Trainer(max_epochs=epochs, enable_checkpointing=False)
trainer.fit(dcca, train_loader, val_loader)
plot_latent_label(dcca_sdl.model, train_loader)
plt.title("DCCA by Barlow Twins")
plt.show()
Out:
Validation sanity check: 0it [00:00, ?it/s]
Validation sanity check: 0%| | 0/1 [00:00<?, ?it/s]
/home/docs/checkouts/readthedocs.org/user_builds/cca-zoo/envs/v1.10.5/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:408: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
f"The number of training samples ({self.num_training_batches}) is smaller than the logging interval"
Training: 0it [00:00, ?it/s]
Training: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s]
Epoch 0: 25%|##5 | 1/4 [00:00<00:00, 68.01it/s, loss=-1.47, v_num=7]
Epoch 0: 50%|##### | 2/4 [00:00<00:00, 79.94it/s, loss=-1.5, v_num=7]
Epoch 0: 75%|#######5 | 3/4 [00:00<00:00, 96.81it/s, loss=-1.49, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 0: 100%|##########| 4/4 [00:00<00:00, 90.25it/s, loss=-1.49, v_num=7]
Epoch 0: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.49, v_num=7]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.49, v_num=7]
Epoch 1: 25%|##5 | 1/4 [00:00<00:00, 66.69it/s, loss=-1.54, v_num=7]
Epoch 1: 50%|##### | 2/4 [00:00<00:00, 79.44it/s, loss=-1.57, v_num=7]
Epoch 1: 75%|#######5 | 3/4 [00:00<00:00, 96.25it/s, loss=-1.61, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 1: 100%|##########| 4/4 [00:00<00:00, 90.42it/s, loss=-1.61, v_num=7]
Epoch 1: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.61, v_num=7]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.61, v_num=7]
Epoch 2: 25%|##5 | 1/4 [00:00<00:00, 66.78it/s, loss=-1.64, v_num=7]
Epoch 2: 50%|##### | 2/4 [00:00<00:00, 79.24it/s, loss=-1.66, v_num=7]
Epoch 2: 75%|#######5 | 3/4 [00:00<00:00, 96.04it/s, loss=-1.68, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 2: 100%|##########| 4/4 [00:00<00:00, 91.05it/s, loss=-1.68, v_num=7]
Epoch 2: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.68, v_num=7]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.68, v_num=7]
Epoch 3: 25%|##5 | 1/4 [00:00<00:00, 66.91it/s, loss=-1.69, v_num=7]
Epoch 3: 50%|##### | 2/4 [00:00<00:00, 79.50it/s, loss=-1.71, v_num=7]
Epoch 3: 75%|#######5 | 3/4 [00:00<00:00, 96.22it/s, loss=-1.73, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 3: 100%|##########| 4/4 [00:00<00:00, 90.27it/s, loss=-1.73, v_num=7]
Epoch 3: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.73, v_num=7]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.73, v_num=7]
Epoch 4: 25%|##5 | 1/4 [00:00<00:00, 69.76it/s, loss=-1.74, v_num=7]
Epoch 4: 50%|##### | 2/4 [00:00<00:00, 82.69it/s, loss=-1.75, v_num=7]
Epoch 4: 75%|#######5 | 3/4 [00:00<00:00, 99.99it/s, loss=-1.76, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 4: 100%|##########| 4/4 [00:00<00:00, 93.73it/s, loss=-1.76, v_num=7]
Epoch 4: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.76, v_num=7]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.76, v_num=7]
Epoch 5: 25%|##5 | 1/4 [00:00<00:00, 68.50it/s, loss=-1.77, v_num=7]
Epoch 5: 50%|##### | 2/4 [00:00<00:00, 81.43it/s, loss=-1.78, v_num=7]
Epoch 5: 75%|#######5 | 3/4 [00:00<00:00, 98.41it/s, loss=-1.78, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 5: 100%|##########| 4/4 [00:00<00:00, 90.72it/s, loss=-1.78, v_num=7]
Epoch 5: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.78, v_num=7]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.78, v_num=7]
Epoch 6: 25%|##5 | 1/4 [00:00<00:00, 67.64it/s, loss=-1.79, v_num=7]
Epoch 6: 50%|##### | 2/4 [00:00<00:00, 79.69it/s, loss=-1.8, v_num=7]
Epoch 6: 75%|#######5 | 3/4 [00:00<00:00, 96.86it/s, loss=-1.82, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 6: 100%|##########| 4/4 [00:00<00:00, 91.61it/s, loss=-1.82, v_num=7]
Epoch 6: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.82, v_num=7]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.82, v_num=7]
Epoch 7: 25%|##5 | 1/4 [00:00<00:00, 68.72it/s, loss=-1.84, v_num=7]
Epoch 7: 50%|##### | 2/4 [00:00<00:00, 81.87it/s, loss=-1.86, v_num=7]
Epoch 7: 75%|#######5 | 3/4 [00:00<00:00, 99.11it/s, loss=-1.88, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 7: 100%|##########| 4/4 [00:00<00:00, 93.21it/s, loss=-1.88, v_num=7]
Epoch 7: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.88, v_num=7]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.88, v_num=7]
Epoch 8: 25%|##5 | 1/4 [00:00<00:00, 65.83it/s, loss=-1.89, v_num=7]
Epoch 8: 50%|##### | 2/4 [00:00<00:00, 79.01it/s, loss=-1.9, v_num=7]
Epoch 8: 75%|#######5 | 3/4 [00:00<00:00, 95.09it/s, loss=-1.9, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 8: 100%|##########| 4/4 [00:00<00:00, 88.56it/s, loss=-1.9, v_num=7]
Epoch 8: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.9, v_num=7]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.9, v_num=7]
Epoch 9: 25%|##5 | 1/4 [00:00<00:00, 67.84it/s, loss=-1.91, v_num=7]
Epoch 9: 50%|##### | 2/4 [00:00<00:00, 79.39it/s, loss=-1.91, v_num=7]
Epoch 9: 75%|#######5 | 3/4 [00:00<00:00, 95.19it/s, loss=-1.92, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 9: 100%|##########| 4/4 [00:00<00:00, 89.08it/s, loss=-1.92, v_num=7]
Epoch 9: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.92, v_num=7]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.92, v_num=7]
Epoch 10: 25%|##5 | 1/4 [00:00<00:00, 67.53it/s, loss=-1.92, v_num=7]
Epoch 10: 50%|##### | 2/4 [00:00<00:00, 80.56it/s, loss=-1.93, v_num=7]
Epoch 10: 75%|#######5 | 3/4 [00:00<00:00, 96.99it/s, loss=-1.93, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 10: 100%|##########| 4/4 [00:00<00:00, 91.95it/s, loss=-1.93, v_num=7]
Epoch 10: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=7]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.93, v_num=7]
Epoch 11: 25%|##5 | 1/4 [00:00<00:00, 69.96it/s, loss=-1.93, v_num=7]
Epoch 11: 50%|##### | 2/4 [00:00<00:00, 81.40it/s, loss=-1.94, v_num=7]
Epoch 11: 75%|#######5 | 3/4 [00:00<00:00, 98.93it/s, loss=-1.94, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 11: 100%|##########| 4/4 [00:00<00:00, 93.38it/s, loss=-1.94, v_num=7]
Epoch 11: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=7]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.94, v_num=7]
Epoch 12: 25%|##5 | 1/4 [00:00<00:00, 70.24it/s, loss=-1.94, v_num=7]
Epoch 12: 50%|##### | 2/4 [00:00<00:00, 83.18it/s, loss=-1.95, v_num=7]
Epoch 12: 75%|#######5 | 3/4 [00:00<00:00, 100.83it/s, loss=-1.95, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 12: 100%|##########| 4/4 [00:00<00:00, 93.96it/s, loss=-1.95, v_num=7]
Epoch 12: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=7]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=7]
Epoch 13: 25%|##5 | 1/4 [00:00<00:00, 68.26it/s, loss=-1.95, v_num=7]
Epoch 13: 50%|##### | 2/4 [00:00<00:00, 80.40it/s, loss=-1.95, v_num=7]
Epoch 13: 75%|#######5 | 3/4 [00:00<00:00, 97.71it/s, loss=-1.95, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 13: 100%|##########| 4/4 [00:00<00:00, 91.57it/s, loss=-1.95, v_num=7]
Epoch 13: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=7]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.95, v_num=7]
Epoch 14: 25%|##5 | 1/4 [00:00<00:00, 69.15it/s, loss=-1.96, v_num=7]
Epoch 14: 50%|##### | 2/4 [00:00<00:00, 79.76it/s, loss=-1.96, v_num=7]
Epoch 14: 75%|#######5 | 3/4 [00:00<00:00, 96.54it/s, loss=-1.96, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 14: 100%|##########| 4/4 [00:00<00:00, 91.59it/s, loss=-1.96, v_num=7]
Epoch 14: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.96, v_num=7]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.96, v_num=7]
Epoch 15: 25%|##5 | 1/4 [00:00<00:00, 66.38it/s, loss=-1.96, v_num=7]
Epoch 15: 50%|##### | 2/4 [00:00<00:00, 79.46it/s, loss=-1.96, v_num=7]
Epoch 15: 75%|#######5 | 3/4 [00:00<00:00, 96.57it/s, loss=-1.96, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 15: 100%|##########| 4/4 [00:00<00:00, 91.10it/s, loss=-1.96, v_num=7]
Epoch 15: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.96, v_num=7]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.96, v_num=7]
Epoch 16: 25%|##5 | 1/4 [00:00<00:00, 66.02it/s, loss=-1.97, v_num=7]
Epoch 16: 50%|##### | 2/4 [00:00<00:00, 79.47it/s, loss=-1.97, v_num=7]
Epoch 16: 75%|#######5 | 3/4 [00:00<00:00, 96.22it/s, loss=-1.97, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 16: 100%|##########| 4/4 [00:00<00:00, 90.71it/s, loss=-1.97, v_num=7]
Epoch 16: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 17: 25%|##5 | 1/4 [00:00<00:00, 66.67it/s, loss=-1.97, v_num=7]
Epoch 17: 50%|##### | 2/4 [00:00<00:00, 78.56it/s, loss=-1.97, v_num=7]
Epoch 17: 75%|#######5 | 3/4 [00:00<00:00, 95.01it/s, loss=-1.97, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 17: 100%|##########| 4/4 [00:00<00:00, 90.10it/s, loss=-1.97, v_num=7]
Epoch 17: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 18: 25%|##5 | 1/4 [00:00<00:00, 67.19it/s, loss=-1.97, v_num=7]
Epoch 18: 50%|##### | 2/4 [00:00<00:00, 78.30it/s, loss=-1.97, v_num=7]
Epoch 18: 75%|#######5 | 3/4 [00:00<00:00, 95.58it/s, loss=-1.97, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 18: 100%|##########| 4/4 [00:00<00:00, 89.94it/s, loss=-1.97, v_num=7]
Epoch 18: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 19: 0%| | 0/4 [00:00<?, ?it/s, loss=-1.97, v_num=7]
Epoch 19: 25%|##5 | 1/4 [00:00<00:00, 67.98it/s, loss=-1.98, v_num=7]
Epoch 19: 50%|##### | 2/4 [00:00<00:00, 80.53it/s, loss=-1.98, v_num=7]
Epoch 19: 75%|#######5 | 3/4 [00:00<00:00, 96.65it/s, loss=-1.98, v_num=7]
Validating: 0it [00:00, ?it/s]
Validating: 0%| | 0/1 [00:00<?, ?it/s]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 89.67it/s, loss=-1.98, v_num=7]
Epoch 19: 100%|##########| 4/4 [00:00<00:00, 63.97it/s, loss=-1.98, v_num=7]
Total running time of the script: ( 0 minutes 10.591 seconds)