edit 13_feedforward.py

.gitignore

@@ -2,5 +2,9 @@ bin/
 include/
 lib/
 lib64
+share/
 pyvenv.cfg
 .python-version
+data/*
+!data/wine/
+runs/

05_01_gradient.py

@@ -1,9 +1,14 @@
-import numpy as np
+# prediction            manual
+# gradient computation  manual
+# loss computation      manual
+# parameter update      manual
 
 # linear regression, no bias
-# f = w*x
 # f = 2*x
 
+import numpy as np
+
+
 X = np.array([1, 2, 3, 4], dtype=np.float32)
 Y = np.array([2, 4, 6, 8], dtype=np.float32)
 

05_02_gradient_autograd.py

@@ -1,9 +1,14 @@
-import torch
+# prediction            manual
+# gradient computation  autograd    -> gradient computation gets replaced by backward()
+# loss computation      manual
+# parameter update      manual
 
 # linear regression, no bias
-# f = w*x
 # f = 2*x
 
+import torch
+
+
 X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)
 Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)
 

06_03_gradient_torch_loss_optim.py

@@ -0,0 +1,52 @@
+# prediction            manual
+# gradient computation  autograd
+# loss computation      pytorch loss        -> loss function gets replaced by pytorch function
+# parameter update      pytorch optimizer   -> update weights gets replaced by optimizer.step()
+
+import torch
+import torch.nn as nn # neural network module
+
+
+# linear regression, no bias
+# f = w*x
+# f = 2*x
+
+X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)
+Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)
+
+w = torch.tensor(0.0, dtype=torch.float32, requires_grad=True) #requires grad for gradient
+
+# model prediction
+def forward(x):
+    return w*x
+
+print(f'Prediction before training: f(5) = {forward(5):.3f}')
+
+#Training
+learning_rate = .01
+n_iters = 100
+
+loss = nn.MSELoss() # use pytorch built in MSE loss function
+optimizer = torch.optim.SGD([w], lr = learning_rate) # use pytorch built in optimizer to optimize parameter 'w' with learning rate
+
+
+for epoch in range(n_iters):
+    # prediction = forward pass
+    y_pred = forward(X)
+
+    # loss
+    l = loss(Y, y_pred)
+
+    # gradients = backward pass
+    l.backward()
+
+    #update weights
+    optimizer.step()
+
+    # clear gradients
+    optimizer.zero_grad()
+
+    if epoch % 10 == 0: #every nth epoch
+        print(f'epoch {epoch+1}: w = {w:.3f}, loss = {l:.8f}')
+
+print(f'Prediction after training: f(5) = {forward(5):.3f}')

06_04_gradient_torch_model.py

@@ -0,0 +1,68 @@
+# prediction            pytorch model       -> forward function gets replaced by pytroch model
+# gradient computation  autograd
+# loss computation      pytorch loss
+# parameter update      pytorch optimizer
+
+import torch
+import torch.nn as nn # neural network module
+
+
+# linear regression, no bias
+# f = w*x
+# f = 2*x
+
+X = torch.tensor([[1], [2], [3], [4]], dtype=torch.float32) #reshape for pytorch model
+Y = torch.tensor([[2], [4], [6], [8]], dtype=torch.float32)
+X_test = torch.tensor([5], dtype=torch.float32)
+
+n_samples, n_features = X.shape
+print(n_samples, n_features)
+
+input_size = n_features
+output_size = n_features
+
+# model = nn.Linear(input_size, output_size, bias=False) 
+
+#custom linear regression model (just a wrapper in this case, but you can add more layers)
+class LinearRegression(nn.Module):
+    def __init__(self, input_dim, output_dim, bias=True):
+        super(LinearRegression, self).__init__()
+        #define layers
+        self.lin = nn.Linear(input_dim, output_dim, bias)
+
+    def forward(self, x):
+        return self.lin(x)
+
+model = LinearRegression(input_size, output_size, bias=False)
+
+print(f'Prediction before training: f(5) = {model(X_test).item():.3f}')
+
+#Training
+learning_rate = .01
+n_iters = 100
+
+loss = nn.MSELoss() # use pytorch built in MSE loss function
+optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate) # use pytorch built in optimizer to optimize parameter 'w' with learning rate
+
+
+for epoch in range(n_iters):
+    # prediction = forward pass
+    y_pred = model(X)
+
+    # loss
+    l = loss(Y, y_pred)
+
+    # gradients = backward pass
+    l.backward()
+
+    #update weights
+    optimizer.step()
+
+    # clear gradients
+    optimizer.zero_grad()
+
+    if epoch % 10 == 0: #every nth epoch
+        [w] = model.parameters()
+        print(f'epoch {epoch+1}: w = {w[0].item():.3f}, loss = {l:.8f}')
+
+print(f'Prediction after training: f(5) = {model(X_test).item():.3f}')

06_training pipeline.md

@@ -0,0 +1,11 @@
+# Training Pipeline
+
+a training pipeline generally consists of 3 steps:
+1. Design model (input, output size, forward pass (layers))
+2. Construct loss and optimizer
+3. Training loop
+    - forward pass: compute prediction
+    - backward pass: gradient computation
+    - update parameters
+
+(iterate step 3)

07_linear_regression.py

@@ -0,0 +1,60 @@
+# this is a recap
+
+# a training pipeline generally consists of 3 steps:
+# 1. Design model (input, output size, forward pass (layers))
+# 2. Construct loss and optimizer
+# 3. Training loop
+#     - forward pass: compute prediction
+#     - backward pass: gradient computation
+#     - update parameters
+
+# (iterate step 3)
+
+import torch
+import torch.nn as nn
+import numpy as np
+from sklearn import datasets
+import matplotlib.pyplot as plt
+
+# 0. data preparation
+X_numpy, Y_numpy = datasets.make_regression(n_samples=100, n_features=1, noise=20, random_state=1)
+
+X = torch.from_numpy(X_numpy.astype(np.float32))
+y = torch.from_numpy(Y_numpy.astype(np.float32))
+y = y.view(y.shape[0], 1) # reshape into column vector
+
+n_samples, n_features = X.shape
+
+# 1. model
+input_size = n_features
+output_size = 1
+model = nn.Linear(input_size, output_size)
+
+# 2. loss and optimizer
+learning_rate = .01
+criterion = nn.MSELoss()
+optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)
+
+# 3. training loop
+n_epochs = 10000
+for epoch in range(n_epochs):
+    #forward pass and loss
+    y_pred = model(X)
+    loss = criterion(y, y_pred)
+
+    #backward pass
+    loss.backward()
+
+    #update
+    optimizer.step()
+    optimizer.zero_grad()
+
+    if (epoch+1) % (n_epochs/10) == 0:
+        print(f'Epoch {epoch+1}: loss = {loss.item():.4f}')
+
+# plot
+
+predicted = model(X).detach().numpy()
+plt.plot(X_numpy, Y_numpy, 'ro')
+plt.plot(X_numpy, predicted, 'b')
+plt.show()

08_logistic_regression.py

@@ -0,0 +1,95 @@
+# this is a recap
+
+# a training pipeline generally consists of 3 steps:
+# 1. Design model (input, output size, forward pass (layers))
+# 2. Construct loss and optimizer
+# 3. Training loop
+#     - forward pass: compute prediction
+#     - backward pass: gradient computation
+#     - update parameters
+
+# (iterate step 3)
+
+import torch
+import torch.nn as nn
+import numpy as np
+from sklearn import datasets
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+# import matplotlib.pyplot as plt
+
+# 0. data preparation
+bc = datasets.load_breast_cancer()
+X, y = bc.data, bc.target
+
+n_samples, n_features = X.shape # 569 30
+
+# split data into training and test data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1234)
+
+# scale
+sc = StandardScaler()
+X_train = sc.fit_transform(X_train)
+X_test = sc.transform(X_test)
+
+X_train = torch.from_numpy(X_train.astype(np.float32))
+X_test = torch.from_numpy(X_test.astype(np.float32))
+y_train = torch.from_numpy(y_train.astype(np.float32))
+y_test = torch.from_numpy(y_test.astype(np.float32))
+
+# reshape y
+y_train = y_train.view(-1, 1) # y_train.view(y_train.shape[0], 1) 
+y_test = y_test.view(-1, 1)
+
+X_train = X_train.to('cuda')
+X_test = X_test.to('cuda')
+y_train = y_train.to('cuda')
+y_test = y_test.to('cuda')
+
+
+# 1. model
+# f = wx + b, sigmoid at the end
+class LogReg(nn.Module):
+    def __init__(self, n_input_features):
+        super(LogReg, self).__init__()
+        self.linear = nn.Linear(n_input_features, 1)
+
+    def forward(self, x):
+        y_pred = torch.sigmoid(self.linear(x))
+        return y_pred
+
+model = LogReg(n_features)
+model = model.to('cuda')
+
+# 2. loss + optimizer
+learning_rate = .0001
+criterion = nn.BCELoss() # binary cross entropy                             #accuracy at lr = .0001, n_epochs = 100000
+# optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)       # 0.93
+# optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)      # 0.96
+# optimizer = torch.optim.AdamW(model.parameters(), lr = learning_rate)     # 0.96
+optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate, momentum=0.9)     # 0.95
+
+# 3. training
+n_epochs = 100000
+
+for epoch in range(n_epochs):
+    # forward pass and loss
+    y_pred = model(X_train)
+    loss = criterion(y_pred, y_train)
+
+    # backward pass
+    loss.backward()
+
+    # updates and zero gradients
+    optimizer.step()
+    optimizer.zero_grad()
+
+    if (epoch+1) % (10000) == 0:
+        print(f'{(epoch+1)/n_epochs*100:.1f}% Epoch {epoch+1:>8}: loss = {loss.item():.5f}')
+
+# evaluation
+with torch.no_grad(): # don't track these operations for the gradient computation
+    y_pred = model(X_test)
+    y_pred_cls = y_pred.round()
+    acc = y_pred_cls.eq(y_test).sum() / float(y_test.shape[0])
+    print(f'Accuracy = {acc:.4f}')

09_dataloaders.py

@@ -0,0 +1,48 @@
+import torch
+import torchvision
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+import math
+
+class WineDataset(Dataset):
+    def __init__(self):
+        #data loading
+        xy = np.loadtxt('./data/wine/wine.csv', delimiter=',', dtype=np.float32, skiprows=1)
+        self.x = torch.from_numpy(xy[:,1:])  # n_samples x features
+        self.y = torch.from_numpy(xy[:,[0]]) # n_samples x 1
+        self.n_samples = xy.shape[0]
+
+    def __getitem__(self, idx):
+        return self.x[idx], self.y[idx]
+
+    def __len__(self):
+        return self.n_samples
+
+dataset = WineDataset()
+# first_data = dataset[0]
+# features, labels = first_data
+# print(features, labels)
+batch_size = 4
+dataloader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=2)
+
+# dataiter = iter(dataloader)
+# data = dataiter.next()
+# features, labels = data
+# print(features, labels)
+
+# dummy training loop
+n_epochs = 2
+total_samples = len(dataset)
+n_iter = math.ceil(total_samples/batch_size)
+print(total_samples, n_iter)
+
+for epoch in range(n_epochs):
+    for i, (inputs, labels) in enumerate(dataloader):
+        # forward backward update
+        if (i+1) % 5 == 0:
+            print(f'Epoch {epoch+1}/{n_epochs}, step {i+1}/{n_iter}, inputs {inputs.shape}')
+
+
+# pytorch built in datasets
+# torchvision.datasets.MNIST()
+# fashion-mnist, cifar, coco

09_epochs_batches.md

@@ -0,0 +1,12 @@
+# basics of batches
+
+DataSet and DataLoader classes for batches of data instead of laoding the entire dataset at once
+for training we loop over epochs and batches
+
+epoch = 1 forward and backward pass of ALL training samples
+batch_size = number of sample in one forward/backward pass
+number of iterations = number of passes, each pass using [batch_size] number of samples
+-> each epoch gets split up into [number of iterations] passes
+
+e.g. 100 samples, batch_size=20 -> 100/20 = 5 iterations for 1 epoch
+

10_dataset_transforms.py

@@ -0,0 +1,96 @@
+'''
+Transforms can be applied to PIL images, tensors, ndarrays, or custom data
+during creation of the DataSet
+
+complete list of built-in transforms: 
+https://pytorch.org/docs/stable/torchvision/transforms.html
+
+On Images
+---------
+CenterCrop, Grayscale, Pad, RandomAffine
+RandomCrop, RandomHorizontalFlip, RandomRotation
+Resize, Scale
+
+On Tensors
+----------
+LinearTransformation, Normalize, RandomErasing
+
+Conversion
+----------
+ToPILImage: from tensor or ndrarray
+ToTensor : from numpy.ndarray or PILImage
+
+Generic
+-------
+Use Lambda 
+
+Custom
+------
+Write own class
+
+Compose multiple Transforms
+---------------------------
+composed = transforms.Compose([Rescale(256),
+                               RandomCrop(224)])
+'''
+
+import torch
+import torchvision
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+import math
+
+#example
+dataset = torchvision.datasets.MNIST(root='./data', transform=torchvision.transforms.ToTensor(), download=True)
+
+class WineDataset(Dataset):
+    def __init__(self, transform=None):
+        xy = np.loadtxt('./data/wine/wine.csv', delimiter=',', dtype=np.float32, skiprows=1)
+        self.n_samples = xy.shape[0]
+
+        self.x = xy[:,1:]  # n_samples x features
+        self.y = xy[:,[0]] # n_samples x 1
+
+        self.transform = transform
+
+    def __getitem__(self, idx):
+        sample = self.x[idx], self.y[idx]
+        if self.transform:
+            sample = self.transform(sample)
+        return sample
+
+    def __len__(self):
+        return self.n_samples
+
+class ToTensor:
+    def __call__(self, sample):
+        x, y = sample
+        return torch.from_numpy(x), torch.from_numpy(y)
+
+class MulTransform:
+    def __init__(self, factor):
+        self.factor = factor
+
+    def __call__(self, sample):
+        x, y = sample
+        x *= self.factor
+        return x, y
+
+dataset = WineDataset()
+first_data = dataset[0]
+features, labels = first_data
+print(type(features), type(labels))
+
+dataset = WineDataset(transform=ToTensor())
+first_data = dataset[0]
+features, labels = first_data
+print(features)
+print(type(features), type(labels))
+
+composed = torchvision.transforms.Compose([ToTensor(), MulTransform(4.)])
+
+dataset = WineDataset(transform=composed)
+first_data = dataset[0]
+features, labels = first_data
+print(features)
+print(type(features), type(labels))

11_01_softmax.py

@@ -0,0 +1,22 @@
+# softmax squashes outputs so that the sum of the outputs equals 1 while preserving the order
+import torch
+import torch.nn as nn
+import numpy as np
+
+def softmax(x):
+    return np.exp(x)/np.sum(np.exp(x), axis=0)
+
+x = np.array([2., 1., .1])
+outputs = softmax(x)
+print('inputs: ', x)
+print('softmax numpy:', outputs)
+
+x = torch.tensor([2., 1., .1])
+outputs = torch.softmax(x, dim=0)
+print('inputs: ', x)
+print('softmax numpy:', outputs)
+
+outputs2 = torch.softmax(outputs, dim=0)
+print('inputs: ', outputs)
+print('softmax numpy:', outputs2)
+

11_02_crossentropy.py

@@ -0,0 +1,52 @@
+# loss function for multiclass problems -> labels must be one-hot encoded, predictions is a vector of probabilities (after applying softmax)
+
+import torch
+import torch.nn as nn
+import numpy as np
+
+# numpy
+def cross_entropy(actual, predicted, normalize=False):
+    loss = -np.sum(actual * np.log(predicted))
+    if normalize:
+        loss /= float(predicted.shape[0]) 
+    return loss
+
+# y must be one-hot encoded
+# class 0 [1 0 0]
+# class 1 [0 1 0]
+# class 2 [0 0 1]
+
+Y = np.array([1, 0, 0]) # class 0
+
+# y_pred has probabilities
+Y_pred_good = np.array([.7, .2, .1])
+Y_pred_bad = np.array([.1, .3, .6])
+l1 = cross_entropy(Y, Y_pred_good)
+# l1_norm = cross_entropy(Y, Y_pred_good, normalize=True)
+l2 = cross_entropy(Y, Y_pred_bad)
+# l2_norm = cross_entropy(Y, Y_pred_bad, normalize=True)
+print(f'Loss1 numpy: {l1:.4f}')
+# print(f'Loss1 numpy normalized: {l1_norm:.4f}')
+print(f'Loss2 numpy: {l2:.4f}')
+# print(f'Loss1 numpy normalized: {l2_norm:.4f}')
+
+#pytorch
+# 3 samples
+loss = nn.CrossEntropyLoss() # includes softmax -> y_pred has raw scores, y has class labels, not one-hot
+Y = torch.tensor([2, 0, 1])
+# nsamples x nclasses = 3x3
+Y_pred_good = torch.tensor([[0.1, 1., 2.1], [2., 1., .1], [0.5, 2., .3]])
+Y_pred_bad = torch.tensor([[2.1, 1., .1], [.1, 1., 2.1], [.1, 3., .1]])
+
+l1 = loss(Y_pred_good, Y)
+l2 = loss(Y_pred_bad, Y)
+
+print(f'Loss1 pytorch: {l1.item():.4f}')
+print(f'Loss2 pytorch: {l2.item():.4f}')
+
+_, prediction1 = torch.max(Y_pred_good, 1)
+_, prediction2 = torch.max(Y_pred_bad, 1)
+print(prediction1)
+print(prediction2)
+
+

12_activation_functions.py

@@ -0,0 +1,49 @@
+# activation functions apply non-linear transform to layer output
+# without activation functions, the model would just be a stacked linear regression model -> not suited for complex tasks
+
+
+# step
+# f(x) = 1 if x>=thresh else 0
+
+# sigmoid
+# f(x) = 1/(1+exp(-x))
+# between 0 and 1, typically last layer in binary classification
+
+# tanh
+# f(x) = 2/(1+exp(-2x)) -1
+# for hidden layers
+
+# ReLU
+# f(x) = max(0,x)
+# if you don't know what to use, use ReLU ;)
+
+# Leaky ReLU
+# f(x) = x if x>=0, else a*x, a is very small
+# improved ReLU, tries to solve vanishing gradient problem (against dead neurons)
+
+# softmax
+# f(x) = exp(y_i)/sum(exp(y_i))
+# last layer in multiclass classification problem
+
+import torch
+import torch.nn as nn
+
+class NeuralNet(nn.Module):
+    def __init__(self, input_size, hidden_size):
+        super(NeuralNet, self).__init__()
+        self.layers = [
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(),
+            # nn.Sigmoid(),
+            # nn.Softmax(),
+            # nn.Tanh(),
+            # nn.LeakyReLU(),
+            nn.Linear(hidden_size, 1),
+            nn.Sigmoid()
+        ]
+
+    def forward(self, x):
+        out = x
+        for layer in self.layers:
+            out = layer(out)
+        return out

13_feedforward.py

@@ -0,0 +1,105 @@
+'''
+put the last few chapters together to classify digits from MNIST dataset
+-> MNIST
+-> DataLoader, Transformation
+-> Loss + Optimizer
+-> Training Loop with batch training
+-> Model Evaluation
+-> GPU Support
+'''
+
+import torch
+import torch.nn as nn
+import torchvision
+import torchvision.transforms as transforms
+import matplotlib.pyplot as plt
+
+# device config
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f'Device is {device}')
+
+# hyper parameters
+input_size = 784 # 28x28 pixel images
+hidden_size = 100 # PLAY WITH THIS
+num_classes = 10 # digits 0..9
+num_epochs = 2 # PLAY WITH THIS
+batch_size = 100
+learning_rate = .001
+
+# MNIST
+train_dataset = torchvision.datasets.MNIST(root='./data', train=True,
+    download=True, transform=transforms.ToTensor())
+
+test_dataset = torchvision.datasets.MNIST(root='./data', train=False,
+    transform=transforms.ToTensor())
+
+train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, 
+    shuffle=True)
+
+test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size)
+
+examples = iter(train_loader)
+samples, labels = examples.next()
+print(samples.shape, labels.shape)
+
+for i in range(6):
+    plt.subplot(2, 3, i+1)
+    plt.imshow(samples[i][0], cmap='gray')
+plt.show()
+
+class NeuralNet(nn.Module):
+    def __init__(self, input_size, hidden_size, num_classes):
+        super(NeuralNet, self).__init__()
+        self.layers = nn.Sequential(
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(),
+            nn.Linear(hidden_size, num_classes)
+            # no softmax because its included in the CE loss function
+        )
+    
+    def forward(self, x):
+        return self.layers(x)
+
+model = NeuralNet(input_size, hidden_size, num_classes).to(device)
+print(model)
+
+# loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+
+# training loop
+num_total_steps = len(train_loader)
+for epoch in range(num_epochs):
+    for batch, (images, labels) in enumerate(train_loader):
+        # reshape 100, 1, 28, 28 -> 100, 784
+        images = images.reshape(-1, 28*28).to(device) # reshape and send to gpu if available
+        labels = labels.to(device)
+
+        # forward
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+
+        # backward + update
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        if (batch+1) % 100 == 0:
+            print(f'Epoch {epoch+1}/{num_epochs}, step {batch+1}/{num_total_steps}, loss = {loss.item():.4f}')
+
+# test
+with torch.no_grad():
+    n_correct = 0
+    n_samples = 0
+    for images, labels in test_loader:
+        images = images.reshape(-1, 28*28).to(device)
+        labels = labels.to(device)
+        outputs = model(images)
+
+        # value, index (index is class label)
+        _, predictions = torch.max(outputs, 1)
+        n_samples += labels.shape[0]
+        n_correct += (predictions==labels).sum().item()
+
+    acc = 100.*n_correct/n_samples
+    print(f'Accuracy = {acc}%')

14_cnn.py

@@ -0,0 +1,148 @@
+# cnn on cifar-10
+'''
+convolutional net
+similar to ff net, but applies convolutional filters (mainly on images)
+
+also include pooling layers
+specifically max pooling
+downsamples image by getting max value in a region
+
+ 12  20  30  0
+ 8   12  2   0                           20  30
+ 34  70  37  4     -- 2x2 max-pool -->  112  37
+112 100  25  12 
+ helps avoid overfitting by providing abstract form of input
+'''
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision
+import torchvision.transforms as transforms
+import matplotlib.pyplot as plt
+import numpy as np
+
+# device config
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f'Device is {device}')
+
+# hyper parameters
+num_epochs = 10
+batch_size = 10
+learning_rate = 0.001
+
+# dataset has PILImage images of range [0,1]
+# transform to tensors of normalized range [-1, 1]
+
+transform = transforms.Compose([
+    transforms.ToTensor(), 
+    transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
+    ])
+
+train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, 
+    download=True, transform=transform)
+
+test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, 
+    download=False, transform=transform)
+
+train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size)
+
+classes = ('plane', 'car', 'bird', 'cat', 'deer', 
+           'dog', 'frog', 'horse', 'ship', 'truck') # can also get them from the data, probably
+
+
+# #implement conv net
+# class ConvNet(nn.Module):
+#     def __init__(self):
+#         super(ConvNet, self).__init__()
+#         self.layers = nn.Sequential(                                                                                                   # -> 32x32
+#             nn.Conv2d(3, 6, 5), # 3 color channels, 6 output channels, kernel size 5 -> image size shrinks by 2 pixels in each direction -> 28x28
+#             nn.ReLU(),
+#             nn.MaxPool2d(2, 2), # kernel size 2, stride 2 -> shift by 2 pixels after each max-pooling -> image size shrinks to half size -> 14x14
+#             nn.Conv2d(6, 16, 5), # input size is output size of previous conv layer, -> image size shrinks by 2 pixels in each direction -> 10x10
+#             nn.ReLU(),
+#             nn.MaxPool2d(2, 2), # kernel size 2, stride 2 -> shift by 2 pixels after each max-pooling -> image size shrinks to half size ->  5x5
+#             nn.Flatten(),
+#             nn.Linear(16*5*5, 120), # 16 channels * 5px * 5px
+#             nn.ReLU(),
+#             nn.Linear(120, 84),
+#             nn.ReLU(),
+#             nn.Linear(84, 10) # output size 10 for 10 classes
+#         )
+    
+#     def forward(self, x):
+#         return self.layers(x)
+
+# model = ConvNet().to(device)
+model = nn.Sequential(                                                                                                   # -> 32x32
+            nn.Conv2d(3, 6, 5), # 3 color channels, 6 output channels, kernel size 5 -> image size shrinks by 2 pixels in each direction -> 28x28
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2), # kernel size 2, stride 2 -> shift by 2 pixels after each max-pooling -> image size shrinks to half size -> 14x14
+            nn.Conv2d(6, 16, 5), # input size is output size of previous conv layer, -> image size shrinks by 2 pixels in each direction -> 10x10
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2), # kernel size 2, stride 2 -> shift by 2 pixels after each max-pooling -> image size shrinks to half size ->  5x5
+            nn.Flatten(),
+            nn.Linear(16*5*5, 120), # 16 channels * 5px * 5px
+            nn.ReLU(),
+            nn.Linear(120, 84),
+            nn.ReLU(),
+            nn.Linear(84, 10) # output size 10 for 10 classes
+        ).to(device)
+
+criterion = nn.CrossEntropyLoss()
+optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)
+
+n_total_steps = len(train_loader)
+for epoch in range(num_epochs):
+    for i, (images, labels) in enumerate(train_loader):
+        # origin shape: [4, 3, 32, 32] = 4, 3, 1024
+        # input_layer: 3 input channels, 6 output channels, 5 kernel size
+        images = images.to(device)
+        labels = labels.to(device)
+
+        #forward pass
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+
+        # backward pass + update
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        if (i+1) % 2000 == 0:
+            print(f'Epoch {(epoch+1)}/{num_epochs}, Step {(i+1)}/{n_total_steps}, loss = {loss.item():.4f}')
+
+print('Finished Training')
+
+with torch.no_grad():
+    n_correct = 0
+    n_samples = 0
+    n_class_correct = [0 for _ in range(10)]
+    n_class_samples = [0 for _ in range(10)]
+    for images, labels in test_loader:
+        images = images.to(device)
+        labels = labels.to(device)
+        outputs = model(images)
+        # max returns (value, index)
+        _, predicted = torch.max(outputs, 1)
+        n_samples += labels.size(0)
+        n_correct += (predicted == labels).sum().item()
+
+        for i in range(batch_size):
+            label = labels[i]
+            pred = predicted[i]
+            if (label == pred):
+                n_class_correct[label] += 1
+            n_class_samples[label] += 1
+
+    acc = 100. * n_correct / n_samples
+    print(f'Accuracy of network: {acc:.1f}%')
+
+    for i in range(10):
+        acc = 100. * n_class_correct[i]/n_class_samples[i]
+        print(f'Accuracy of {classes[i]}: {acc:.1f}%')
+
+
+
+

15_transfer.py

@@ -0,0 +1,153 @@
+# new:
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.optim import lr_scheduler
+import numpy as np
+import torchvision
+from torchvision import datasets, models, transforms
+import matplotlib.pyplot as plt
+import time
+import os
+import copy
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+mean = np.array([0.485, 0.456, 0.406])
+std = np.array([0.229, 0.224, 0.225])
+
+data_transforms = {
+    'train': transforms.Compose([
+        transforms.RandomResizedCrop(224),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize(mean, std)
+    ]),
+    'val': transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean, std)
+    ])
+}
+
+#import data
+data_dir = 'data/hymenoptera_data'
+sets = ['train', 'val']
+image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in sets}
+dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4, shuffle=True, num_workers=0) for x in sets}
+dataset_sizes = {x: len(image_datasets[x]) for x in sets}
+class_names = image_datasets['train'].classes
+print(f'Classes: {class_names}')
+
+
+# # show sample images
+# def imshow(inp, title):
+#     """Imshow for Tensor."""
+#     inp = inp.numpy().transpose((1, 2, 0))
+#     inp = std * inp + mean
+#     inp = np.clip(inp, 0, 1)
+#     plt.imshow(inp)
+#     plt.title(title)
+#     plt.show()
+# inputs, classes = next(iter(dataloaders['train']))
+# out = torchvision.utils.make_grid(inputs)
+# imshow(out, title=[class_names[x] for x in classes])
+
+
+def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
+    since = time.time()
+    best_model_wts = copy.deepcopy(model.state_dict())
+    best_acc = 0.0
+
+    for epoch in range(num_epochs):
+        print(F'Epoch {epoch+1}/{num_epochs}')
+        print('-'*11)
+
+        for phase in sets:
+            if phase == 'train':
+                model.train()
+            else:
+                model.eval()
+            
+            running_loss = 0.0
+            running_corrects = 0
+
+            for inputs, labels in dataloaders[phase]:
+                inputs = inputs.to(device)
+                labels = labels.to(device)
+
+                with torch.set_grad_enabled(phase == 'train'):
+                    outputs = model(inputs)
+                    _, preds = torch.max(outputs, 1)
+                    loss = criterion(outputs, labels)
+
+                    if phase == 'train':
+                        optimizer.zero_grad()
+                        loss.backward()
+                        optimizer.step()
+                
+                running_loss += loss.item() * inputs.size(0)
+                running_corrects += torch.sum(preds == labels.data)
+            
+            if phase == 'train':
+                scheduler.step()
+            
+            epoch_loss = running_loss/dataset_sizes[phase]
+            epoch_acc = running_corrects.double()/dataset_sizes[phase]
+
+            print(f'{phase} Loss: {epoch_loss:.4f}, Acc: {epoch_acc:.4f}')
+
+            if phase == 'val' and epoch_acc > best_acc:
+                best_acc = epoch_acc
+                best_model_wts = copy.deepcopy(model.state_dict())
+            
+        print()
+    
+    time_elapsed = time.time() - since
+    print(f'Training complete in {time_elapsed//(60*60):02.0f}:{(time_elapsed%(60*60)//60):02.0f}:{time_elapsed%60:02.0f}')
+    print(f'Best val Acc: {best_acc:.4f}')
+
+    model.load_state_dict(best_model_wts)
+    return model
+
+## transfer learning - finetuning ##
+model = torchvision.models.resnet18(weights=torchvision.models.ResNet18_Weights.DEFAULT) # import pretrained model
+
+#change last layer
+num_ftrs = model.fc.in_features
+model.fc = nn.Linear(num_ftrs, 2)
+model.to(device)
+
+criterion = nn.CrossEntropyLoss()
+optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
+
+# scheduler
+# updates the learning rate
+step_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) # every seven epochs: lr = lr*gamma
+
+model = train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs=50)
+
+print('-'*50)
+print()
+
+## transfer learning - feature extractor ##
+model = torchvision.models.resnet18(weights=torchvision.models.ResNet18_Weights.DEFAULT) # import pretrained model # import pretrained model
+for param in model.parameters():
+    param.requires_grad = False
+    # freezes all layers in beginning
+
+#change last layer
+num_ftrs = model.fc.in_features
+model.fc = nn.Linear(num_ftrs, 2)
+model.to(device)
+
+criterion = nn.CrossEntropyLoss()
+optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
+
+# scheduler
+# updates the learning rate
+step_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) # every seven epochs: lr = lr*gamma
+
+model = train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs=50)

16_tensorboard.py

@@ -0,0 +1,134 @@
+# 13_feedforward with tensorboard
+import torch
+import torch.nn as nn
+import torchvision
+import torchvision.transforms as transforms
+import matplotlib.pyplot as plt
+
+import sys
+from torch.utils.tensorboard import SummaryWriter
+writer = SummaryWriter('runs/mnist')
+
+# device config
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f'Device is {device}')
+
+# hyper parameters
+input_size = 784 # 28x28 pixel images
+hidden_size = 100 # PLAY WITH THIS
+num_classes = 10 # digits 0..9
+num_epochs = 2 # PLAY WITH THIS
+batch_size = 100
+learning_rate = .001
+
+# MNIST
+train_dataset = torchvision.datasets.MNIST(root='./data', train=True,
+    download=True, transform=transforms.ToTensor())
+
+test_dataset = torchvision.datasets.MNIST(root='./data', train=False,
+    transform=transforms.ToTensor())
+
+train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, 
+    shuffle=True)
+
+test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size)
+
+examples = iter(train_loader)
+samples, labels = examples.next()
+print(samples.shape, labels.shape)
+
+for i in range(6):
+    plt.subplot(2, 3, i+1)
+    plt.imshow(samples[i][0], cmap='gray')
+# plt.show()
+img_grid = torchvision.utils.make_grid(samples)
+writer.add_image('mnist_images', img_grid)
+writer.close()
+# sys.exit()
+
+model = nn.Sequential(
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(),
+            nn.Linear(hidden_size, num_classes)
+            # no softmax because its included in the CE loss function
+        ).to(device)
+# print(model)
+
+# loss and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+
+writer.add_graph(model.to('cpu'), samples.to('cpu').reshape(-1, 28*28))
+writer.close()
+# sys.exit()
+
+# training loop
+num_total_steps = len(train_loader)
+
+running_loss = 0.0
+running_correct = 0
+for epoch in range(num_epochs):
+    for batch, (images, labels) in enumerate(train_loader):
+        # reshape 100, 1, 28, 28 -> 100, 784
+        images = images.reshape(-1, 28*28).to(device) # reshape and send to gpu if available
+        labels = labels.to(device)
+        model = model.to(device)
+
+        # forward
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+
+        # backward + update
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        running_loss += loss.item()
+        _, predictions = torch.max(outputs, 1)
+        running_correct += (predictions == labels).sum().item()
+        writer.add_scalar('training loss each', loss.item(), epoch * num_total_steps + batch)
+        writer.add_scalar('accuracy each', (predictions == labels).sum().item(), epoch * num_total_steps + batch)
+
+
+        if (batch+1) % 100 == 0:
+            writer.add_scalar('training loss', running_loss/100, epoch * num_total_steps + batch)
+            writer.add_scalar('accuracy', running_correct/100, epoch * num_total_steps + batch)
+            print(f'Epoch {epoch+1}/{num_epochs}, step {batch+1}/{num_total_steps}, loss = {loss.item():.4f}')
+            running_loss = 0.0
+            running_correct = 0
+
+# test
+
+b_labels = []
+b_preds = []
+with torch.no_grad():
+    n_correct = 0
+    n_samples = 0
+    for images, labels in test_loader:
+        images = images.reshape(-1, 28*28).to(device)
+        labels = labels.to(device)
+        outputs = model(images)
+
+        # value, index (index is class label)
+        _, predictions = torch.max(outputs.data, 1)
+        n_samples += labels.shape[0]
+        n_correct += (predictions==labels).sum().item()
+
+        sm = nn.Softmax(dim=0)
+        class_predictions = [sm(output) for output in outputs]
+        b_preds.append(class_predictions)
+        b_labels.append(predictions)
+
+    b_preds = torch.cat([torch.stack(batch) for batch in b_preds])
+    b_labels = torch.cat(b_labels)
+
+    acc = 100.*n_correct/n_samples
+    print(f'Accuracy = {acc}%')
+
+    classes = range(10)
+    for i in classes:
+        labels_i = b_labels == i
+        preds_i = b_preds[:,i]
+        writer.add_pr_curve(str(i), labels_i, preds_i, global_step=0)
+
+writer.close()

README.md

@@ -10,4 +10,13 @@ pyenv local 3.7.7
 source bin/activate
 ```
 
-video is in directory "Video"
+create venv:
+```
+python -m venv .
+```
+
+install requirements
+```
+python -m pip install --upgrade pip
+pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
+```

data/wine/wine.csv

@@ -0,0 +1,179 @@
+Wine,Alcohol,Malic.acid,Ash,Acl,Mg,Phenols,Flavanoids,Nonflavanoid.phenols,Proanth,Color.int,Hue,OD,Proline
+1,14.23,1.71,2.43,15.6,127,2.8,3.06,.28,2.29,5.64,1.04,3.92,1065
+1,13.2,1.78,2.14,11.2,100,2.65,2.76,.26,1.28,4.38,1.05,3.4,1050
+1,13.16,2.36,2.67,18.6,101,2.8,3.24,.3,2.81,5.68,1.03,3.17,1185
+1,14.37,1.95,2.5,16.8,113,3.85,3.49,.24,2.18,7.8,.86,3.45,1480
+1,13.24,2.59,2.87,21,118,2.8,2.69,.39,1.82,4.32,1.04,2.93,735
+1,14.2,1.76,2.45,15.2,112,3.27,3.39,.34,1.97,6.75,1.05,2.85,1450
+1,14.39,1.87,2.45,14.6,96,2.5,2.52,.3,1.98,5.25,1.02,3.58,1290
+1,14.06,2.15,2.61,17.6,121,2.6,2.51,.31,1.25,5.05,1.06,3.58,1295
+1,14.83,1.64,2.17,14,97,2.8,2.98,.29,1.98,5.2,1.08,2.85,1045
+1,13.86,1.35,2.27,16,98,2.98,3.15,.22,1.85,7.22,1.01,3.55,1045
+1,14.1,2.16,2.3,18,105,2.95,3.32,.22,2.38,5.75,1.25,3.17,1510
+1,14.12,1.48,2.32,16.8,95,2.2,2.43,.26,1.57,5,1.17,2.82,1280
+1,13.75,1.73,2.41,16,89,2.6,2.76,.29,1.81,5.6,1.15,2.9,1320
+1,14.75,1.73,2.39,11.4,91,3.1,3.69,.43,2.81,5.4,1.25,2.73,1150
+1,14.38,1.87,2.38,12,102,3.3,3.64,.29,2.96,7.5,1.2,3,1547
+1,13.63,1.81,2.7,17.2,112,2.85,2.91,.3,1.46,7.3,1.28,2.88,1310
+1,14.3,1.92,2.72,20,120,2.8,3.14,.33,1.97,6.2,1.07,2.65,1280
+1,13.83,1.57,2.62,20,115,2.95,3.4,.4,1.72,6.6,1.13,2.57,1130
+1,14.19,1.59,2.48,16.5,108,3.3,3.93,.32,1.86,8.7,1.23,2.82,1680
+1,13.64,3.1,2.56,15.2,116,2.7,3.03,.17,1.66,5.1,.96,3.36,845
+1,14.06,1.63,2.28,16,126,3,3.17,.24,2.1,5.65,1.09,3.71,780
+1,12.93,3.8,2.65,18.6,102,2.41,2.41,.25,1.98,4.5,1.03,3.52,770
+1,13.71,1.86,2.36,16.6,101,2.61,2.88,.27,1.69,3.8,1.11,4,1035
+1,12.85,1.6,2.52,17.8,95,2.48,2.37,.26,1.46,3.93,1.09,3.63,1015
+1,13.5,1.81,2.61,20,96,2.53,2.61,.28,1.66,3.52,1.12,3.82,845
+1,13.05,2.05,3.22,25,124,2.63,2.68,.47,1.92,3.58,1.13,3.2,830
+1,13.39,1.77,2.62,16.1,93,2.85,2.94,.34,1.45,4.8,.92,3.22,1195
+1,13.3,1.72,2.14,17,94,2.4,2.19,.27,1.35,3.95,1.02,2.77,1285
+1,13.87,1.9,2.8,19.4,107,2.95,2.97,.37,1.76,4.5,1.25,3.4,915
+1,14.02,1.68,2.21,16,96,2.65,2.33,.26,1.98,4.7,1.04,3.59,1035
+1,13.73,1.5,2.7,22.5,101,3,3.25,.29,2.38,5.7,1.19,2.71,1285
+1,13.58,1.66,2.36,19.1,106,2.86,3.19,.22,1.95,6.9,1.09,2.88,1515
+1,13.68,1.83,2.36,17.2,104,2.42,2.69,.42,1.97,3.84,1.23,2.87,990
+1,13.76,1.53,2.7,19.5,132,2.95,2.74,.5,1.35,5.4,1.25,3,1235
+1,13.51,1.8,2.65,19,110,2.35,2.53,.29,1.54,4.2,1.1,2.87,1095
+1,13.48,1.81,2.41,20.5,100,2.7,2.98,.26,1.86,5.1,1.04,3.47,920
+1,13.28,1.64,2.84,15.5,110,2.6,2.68,.34,1.36,4.6,1.09,2.78,880
+1,13.05,1.65,2.55,18,98,2.45,2.43,.29,1.44,4.25,1.12,2.51,1105
+1,13.07,1.5,2.1,15.5,98,2.4,2.64,.28,1.37,3.7,1.18,2.69,1020
+1,14.22,3.99,2.51,13.2,128,3,3.04,.2,2.08,5.1,.89,3.53,760
+1,13.56,1.71,2.31,16.2,117,3.15,3.29,.34,2.34,6.13,.95,3.38,795
+1,13.41,3.84,2.12,18.8,90,2.45,2.68,.27,1.48,4.28,.91,3,1035
+1,13.88,1.89,2.59,15,101,3.25,3.56,.17,1.7,5.43,.88,3.56,1095
+1,13.24,3.98,2.29,17.5,103,2.64,2.63,.32,1.66,4.36,.82,3,680
+1,13.05,1.77,2.1,17,107,3,3,.28,2.03,5.04,.88,3.35,885
+1,14.21,4.04,2.44,18.9,111,2.85,2.65,.3,1.25,5.24,.87,3.33,1080
+1,14.38,3.59,2.28,16,102,3.25,3.17,.27,2.19,4.9,1.04,3.44,1065
+1,13.9,1.68,2.12,16,101,3.1,3.39,.21,2.14,6.1,.91,3.33,985
+1,14.1,2.02,2.4,18.8,103,2.75,2.92,.32,2.38,6.2,1.07,2.75,1060
+1,13.94,1.73,2.27,17.4,108,2.88,3.54,.32,2.08,8.90,1.12,3.1,1260
+1,13.05,1.73,2.04,12.4,92,2.72,3.27,.17,2.91,7.2,1.12,2.91,1150
+1,13.83,1.65,2.6,17.2,94,2.45,2.99,.22,2.29,5.6,1.24,3.37,1265
+1,13.82,1.75,2.42,14,111,3.88,3.74,.32,1.87,7.05,1.01,3.26,1190
+1,13.77,1.9,2.68,17.1,115,3,2.79,.39,1.68,6.3,1.13,2.93,1375
+1,13.74,1.67,2.25,16.4,118,2.6,2.9,.21,1.62,5.85,.92,3.2,1060
+1,13.56,1.73,2.46,20.5,116,2.96,2.78,.2,2.45,6.25,.98,3.03,1120
+1,14.22,1.7,2.3,16.3,118,3.2,3,.26,2.03,6.38,.94,3.31,970
+1,13.29,1.97,2.68,16.8,102,3,3.23,.31,1.66,6,1.07,2.84,1270
+1,13.72,1.43,2.5,16.7,108,3.4,3.67,.19,2.04,6.8,.89,2.87,1285
+2,12.37,.94,1.36,10.6,88,1.98,.57,.28,.42,1.95,1.05,1.82,520
+2,12.33,1.1,2.28,16,101,2.05,1.09,.63,.41,3.27,1.25,1.67,680
+2,12.64,1.36,2.02,16.8,100,2.02,1.41,.53,.62,5.75,.98,1.59,450
+2,13.67,1.25,1.92,18,94,2.1,1.79,.32,.73,3.8,1.23,2.46,630
+2,12.37,1.13,2.16,19,87,3.5,3.1,.19,1.87,4.45,1.22,2.87,420
+2,12.17,1.45,2.53,19,104,1.89,1.75,.45,1.03,2.95,1.45,2.23,355
+2,12.37,1.21,2.56,18.1,98,2.42,2.65,.37,2.08,4.6,1.19,2.3,678
+2,13.11,1.01,1.7,15,78,2.98,3.18,.26,2.28,5.3,1.12,3.18,502
+2,12.37,1.17,1.92,19.6,78,2.11,2,.27,1.04,4.68,1.12,3.48,510
+2,13.34,.94,2.36,17,110,2.53,1.3,.55,.42,3.17,1.02,1.93,750
+2,12.21,1.19,1.75,16.8,151,1.85,1.28,.14,2.5,2.85,1.28,3.07,718
+2,12.29,1.61,2.21,20.4,103,1.1,1.02,.37,1.46,3.05,.906,1.82,870
+2,13.86,1.51,2.67,25,86,2.95,2.86,.21,1.87,3.38,1.36,3.16,410
+2,13.49,1.66,2.24,24,87,1.88,1.84,.27,1.03,3.74,.98,2.78,472
+2,12.99,1.67,2.6,30,139,3.3,2.89,.21,1.96,3.35,1.31,3.5,985
+2,11.96,1.09,2.3,21,101,3.38,2.14,.13,1.65,3.21,.99,3.13,886
+2,11.66,1.88,1.92,16,97,1.61,1.57,.34,1.15,3.8,1.23,2.14,428
+2,13.03,.9,1.71,16,86,1.95,2.03,.24,1.46,4.6,1.19,2.48,392
+2,11.84,2.89,2.23,18,112,1.72,1.32,.43,.95,2.65,.96,2.52,500
+2,12.33,.99,1.95,14.8,136,1.9,1.85,.35,2.76,3.4,1.06,2.31,750
+2,12.7,3.87,2.4,23,101,2.83,2.55,.43,1.95,2.57,1.19,3.13,463
+2,12,.92,2,19,86,2.42,2.26,.3,1.43,2.5,1.38,3.12,278
+2,12.72,1.81,2.2,18.8,86,2.2,2.53,.26,1.77,3.9,1.16,3.14,714
+2,12.08,1.13,2.51,24,78,2,1.58,.4,1.4,2.2,1.31,2.72,630
+2,13.05,3.86,2.32,22.5,85,1.65,1.59,.61,1.62,4.8,.84,2.01,515
+2,11.84,.89,2.58,18,94,2.2,2.21,.22,2.35,3.05,.79,3.08,520
+2,12.67,.98,2.24,18,99,2.2,1.94,.3,1.46,2.62,1.23,3.16,450
+2,12.16,1.61,2.31,22.8,90,1.78,1.69,.43,1.56,2.45,1.33,2.26,495
+2,11.65,1.67,2.62,26,88,1.92,1.61,.4,1.34,2.6,1.36,3.21,562
+2,11.64,2.06,2.46,21.6,84,1.95,1.69,.48,1.35,2.8,1,2.75,680
+2,12.08,1.33,2.3,23.6,70,2.2,1.59,.42,1.38,1.74,1.07,3.21,625
+2,12.08,1.83,2.32,18.5,81,1.6,1.5,.52,1.64,2.4,1.08,2.27,480
+2,12,1.51,2.42,22,86,1.45,1.25,.5,1.63,3.6,1.05,2.65,450
+2,12.69,1.53,2.26,20.7,80,1.38,1.46,.58,1.62,3.05,.96,2.06,495
+2,12.29,2.83,2.22,18,88,2.45,2.25,.25,1.99,2.15,1.15,3.3,290
+2,11.62,1.99,2.28,18,98,3.02,2.26,.17,1.35,3.25,1.16,2.96,345
+2,12.47,1.52,2.2,19,162,2.5,2.27,.32,3.28,2.6,1.16,2.63,937
+2,11.81,2.12,2.74,21.5,134,1.6,.99,.14,1.56,2.5,.95,2.26,625
+2,12.29,1.41,1.98,16,85,2.55,2.5,.29,1.77,2.9,1.23,2.74,428
+2,12.37,1.07,2.1,18.5,88,3.52,3.75,.24,1.95,4.5,1.04,2.77,660
+2,12.29,3.17,2.21,18,88,2.85,2.99,.45,2.81,2.3,1.42,2.83,406
+2,12.08,2.08,1.7,17.5,97,2.23,2.17,.26,1.4,3.3,1.27,2.96,710
+2,12.6,1.34,1.9,18.5,88,1.45,1.36,.29,1.35,2.45,1.04,2.77,562
+2,12.34,2.45,2.46,21,98,2.56,2.11,.34,1.31,2.8,.8,3.38,438
+2,11.82,1.72,1.88,19.5,86,2.5,1.64,.37,1.42,2.06,.94,2.44,415
+2,12.51,1.73,1.98,20.5,85,2.2,1.92,.32,1.48,2.94,1.04,3.57,672
+2,12.42,2.55,2.27,22,90,1.68,1.84,.66,1.42,2.7,.86,3.3,315
+2,12.25,1.73,2.12,19,80,1.65,2.03,.37,1.63,3.4,1,3.17,510
+2,12.72,1.75,2.28,22.5,84,1.38,1.76,.48,1.63,3.3,.88,2.42,488
+2,12.22,1.29,1.94,19,92,2.36,2.04,.39,2.08,2.7,.86,3.02,312
+2,11.61,1.35,2.7,20,94,2.74,2.92,.29,2.49,2.65,.96,3.26,680
+2,11.46,3.74,1.82,19.5,107,3.18,2.58,.24,3.58,2.9,.75,2.81,562
+2,12.52,2.43,2.17,21,88,2.55,2.27,.26,1.22,2,.9,2.78,325
+2,11.76,2.68,2.92,20,103,1.75,2.03,.6,1.05,3.8,1.23,2.5,607
+2,11.41,.74,2.5,21,88,2.48,2.01,.42,1.44,3.08,1.1,2.31,434
+2,12.08,1.39,2.5,22.5,84,2.56,2.29,.43,1.04,2.9,.93,3.19,385
+2,11.03,1.51,2.2,21.5,85,2.46,2.17,.52,2.01,1.9,1.71,2.87,407
+2,11.82,1.47,1.99,20.8,86,1.98,1.6,.3,1.53,1.95,.95,3.33,495
+2,12.42,1.61,2.19,22.5,108,2,2.09,.34,1.61,2.06,1.06,2.96,345
+2,12.77,3.43,1.98,16,80,1.63,1.25,.43,.83,3.4,.7,2.12,372
+2,12,3.43,2,19,87,2,1.64,.37,1.87,1.28,.93,3.05,564
+2,11.45,2.4,2.42,20,96,2.9,2.79,.32,1.83,3.25,.8,3.39,625
+2,11.56,2.05,3.23,28.5,119,3.18,5.08,.47,1.87,6,.93,3.69,465
+2,12.42,4.43,2.73,26.5,102,2.2,2.13,.43,1.71,2.08,.92,3.12,365
+2,13.05,5.8,2.13,21.5,86,2.62,2.65,.3,2.01,2.6,.73,3.1,380
+2,11.87,4.31,2.39,21,82,2.86,3.03,.21,2.91,2.8,.75,3.64,380
+2,12.07,2.16,2.17,21,85,2.6,2.65,.37,1.35,2.76,.86,3.28,378
+2,12.43,1.53,2.29,21.5,86,2.74,3.15,.39,1.77,3.94,.69,2.84,352
+2,11.79,2.13,2.78,28.5,92,2.13,2.24,.58,1.76,3,.97,2.44,466
+2,12.37,1.63,2.3,24.5,88,2.22,2.45,.4,1.9,2.12,.89,2.78,342
+2,12.04,4.3,2.38,22,80,2.1,1.75,.42,1.35,2.6,.79,2.57,580
+3,12.86,1.35,2.32,18,122,1.51,1.25,.21,.94,4.1,.76,1.29,630
+3,12.88,2.99,2.4,20,104,1.3,1.22,.24,.83,5.4,.74,1.42,530
+3,12.81,2.31,2.4,24,98,1.15,1.09,.27,.83,5.7,.66,1.36,560
+3,12.7,3.55,2.36,21.5,106,1.7,1.2,.17,.84,5,.78,1.29,600
+3,12.51,1.24,2.25,17.5,85,2,.58,.6,1.25,5.45,.75,1.51,650
+3,12.6,2.46,2.2,18.5,94,1.62,.66,.63,.94,7.1,.73,1.58,695
+3,12.25,4.72,2.54,21,89,1.38,.47,.53,.8,3.85,.75,1.27,720
+3,12.53,5.51,2.64,25,96,1.79,.6,.63,1.1,5,.82,1.69,515
+3,13.49,3.59,2.19,19.5,88,1.62,.48,.58,.88,5.7,.81,1.82,580
+3,12.84,2.96,2.61,24,101,2.32,.6,.53,.81,4.92,.89,2.15,590
+3,12.93,2.81,2.7,21,96,1.54,.5,.53,.75,4.6,.77,2.31,600
+3,13.36,2.56,2.35,20,89,1.4,.5,.37,.64,5.6,.7,2.47,780
+3,13.52,3.17,2.72,23.5,97,1.55,.52,.5,.55,4.35,.89,2.06,520
+3,13.62,4.95,2.35,20,92,2,.8,.47,1.02,4.4,.91,2.05,550
+3,12.25,3.88,2.2,18.5,112,1.38,.78,.29,1.14,8.21,.65,2,855
+3,13.16,3.57,2.15,21,102,1.5,.55,.43,1.3,4,.6,1.68,830
+3,13.88,5.04,2.23,20,80,.98,.34,.4,.68,4.9,.58,1.33,415
+3,12.87,4.61,2.48,21.5,86,1.7,.65,.47,.86,7.65,.54,1.86,625
+3,13.32,3.24,2.38,21.5,92,1.93,.76,.45,1.25,8.42,.55,1.62,650
+3,13.08,3.9,2.36,21.5,113,1.41,1.39,.34,1.14,9.40,.57,1.33,550
+3,13.5,3.12,2.62,24,123,1.4,1.57,.22,1.25,8.60,.59,1.3,500
+3,12.79,2.67,2.48,22,112,1.48,1.36,.24,1.26,10.8,.48,1.47,480
+3,13.11,1.9,2.75,25.5,116,2.2,1.28,.26,1.56,7.1,.61,1.33,425
+3,13.23,3.3,2.28,18.5,98,1.8,.83,.61,1.87,10.52,.56,1.51,675
+3,12.58,1.29,2.1,20,103,1.48,.58,.53,1.4,7.6,.58,1.55,640
+3,13.17,5.19,2.32,22,93,1.74,.63,.61,1.55,7.9,.6,1.48,725
+3,13.84,4.12,2.38,19.5,89,1.8,.83,.48,1.56,9.01,.57,1.64,480
+3,12.45,3.03,2.64,27,97,1.9,.58,.63,1.14,7.5,.67,1.73,880
+3,14.34,1.68,2.7,25,98,2.8,1.31,.53,2.7,13,.57,1.96,660
+3,13.48,1.67,2.64,22.5,89,2.6,1.1,.52,2.29,11.75,.57,1.78,620
+3,12.36,3.83,2.38,21,88,2.3,.92,.5,1.04,7.65,.56,1.58,520
+3,13.69,3.26,2.54,20,107,1.83,.56,.5,.8,5.88,.96,1.82,680
+3,12.85,3.27,2.58,22,106,1.65,.6,.6,.96,5.58,.87,2.11,570
+3,12.96,3.45,2.35,18.5,106,1.39,.7,.4,.94,5.28,.68,1.75,675
+3,13.78,2.76,2.3,22,90,1.35,.68,.41,1.03,9.58,.7,1.68,615
+3,13.73,4.36,2.26,22.5,88,1.28,.47,.52,1.15,6.62,.78,1.75,520
+3,13.45,3.7,2.6,23,111,1.7,.92,.43,1.46,10.68,.85,1.56,695
+3,12.82,3.37,2.3,19.5,88,1.48,.66,.4,.97,10.26,.72,1.75,685
+3,13.58,2.58,2.69,24.5,105,1.55,.84,.39,1.54,8.66,.74,1.8,750
+3,13.4,4.6,2.86,25,112,1.98,.96,.27,1.11,8.5,.67,1.92,630
+3,12.2,3.03,2.32,19,96,1.25,.49,.4,.73,5.5,.66,1.83,510
+3,12.77,2.39,2.28,19.5,86,1.39,.51,.48,.64,9.899999,.57,1.63,470
+3,14.16,2.51,2.48,20,91,1.68,.7,.44,1.24,9.7,.62,1.71,660
+3,13.71,5.65,2.45,20.5,95,1.68,.61,.52,1.06,7.7,.64,1.74,740
+3,13.4,3.91,2.48,23,102,1.8,.75,.43,1.41,7.3,.7,1.56,750
+3,13.27,4.28,2.26,20,120,1.59,.69,.43,1.35,10.2,.59,1.56,835
+3,13.17,2.59,2.37,20,120,1.65,.68,.53,1.46,9.3,.6,1.62,840
+3,14.13,4.1,2.74,24.5,96,2.05,.76,.56,1.35,9.2,.61,1.6,560

my07_linear_regression.py

@@ -0,0 +1,46 @@
+# this is a recap
+
+# a training pipeline generally consists of 3 steps:
+# 1. Design model (input, output size, forward pass (layers))
+# 2. Construct loss and optimizer
+# 3. Training loop
+#     - forward pass: compute prediction
+#     - backward pass: gradient computation
+#     - update parameters
+
+# (iterate step 3)
+
+import torch
+import torch.nn as nn
+
+X = torch.tensor([[1], [2], [3], [4]], dtype=torch.float32)
+Y = torch.tensor([[3], [6], [9], [12]], dtype=torch.float32)
+
+X_test = torch.tensor([5], dtype=torch.float32)
+
+n_samples, n_features = X.shape
+input_size = output_size = n_features
+
+learning_rate = 0.01
+n_iter = 100
+
+
+model = nn.Linear(input_size, output_size, bias=False)
+loss = nn.MSELoss()
+optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)
+
+print(f'Prediction before training: f(5) = {model(X_test).item():.3f}')
+for epoch in range(n_iter):
+    Y_pred = model(X)
+    l = loss(Y, Y_pred)
+    l.backward()
+    optimizer.step()
+    optimizer.zero_grad()
+
+    [w] = model.parameters()
+    w = w.item()
+
+    if epoch % 10 == 0:
+        print(f'Epoch {epoch}: w = {w:.3f}, loss = {l:.5f}')
+
+print(f'Prediction after training: f(5) = {model(X_test).item():.3f}')

test.py

@@ -0,0 +1,8 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+
+v = np.array([[0.1, 0.2, 0.3, -0.4, -0.1], [0.03, -0.1, 0.6, -0.4, 0.25]], dtype=np.float128)
+
+fig = plt.figure()
+ax = fig.add_subplot(111, projection='3d')