From 4d121641d11c02f62fdcfd6a5eae703b27f9ce37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joseph=20Hopfm=C3=BCller?= <hopfmuel@hm.edu>
Date: Mon, 17 Oct 2022 16:25:41 +0200
Subject: [PATCH] finish chapter 12

---
 .gitignore                                |  1 +
 02_tensors_playground.py => 02_tensors.py |  0
 10_dataset_transforms.py                  | 96 +++++++++++++++++++++++
 11_01_softmax.py                          | 17 ++++
 11_02_crossentropy.py                     | 52 ++++++++++++
 12_activation_functions.py                | 49 ++++++++++++
 6 files changed, 215 insertions(+)
 rename 02_tensors_playground.py => 02_tensors.py (100%)
 create mode 100644 10_dataset_transforms.py
 create mode 100644 11_01_softmax.py
 create mode 100644 11_02_crossentropy.py
 create mode 100644 12_activation_functions.py

diff --git a/.gitignore b/.gitignore
index 9a230a0..6d4ba14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ lib64
 share/
 pyvenv.cfg
 .python-version
+data/MNIST/
diff --git a/02_tensors_playground.py b/02_tensors.py
similarity index 100%
rename from 02_tensors_playground.py
rename to 02_tensors.py
diff --git a/10_dataset_transforms.py b/10_dataset_transforms.py
new file mode 100644
index 0000000..c5b0b35
--- /dev/null
+++ b/10_dataset_transforms.py
@@ -0,0 +1,96 @@
+'''
+Transforms can be applied to PIL images, tensors, ndarrays, or custom data
+during creation of the DataSet
+
+complete list of built-in transforms: 
+https://pytorch.org/docs/stable/torchvision/transforms.html
+
+On Images
+---------
+CenterCrop, Grayscale, Pad, RandomAffine
+RandomCrop, RandomHorizontalFlip, RandomRotation
+Resize, Scale
+
+On Tensors
+----------
+LinearTransformation, Normalize, RandomErasing
+
+Conversion
+----------
+ToPILImage: from tensor or ndrarray
+ToTensor : from numpy.ndarray or PILImage
+
+Generic
+-------
+Use Lambda 
+
+Custom
+------
+Write own class
+
+Compose multiple Transforms
+---------------------------
+composed = transforms.Compose([Rescale(256),
+                               RandomCrop(224)])
+'''
+
+import torch
+import torchvision
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+import math
+
+#example
+dataset = torchvision.datasets.MNIST(root='./data', transform=torchvision.transforms.ToTensor(), download=True)
+
+class WineDataset(Dataset):
+    def __init__(self, transform=None):
+        xy = np.loadtxt('./data/wine/wine.csv', delimiter=',', dtype=np.float32, skiprows=1)
+        self.n_samples = xy.shape[0]
+
+        self.x = xy[:,1:]  # n_samples x features
+        self.y = xy[:,[0]] # n_samples x 1
+
+        self.transform = transform
+
+    def __getitem__(self, idx):
+        sample = self.x[idx], self.y[idx]
+        if self.transform:
+            sample = self.transform(sample)
+        return sample
+
+    def __len__(self):
+        return self.n_samples
+
+class ToTensor:
+    def __call__(self, sample):
+        x, y = sample
+        return torch.from_numpy(x), torch.from_numpy(y)
+
+class MulTransform:
+    def __init__(self, factor):
+        self.factor = factor
+
+    def __call__(self, sample):
+        x, y = sample
+        x *= self.factor
+        return x, y
+
+dataset = WineDataset()
+first_data = dataset[0]
+features, labels = first_data
+print(type(features), type(labels))
+
+dataset = WineDataset(transform=ToTensor())
+first_data = dataset[0]
+features, labels = first_data
+print(features)
+print(type(features), type(labels))
+
+composed = torchvision.transforms.Compose([ToTensor(), MulTransform(4.)])
+
+dataset = WineDataset(transform=composed)
+first_data = dataset[0]
+features, labels = first_data
+print(features)
+print(type(features), type(labels))
\ No newline at end of file
diff --git a/11_01_softmax.py b/11_01_softmax.py
new file mode 100644
index 0000000..42b2cb7
--- /dev/null
+++ b/11_01_softmax.py
@@ -0,0 +1,17 @@
+# softmax squashes outputs so that the sum of the outputs equals 1 while preserving the order
+import torch
+import torch.nn as nn
+import numpy as np
+
+def softmax(x):
+    return np.exp(x)/np.sum(np.exp(x), axis=0)
+
+x = np.array([2., 1., .1])
+outputs = softmax(x)
+print('inputs: ', x)
+print('softmax numpy:', outputs)
+
+x = torch.tensor([2., 1., .1])
+outputs = torch.softmax(x, dim=0)
+print('inputs: ', x)
+print('softmax numpy:', outputs)
\ No newline at end of file
diff --git a/11_02_crossentropy.py b/11_02_crossentropy.py
new file mode 100644
index 0000000..c7a6c6f
--- /dev/null
+++ b/11_02_crossentropy.py
@@ -0,0 +1,52 @@
+# loss function for multiclass problems -> labels must be one-hot encoded, predictions is a vector of probabilities (after applying softmax)
+
+import torch
+import torch.nn as nn
+import numpy as np
+
+# numpy
+def cross_entropy(actual, predicted, normalize=False):
+    loss = -np.sum(actual * np.log(predicted))
+    if normalize:
+        loss /= float(predicted.shape[0]) 
+    return loss
+
+# y must be one-hot encoded
+# class 0 [1 0 0]
+# class 1 [0 1 0]
+# class 2 [0 0 1]
+
+Y = np.array([1, 0, 0]) # class 0
+
+# y_pred has probabilities
+Y_pred_good = np.array([.7, .2, .1])
+Y_pred_bad = np.array([.1, .3, .6])
+l1 = cross_entropy(Y, Y_pred_good)
+# l1_norm = cross_entropy(Y, Y_pred_good, normalize=True)
+l2 = cross_entropy(Y, Y_pred_bad)
+# l2_norm = cross_entropy(Y, Y_pred_bad, normalize=True)
+print(f'Loss1 numpy: {l1:.4f}')
+# print(f'Loss1 numpy normalized: {l1_norm:.4f}')
+print(f'Loss2 numpy: {l2:.4f}')
+# print(f'Loss1 numpy normalized: {l2_norm:.4f}')
+
+#pytorch
+# 3 samples
+loss = nn.CrossEntropyLoss() # includes softmax -> y_pred has raw scores, y has class labels, not one-hot
+Y = torch.tensor([2, 0, 1])
+# nsamples x nclasses = 3x3
+Y_pred_good = torch.tensor([[0.1, 1., 2.1], [2., 1., .1], [0.5, 2., .3]])
+Y_pred_bad = torch.tensor([[2.1, 1., .1], [.1, 1., 2.1], [.1, 3., .1]])
+
+l1 = loss(Y_pred_good, Y)
+l2 = loss(Y_pred_bad, Y)
+
+print(f'Loss1 pytorch: {l1.item():.4f}')
+print(f'Loss2 pytorch: {l2.item():.4f}')
+
+_, prediction1 = torch.max(Y_pred_good, 1)
+_, prediction2 = torch.max(Y_pred_bad, 1)
+print(prediction1)
+print(prediction2)
+
+
diff --git a/12_activation_functions.py b/12_activation_functions.py
new file mode 100644
index 0000000..b3548f3
--- /dev/null
+++ b/12_activation_functions.py
@@ -0,0 +1,49 @@
+# activation functions apply non-linear transform to layer output
+# without activation functions, the model would just be a stacked linear regression model -> not suited for complex tasks
+
+
+# step
+# f(x) = 1 if x>=thresh else 0
+
+# sigmoid
+# f(x) = 1/(1+exp(-x))
+# between 0 and 1, typically last layer in binary classification
+
+# tanh
+# f(x) = 2/(1+exp(-2x)) -1
+# for hidden layers
+
+# ReLU
+# f(x) = max(0,x)
+# if you don't know what to use, use ReLU ;)
+
+# Leaky ReLU
+# f(x) = x if x>=0, else a*x, a is very small
+# improved ReLU, tries to solve vanishing gradient problem (against dead neurons)
+
+# softmax
+# f(x) = exp(y_i)/sum(exp(y_i))
+# last layer in multiclass classification problem
+
+import torch
+import torch.nn as nn
+
+class NeuralNet(nn.Module):
+    def __init__(self, input_size, hidden_size):
+        super(NeuralNet, self).__init__()
+        self.layers = [
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(),
+            # nn.Sigmoid(),
+            # nn.Softmax(),
+            # nn.Tanh(),
+            # nn.LeakyReLU(),
+            nn.Linear(hidden_size, 1),
+            nn.Sigmoid()
+        ]
+
+    def forward(self, x):
+        out = x
+        for layer in self.layers:
+            out = layer(out)
+        return out
\ No newline at end of file