From b5ab293a9b88eab28a273ef3634a73be5fc40aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joseph=20Hopfm=C3=BCller?= Date: Sun, 16 Oct 2022 23:10:06 +0200 Subject: [PATCH] add readme, chapter 03 finished --- 01_tensors/01_tensors_playground.py | 1 - 02_tensors_playground.py | 166 ++++++++++++++++++++++++++++ 03_autograd.py | 83 ++++++++++++++ 04_backpropagation.py | 0 README.md | 4 + 5 files changed, 253 insertions(+), 1 deletion(-) delete mode 100644 01_tensors/01_tensors_playground.py create mode 100644 02_tensors_playground.py create mode 100644 03_autograd.py create mode 100644 04_backpropagation.py create mode 100644 README.md diff --git a/01_tensors/01_tensors_playground.py b/01_tensors/01_tensors_playground.py deleted file mode 100644 index 53df570..0000000 --- a/01_tensors/01_tensors_playground.py +++ /dev/null @@ -1 +0,0 @@ - pip \ No newline at end of file diff --git a/02_tensors_playground.py b/02_tensors_playground.py new file mode 100644 index 0000000..85c6829 --- /dev/null +++ b/02_tensors_playground.py @@ -0,0 +1,166 @@ +import torch + +# empty, zeros, ones of different sizes, specify datatype +print('empty, zeros, ones of different sizes, specify datatype') +x = torch.empty(1) +print(x) +x = torch.empty(3) +print(x) +x = torch.zeros(2, 3) +print(x) +x = torch.ones(2,3,4) +print(x) +x = torch.ones(2, 5, dtype=torch.float64) +print(x.dtype) +print() + +# from data +print('from data') +x = torch.tensor([2.5, 0.1]) +print(x) +print() + +#basic ops +print('basic ops') +x = torch.rand(2,2) +y = torch.rand(2,2) + +print('add') +z1 = x + y +z2 = torch.add(x,y) +print(x) +print(y) +print(z1) +print(z2) +# in place addition +x.add_(y) +print(x) + +print('sub') +z1 = x - y +z2 = torch.sub(x,y) +print(x) +print(y) +print(z1) +print(z2) +# in place addition +x.sub_(y) +print(x) + +print('mul') +z1 = x * y +z2 = torch.mul(x,y) +print(x) +print(y) +print(z1) +print(z2) +# in place addition +x.mul_(y) +print(x) + +print('div') +z1 = x / y +z2 = torch.div(x,y) +print(x) +print(y) +print(z1) +print(z2) +# in place addition +x.div_(y) +print(x) + +print() + +#slicing +print('slicing, item') +x = torch.rand(2,3,2) +print(x) +print(x[:,2,:]) +print(x[1, 2, 1]) +print(x[1, 2, 1].item()) # for single element tensors only + +print() + +# reshaping +print('reshaping') + +x = torch.rand(4,4) +print(x) +y = x.view(16) +print(y) +y = x.view(-1, 8) +print(y) +y = x.view(2, -1) +print(y) + +# y = x.view(3, -1) # fails 'shape is invalid' +# print(y) + +print() + +# transposing +print('transposing') +x = torch.rand(2, 3) +print(x.size()) +x = torch.transpose(x, 0, 1) +print(x.size()) +x = torch.t(x) +print(x.size()) +x = torch.rand(2, 3, 4) +print(f'Original: {x.size()}') +x = torch.transpose(x, 0, 1) +print(f'01: {x.size()}') +x = torch.rand(2, 3, 4) +x = torch.transpose(x, 1, 2) +print(f'12: {x.size()}') +x = torch.rand(2, 3, 4) +x = torch.transpose(x, 0, 2) +print(f'02: {x.size()}') + +print() + +# numpy +import numpy as np +print('numpy') + +a = torch.ones(5) +print(a) +b = a.numpy() +print(b) +print(type(b)) + +a.add_(1) # vectors/tensors share same memory +print(a) +print(b) + +c = np.ones(5) +print(c) +d = torch.from_numpy(c) +print(d) +e = d.to(dtype=torch.float32) # e has its own memory +print(e) + +c += 1 # c and d share same memory +print(c) +print(d) +print(e) + +# device +print('device') +if torch.cuda.is_available(): + device = torch.device('cuda') +else: + device = torch.device('cpu') +print(device) +x = torch.ones(5, device=device) +y = torch.ones(5) +y = y.to(device) +z = x+y +print(z) +z = z.to('cpu') #move tensor back to cpu for conversion into numopy vector +print(z) +a = z.numpy() +print(a) + +x = torch.ones(5, requires_grad=True) # enable grad for autograd +print(x) \ No newline at end of file diff --git a/03_autograd.py b/03_autograd.py new file mode 100644 index 0000000..186935d --- /dev/null +++ b/03_autograd.py @@ -0,0 +1,83 @@ +import torch + +x = torch.tensor([1.0,2.0,3.0], requires_grad=True) +print(x) + +y = x+2 # a function used in backprop for calculating the gradient is created +# y.retain_grad() # for getting grad of y (a non-leaf tensor) +print(y) + +z = y*y*2 +z = y.mean() +print(z) + +z.backward() # no argument needed because z is scalar -> will calculate the gradient pretty accurately +# print(y.grad) +print(x.grad) + +z = y*y*2 +print(z) +# z.backward() will fail because z is not scalar -> create vector vor Jacobian-Vector product (JVP) +# you have to specify the step size for the gradient approximation +# (calculation via chain rule Jacobian * vector = gradient vector) vector is size of step for each element -> very small elements approximate the gradient well +v = torch.tensor([0.000000001, 0.000000001, 0.000000001], dtype=torch.float32) +z.backward(v) # pass vector to JVP +print(x.grad) + + +# prevent operation from being tracked by gradient tracking (requires_grad) +# 3 options +# 1. x.requires_grad_(False) -> turn off requires_grad completely +# 2. x.detach() -> returns new tensor without requires_grad +# 3. with torch.no_grad(): -> lets you do operations without grad tracking temporarily + +x = torch.tensor([1.0,2.0,3.0], requires_grad=True) +y = x*x +print(x) + +# 1 +x.requires_grad_(False) +print(x) + +x = torch.tensor([1.0,2.0,3.0], requires_grad=True) +y = x*x +print(x) + +# 2 +z = x.detach() +print(z) + +x = torch.tensor([1.0,2.0,3.0], requires_grad=True) +y = x*x +print(x) + +#3 +with torch.no_grad(): + a = x+2 + print(a) + +b = x+2 +print(b) + + +# gradients will be summed up! -> empty gradients + +#this is a dummy training +weights = torch.ones(4, requires_grad=True) +for epoch in range(3): + model_output = (weights*3).sum() + model_output.backward() + print(weights.grad) + weights.grad.zero_()# clear gradients + + +#later +optimizer = torch.optim.SGD(weights, lr=0.01) # stochastic gradient descent +optimizer.step() +optimizer.zero_grad() # clear gradients + +# RECAP +# turn on gradient tracking for interesting vectors (f(x) = x², f'(x) = ? -> requires_grad=True for x) +# calculate gradient with f.backward(), specify step size for vectors (not needed for scalar functions like mean()) +# clear gradients with x.grad.zero_() +# prevent operations from being tracked in the comp graph with one of the 3 options above \ No newline at end of file diff --git a/04_backpropagation.py b/04_backpropagation.py new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md new file mode 100644 index 0000000..9c3c801 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# pytorch playground +I'm working through the video [Deep Learning with PyTorch - Full Course](https://www.youtube.com/watch?v=c36lUUr864M) by Python Engineer + +the file names match the chapters in the video \ No newline at end of file