add readme, chapter 03 finished
This commit is contained in:
Joseph Hopfmüller
@@ -1 +0,0 @@
|
||||
pip
|
||||
166
02_tensors_playground.py
Normal file
166
02_tensors_playground.py
Normal file
@@ -0,0 +1,166 @@
|
||||
import torch
|
||||
|
||||
# empty, zeros, ones of different sizes, specify datatype
|
||||
print('empty, zeros, ones of different sizes, specify datatype')
|
||||
x = torch.empty(1)
|
||||
print(x)
|
||||
x = torch.empty(3)
|
||||
print(x)
|
||||
x = torch.zeros(2, 3)
|
||||
print(x)
|
||||
x = torch.ones(2,3,4)
|
||||
print(x)
|
||||
x = torch.ones(2, 5, dtype=torch.float64)
|
||||
print(x.dtype)
|
||||
print()
|
||||
|
||||
# from data
|
||||
print('from data')
|
||||
x = torch.tensor([2.5, 0.1])
|
||||
print(x)
|
||||
print()
|
||||
|
||||
#basic ops
|
||||
print('basic ops')
|
||||
x = torch.rand(2,2)
|
||||
y = torch.rand(2,2)
|
||||
|
||||
print('add')
|
||||
z1 = x + y
|
||||
z2 = torch.add(x,y)
|
||||
print(x)
|
||||
print(y)
|
||||
print(z1)
|
||||
print(z2)
|
||||
# in place addition
|
||||
x.add_(y)
|
||||
print(x)
|
||||
|
||||
print('sub')
|
||||
z1 = x - y
|
||||
z2 = torch.sub(x,y)
|
||||
print(x)
|
||||
print(y)
|
||||
print(z1)
|
||||
print(z2)
|
||||
# in place addition
|
||||
x.sub_(y)
|
||||
print(x)
|
||||
|
||||
print('mul')
|
||||
z1 = x * y
|
||||
z2 = torch.mul(x,y)
|
||||
print(x)
|
||||
print(y)
|
||||
print(z1)
|
||||
print(z2)
|
||||
# in place addition
|
||||
x.mul_(y)
|
||||
print(x)
|
||||
|
||||
print('div')
|
||||
z1 = x / y
|
||||
z2 = torch.div(x,y)
|
||||
print(x)
|
||||
print(y)
|
||||
print(z1)
|
||||
print(z2)
|
||||
# in place addition
|
||||
x.div_(y)
|
||||
print(x)
|
||||
|
||||
print()
|
||||
|
||||
#slicing
|
||||
print('slicing, item')
|
||||
x = torch.rand(2,3,2)
|
||||
print(x)
|
||||
print(x[:,2,:])
|
||||
print(x[1, 2, 1])
|
||||
print(x[1, 2, 1].item()) # for single element tensors only
|
||||
|
||||
print()
|
||||
|
||||
# reshaping
|
||||
print('reshaping')
|
||||
|
||||
x = torch.rand(4,4)
|
||||
print(x)
|
||||
y = x.view(16)
|
||||
print(y)
|
||||
y = x.view(-1, 8)
|
||||
print(y)
|
||||
y = x.view(2, -1)
|
||||
print(y)
|
||||
|
||||
# y = x.view(3, -1) # fails 'shape is invalid'
|
||||
# print(y)
|
||||
|
||||
print()
|
||||
|
||||
# transposing
|
||||
print('transposing')
|
||||
x = torch.rand(2, 3)
|
||||
print(x.size())
|
||||
x = torch.transpose(x, 0, 1)
|
||||
print(x.size())
|
||||
x = torch.t(x)
|
||||
print(x.size())
|
||||
x = torch.rand(2, 3, 4)
|
||||
print(f'Original: {x.size()}')
|
||||
x = torch.transpose(x, 0, 1)
|
||||
print(f'01: {x.size()}')
|
||||
x = torch.rand(2, 3, 4)
|
||||
x = torch.transpose(x, 1, 2)
|
||||
print(f'12: {x.size()}')
|
||||
x = torch.rand(2, 3, 4)
|
||||
x = torch.transpose(x, 0, 2)
|
||||
print(f'02: {x.size()}')
|
||||
|
||||
print()
|
||||
|
||||
# numpy
|
||||
import numpy as np
|
||||
print('numpy')
|
||||
|
||||
a = torch.ones(5)
|
||||
print(a)
|
||||
b = a.numpy()
|
||||
print(b)
|
||||
print(type(b))
|
||||
|
||||
a.add_(1) # vectors/tensors share same memory
|
||||
print(a)
|
||||
print(b)
|
||||
|
||||
c = np.ones(5)
|
||||
print(c)
|
||||
d = torch.from_numpy(c)
|
||||
print(d)
|
||||
e = d.to(dtype=torch.float32) # e has its own memory
|
||||
print(e)
|
||||
|
||||
c += 1 # c and d share same memory
|
||||
print(c)
|
||||
print(d)
|
||||
print(e)
|
||||
|
||||
# device
|
||||
print('device')
|
||||
if torch.cuda.is_available():
|
||||
device = torch.device('cuda')
|
||||
else:
|
||||
device = torch.device('cpu')
|
||||
print(device)
|
||||
x = torch.ones(5, device=device)
|
||||
y = torch.ones(5)
|
||||
y = y.to(device)
|
||||
z = x+y
|
||||
print(z)
|
||||
z = z.to('cpu') #move tensor back to cpu for conversion into numopy vector
|
||||
print(z)
|
||||
a = z.numpy()
|
||||
print(a)
|
||||
|
||||
x = torch.ones(5, requires_grad=True) # enable grad for autograd
|
||||
print(x)
|
||||
83
03_autograd.py
Normal file
83
03_autograd.py
Normal file
@@ -0,0 +1,83 @@
|
||||
import torch
|
||||
|
||||
x = torch.tensor([1.0,2.0,3.0], requires_grad=True)
|
||||
print(x)
|
||||
|
||||
y = x+2 # a function used in backprop for calculating the gradient is created
|
||||
# y.retain_grad() # for getting grad of y (a non-leaf tensor)
|
||||
print(y)
|
||||
|
||||
z = y*y*2
|
||||
z = y.mean()
|
||||
print(z)
|
||||
|
||||
z.backward() # no argument needed because z is scalar -> will calculate the gradient pretty accurately
|
||||
# print(y.grad)
|
||||
print(x.grad)
|
||||
|
||||
z = y*y*2
|
||||
print(z)
|
||||
# z.backward() will fail because z is not scalar -> create vector vor Jacobian-Vector product (JVP)
|
||||
# you have to specify the step size for the gradient approximation
|
||||
# (calculation via chain rule Jacobian * vector = gradient vector) vector is size of step for each element -> very small elements approximate the gradient well
|
||||
v = torch.tensor([0.000000001, 0.000000001, 0.000000001], dtype=torch.float32)
|
||||
z.backward(v) # pass vector to JVP
|
||||
print(x.grad)
|
||||
|
||||
|
||||
# prevent operation from being tracked by gradient tracking (requires_grad)
|
||||
# 3 options
|
||||
# 1. x.requires_grad_(False) -> turn off requires_grad completely
|
||||
# 2. x.detach() -> returns new tensor without requires_grad
|
||||
# 3. with torch.no_grad(): -> lets you do operations without grad tracking temporarily
|
||||
|
||||
x = torch.tensor([1.0,2.0,3.0], requires_grad=True)
|
||||
y = x*x
|
||||
print(x)
|
||||
|
||||
# 1
|
||||
x.requires_grad_(False)
|
||||
print(x)
|
||||
|
||||
x = torch.tensor([1.0,2.0,3.0], requires_grad=True)
|
||||
y = x*x
|
||||
print(x)
|
||||
|
||||
# 2
|
||||
z = x.detach()
|
||||
print(z)
|
||||
|
||||
x = torch.tensor([1.0,2.0,3.0], requires_grad=True)
|
||||
y = x*x
|
||||
print(x)
|
||||
|
||||
#3
|
||||
with torch.no_grad():
|
||||
a = x+2
|
||||
print(a)
|
||||
|
||||
b = x+2
|
||||
print(b)
|
||||
|
||||
|
||||
# gradients will be summed up! -> empty gradients
|
||||
|
||||
#this is a dummy training
|
||||
weights = torch.ones(4, requires_grad=True)
|
||||
for epoch in range(3):
|
||||
model_output = (weights*3).sum()
|
||||
model_output.backward()
|
||||
print(weights.grad)
|
||||
weights.grad.zero_()# clear gradients
|
||||
|
||||
|
||||
#later
|
||||
optimizer = torch.optim.SGD(weights, lr=0.01) # stochastic gradient descent
|
||||
optimizer.step()
|
||||
optimizer.zero_grad() # clear gradients
|
||||
|
||||
# RECAP
|
||||
# turn on gradient tracking for interesting vectors (f(x) = x², f'(x) = ? -> requires_grad=True for x)
|
||||
# calculate gradient with f.backward(), specify step size for vectors (not needed for scalar functions like mean())
|
||||
# clear gradients with x.grad.zero_()
|
||||
# prevent operations from being tracked in the comp graph with one of the 3 options above
|
||||
0
04_backpropagation.py
Normal file
0
04_backpropagation.py
Normal file
Reference in New Issue
Block a user