Indian Institute of Technology Bombay

EE 769:- Introduction to Machine Learning

Assignment 1:- Linear Regression

INPUT:-

#EE-769 Linear Regression Assignment

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Q1. Write a function to generate a data matrix X. Inputs:

Number of samples, feature dimension. Output: Data matrix X.

# N- no of samples , D- Feature Dimensions

def input_data(N,D):
x=np.random.rand(N,D)
x0=np.ones((N,1))
x=np.concatenate((x,x0),axis=1)
return x

N=10 #no of samples

D=5 #Feature dimensions
x=input_data(N,D)
print("x = ",x)
print("\n")

# Q2. Write a function to generated dependent variable column t.

from numpy.core.multiarray import dot

def target_vector(x,w0,noise_Var,N,D):
w=np.random.rand(D,1)# Assigning random weights
arr=[[w0]]
w=np.concatenate((w,arr),axis=0)
SD=np.sqrt(noise_Var)
noise = np.random.normal(0,SD,(N,1))
t=x@w+noise
return t,w;

#target_vector(x,w0,noise_Var,N,D)
t,w=target_vector(x,0.5,4,N,D)
print("t = ",t)
print("\n")
print("w = ",w)
print("\n")

# Q.3 Write a function to compute a linear regression estimate

def y_vector(x,w):
y=x@w
return y

y=y_vector(x,w)
print("y = ",y)
print("\n")

# Q.4 Write a function to compute the mean square error of two

vectors y and t

def MSE_yt(y,t,N):
sub=np.subtract(y,t)
sq=np.square(sub)
MSE=sq/N
return MSE

L=MSE_yt(y,t,N)*0.5 #Loss matrix

print("L = ",L)
print("\n")

# Q.5 Write a function to estimate the weights of linear

regression using pseudo-inverse, assuming L2 regularization

def W_new(x,t,l):
I=np.identity(D+1)*l
W=np.linalg.inv(I+(x.T)@x)@(x.T)@t
y=x@w
MSE=MSE_yt(y,t,N)
return w,y,MSE

l=0.5
w,y,MSE=W_new(x,t,l)
print("w = ",w)
print("\n")
print("y = ",y)
print("\n")
print("MSE = ",MSE)
print("\n")

# Q.6 Write a function to compute the gradient of MSE with

respect to its weight vector.

def Grad_L(t,w,x):
Del_l=(x.T)@(t-x@w)
return Del_l

Del_L=Grad_L(t,w,x)
print ("Del_L = ",Del_L)
print("\n")

# Q.7 Write a function to compute L2 norm of a vector w passed

as a numpy array. Exclude bias w0

def W_L2_norm(w):
W_L2=np.sqrt(np.sum(np.square(w)[0:(D-1)])) #w0 (bias)
excluded
return W_L2

W_L2=W_L2_norm(w)
print ("W_L2 = ",W_L2)
print("\n")

# Q.8 Write a function to compute the gradient of L2 norm with

respect to the weight vectors

def Grad_W_L2_norm(w, W_L2):

del_W_L2 = W_L2 * w
del_W_L2[D] = 0 # Making Grad of w0=0
return del_W_L2

del_W_L2=Grad_W_L2_norm(w,W_L2)
print ("del_W_L2 = ",del_W_L2)
print("\n")

# Q9. Write a function to compute L1 norm of a vector w passed

as a numpy array. Exclude bias w0.
def W_L1_norm(w):
W_L1=np.sum(np.absolute(w[0:D-1]))
return (W_L1)

W_L1=W_L1_norm(w)
print("W_L1 = ",W_L1)
print("\n")

# Q.10 Write a function to compute the gradient of L1 norm with

respect to the weight vectors.

def grad_W_L1_norm(w):
del_W_L1 = w / np.absolute(w)
del_W_L1[D] = 0
return del_W_L1

del_W_L1=grad_W_L1_norm(w)
print ("del_W_L1 = ",del_W_L1)
print("\n")

# Q.11 Write a function for a single update of weights of linear

regression using gradient descent

def GD_one(x, t, w, eta, lambda2, lambda1):

w = w - (eta * Grad_L(t, w, x) / N) + lambda2 *
Grad_W_L2_norm(w, W_L2) + lambda1 * grad_W_L1_norm(w)
y = y_vector(x, w)
MSE = MSE_yt(y, t, N)
return w, y, MSE

eta=0.2
lambda2=1
lambda1=1
w_new,y,MSE=GD_one(x,t,w,eta,lambda2,lambda1)

# Q.12 Write a function to estimate the weights of linear

regression using gradient descent

def GD(x, t, w, eta, lambda2, lambda1, i0, Epsilon):

i = 0
 while (i < i0):
i = i + 1
w_new, y, MSE = GD_one(x, t, w, eta, lambda2, lambda1)
if np.all(w_new - w) <= Epsilon:
break
else:
w = w_new

return w_new, y, MSE

eta=0.1
lambda2=0.1
lambda1=0
i0=100
Epsilon=0.001
w_new,y,MSE=GD(x,t,w,eta,lambda2,lambda1,i0,Epsilon)
print ("w_new",w_new)
print("\n")
print("y = ",y)
print("\n")
print("MSE = ",MSE)
print("\n")
OUTPUT:
x = [[0.49727164 0.62536822 0.06039567 0.38703761 0.38356686 1. ]

[0.91759458 0.25176455 0.29456041 0.9094563 0.44989806 1. ]

[0.7315654 0.67255988 0.76384126 0.5470926 0.6640514 1. ]

[0.17047664 0.58103804 0.13727726 0.70271317 0.26275524 1. ]

[0.63540456 0.24596158 0.20745636 0.42262781 0.25438696 1. ]

[0.22536266 0.36838544 0.97251346 0.4021511 0.54623673 1. ]

[0.53809661 0.58853961 0.81652386 0.46744697 0.52552012 1. ]

[0.64617418 0.91916378 0.51057744 0.6973501 0.6575601 1. ]

[0.16723136 0.48360142 0.73049935 0.86319226 0.63443998 1. ]

[0.68468966 0.36020996 0.3994224 0.81448899 0.5014732 1. ]]

t = [[ 3.4024813 ]

[ 3.1414626 ]

[ 2.48702671]

[-3.96240527]

[-0.36140521]

[ 0.79444583]

[ 5.98651264]

[ 4.86646249]

[ 3.25520315]

[ 1.5212192 ]]
w = [[0.91392294]

[0.33402626]

[0.99494568]

[0.29386997]

[0.65354499]

[0.5 ]]

y = [[1.5878647 ]

[2.27706883]

[2.74798917]

[1.36469656]

[1.65972719]

[2.27178298]

[2.48158308]

[2.54025042]

[2.20948067]

[2.21056591]]

L = [[0.16464167]

[0.03735883]

[0.00340507]

[1.4189007 ]

[0.20424881]

[0.10912625]
[0.61422656]

[0.27056313]

[0.05467678]

[0.02375994]]

w = [[0.91392294]

[0.33402626]

[0.99494568]

[0.29386997]

[0.65354499]

[0.5 ]]

y = [[1.5878647 ]

[2.27706883]

[2.74798917]

[1.36469656]

[1.65972719]

[2.27178298]

[2.48158308]

[2.54025042]

[2.20948067]

[2.21056591]]
MSE = [[0.32928334]

[0.07471766]

[0.00681014]

[2.83780139]

[0.40849761]

[0.21825251]

[1.22845312]

[0.54112626]

[0.10935355]

[0.04751989]]

Del_L = [[ 2.07130747]

[ 1.49867821]

[ 2.11568938]

[-0.24429438]

[ 1.88006843]

[-0.22000607]]

W_L2 = 1.4223590091474934
del_W_L2 = [[1.29992653]

[0.47510525]

[1.41516995]

[0.41798859]

[0.9295756 ]

[0. ]]

W_L1 = 2.5367648454778853

del_W_L1 = [[1.]

[1.]

[1.]

[1.]

[1.]

[0.]]

w_new = [[1.28235276e+12]

[1.25227092e+12]

[1.23766618e+12]

[1.51846020e+12]

[1.21639183e+12]

[1.40854640e+12]]
y = [[3.95837297e+12]

[5.19329674e+12]

[5.77276400e+12]

[3.91133336e+12]

[3.73930918e+12]

[4.63759454e+12]

[5.19520908e+12]

[5.87888524e+12]

[5.21516103e+12]

[5.07874914e+12]]

MSE = [[1.56687166e+24]

[2.69703311e+24]

[3.33248042e+24]

[1.52985287e+24]

[1.39824332e+24]

[2.15072831e+24]

[2.69901974e+24]

[3.45612917e+24]

[2.71979046e+24]

[2.57936928e+24]]