Machine Learning Programming Workshop
2.2E Logistic Regression in Machine Learning
Prepared By: Cheong Shiu Hong (FTFNCE)
In [1]:
import numpy as np # Linear Algebra
import pandas as pd # Data Frames
import matplotlib.pyplot as plt # Visualization
from mpl_toolkits.mplot3d import axes3d # 3D Visualization
import ipywidgets as widgets # Interactivity
from IPython.display import display # Display Widgets
import time # To Track Time
In [2]:
%matplotlib notebook
In [3]:
def sigmoid(x, grad=False):
if grad:
return sigmoid(x) * (1-sigmoid(x))
return 1/(1+np.exp(-x))
In [4]:
epsilon = 1e-10
Import Cancer Dataset from Sci-Kit Learn Library
In [5]:
import sklearn.datasets as datasets
In [6]:
cancer = datasets.load_breast_cancer()
Let's Check Out the Dataset
In [7]:
cancer.keys()
Out[7]:
In [8]:
print("Feature Names:\n", cancer['feature_names'], "\n\nLabel Names:\n", cancer['target_names'])
Shape of X and Y
In [9]:
cancer['data'].shape, cancer['target'].shape
Out[9]:
In [10]:
df = pd.DataFrame(cancer['data'], columns=cancer['feature_names'])
df.head()
Out[10]:
Split the Data</h4>
In [11]:
X = cancer['data'][:480]
Y = cancer['target'][:480]
X_val = cancer['data'][480:]
Y_val = cancer['target'][480:]
Define the Model
In [12]:
def model(theta, X):
return sigmoid(np.dot(theta, X)) # Vectorize the Calculations
Define the Training Algorithm
In [13]:
def train(x, y, learning_rate=3e-6, iterations=1, first=False):
global theta, prev_theta
prev_theta = theta
X = np.vstack([np.ones(y.shape[0]), x])
for _ in range(iterations):
# Model
pred = model(theta, X)
# Calculations for Backpropagation
error = np.mean((y * np.log(pred + epsilon)) + ((1-y) * np.log(1-pred + epsilon)), -1)
cost = - error
dcost_dtheta = np.dot(X, pred-y)
theta = theta - (dcost_dtheta * learning_rate)
class_pred = np.round(pred)
acc = np.sum(class_pred == y)/len(y)
return cost, dcost_dtheta, acc
Instantiate Random Numbers 'Theta' to be Trained
Since there are 30 Features, We will need 31 Parameters (30 Features + 1 Y-Intercept) to be Trained
In [14]:
theta = np.random.randn(X.shape[1]+1)
print(theta.shape)
Training The Parameters for 20 x 20000 Iterations
In [15]:
epochs = 20
total_time = time.time()
start = time.time()
for i in range(1, epochs+1):
lr = 4e-8 if i <= 15 else 2e-8
cost, dcost_dtheta, acc = train(X.T, Y, learning_rate=lr, iterations=20000)
print('Epoch {} - Cost: {:.3f} | Accuracy: {:.2f}%\nTime: {:.2f}s\n'.format(i, cost, acc*100, time.time()-start))
start = time.time()
print('Total Time Taken: {:.2f}s'.format(time.time()-total_time))
Evaluate the Performance of the Model
In [16]:
Xs = np.vstack([np.ones(Y.shape[0]), X.T])
modelpred = model(theta, Xs)
print(- np.mean((Y * np.log(modelpred + epsilon)) + ((1-Y) * np.log(1-modelpred + epsilon)))) # Cross Entropy Loss
print((np.round(modelpred) == Y).sum() / Y.shape[0]) # Accuracy
In [17]:
Xs = np.vstack([np.ones(Y_val.shape[0]), X_val.T])
modelpred = model(theta, Xs)
print(- np.mean((Y_val * np.log(modelpred + epsilon)) + ((1-Y_val) * np.log(1-modelpred + epsilon)))) # Cross Entropy Loss
print((np.round(modelpred) == Y_val).sum() / Y_val.shape[0]) # Accuracy
Sci-Kit Learn is a Powerful Python Library that has Many Built-In Machine Learning Algorithms
Import Sklearn's Logistic Regression Object from sklearn.linear_model
In [18]:
from sklearn.linear_model import LogisticRegression
Instantiate Linear Regression Object
In [19]:
model = LogisticRegression(solver='liblinear')
Fit Model to Data
In [20]:
model.fit(X, Y)
Out[20]:
Evaluate Score of Fitted Model
In [21]:
model.score(X, Y)
Out[21]:
In [22]:
model.score(X_val, Y_val)
Out[22]:
Evaluate Cross Entropy Loss of Fitted Model
In [23]:
skpred = model.predict(X)
- np.mean((Y * np.log(skpred + epsilon)) + ((1-Y) * np.log(1-skpred + epsilon))) # Cross Entropy Loss
Out[23]:
In [24]:
skpred = model.predict(X_val)
- np.mean((Y_val * np.log(skpred + epsilon)) + ((1-Y_val) * np.log(1-skpred + epsilon))) # Cross Entropy Loss
Out[24]:
Bootstrap Aggregating (Bagging)
In [25]:
num_bags = 250
bag_size = 250
In [26]:
bags = []
for i in range(num_bags):
idx = np.random.choice(np.arange(X.shape[0]), bag_size)
bags.append([X[idx], Y[idx]])
In [27]:
models = []
for bag in bags:
models.append(LogisticRegression(solver='liblinear'))
models[-1].fit(bag[0], bag[1])
In [28]:
skpreds = []
for model in models:
skpreds.append(model.predict(X))
avg_preds = np.array(skpreds).mean(0)
print(- np.mean((Y * np.log(avg_preds + epsilon)) + ((1-Y) * np.log(1-avg_preds + epsilon)))) # Cross Entropy Loss
print((np.round(avg_preds) == Y).sum() / Y.shape[0])
In [29]:
skpreds = []
for model in models:
skpreds.append(model.predict(X_val))
avg_preds = np.array(skpreds).mean(0)
print(- np.mean((Y_val * np.log(avg_preds + epsilon)) + ((1-Y_val) * np.log(1-avg_preds + epsilon)))) # Cross Entropy Loss
print((np.round(avg_preds) == Y_val).sum() / Y_val.shape[0])
Naive Bayes
In [30]:
from sklearn.naive_bayes import GaussianNB
In [31]:
NB = GaussianNB()
In [32]:
NB.fit(X, Y)
Out[32]:
In [33]:
NB.score(X, Y)
Out[33]:
Decision Trees
In [34]:
from sklearn.tree import DecisionTreeClassifier
In [35]:
dec_tree = DecisionTreeClassifier()
In [36]:
dec_tree.fit(X, Y)
Out[36]:
In [37]:
dec_tree.score(X, Y)
Out[37]:
Support Vector Machines
In [38]:
from sklearn.svm import SVC
In [39]:
SVM1 = SVC(kernel='linear')
SVM2 = SVC()
In [40]:
SVM1.fit(X, Y)
SVM2.fit(X, Y)
Out[40]:
In [41]:
SVM1.score(X, Y), SVM2.score(X, Y)
Out[41]:
Ensemble Algorithms
In [42]:
from sklearn.ensemble import RandomForestClassifier
In [43]:
RFC = RandomForestClassifier()
In [44]:
RFC.fit(X, Y)
Out[44]:
In [45]:
RFC.score(X, Y)
Out[45]:
Previous:
Next: