In [1]:
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
Naive Bayes is a classic machine learning algorithm which is basic, but one of the very efficient algorithm to identify the images. The simple use of Bayes theorem can help us classify the images. This machine learning algorithm is purely statistical and uses probabality calculations.
In [2]:
class NaiveBayes(object):
def __init__(self, num_class, feature_dim, num_value):
"""Initialize a naive bayes model.
This function will initialize prior and likelihood, where
prior is P(class) with a dimension of (# of class,)
that estimates the empirical frequencies of different classes in the training set.
likelihood is P(F_i = f | class) with a dimension of
(# of features/pixels per image, # of possible values per pixel, # of class),
that computes the probability of every pixel location i being value f for every class label.
Args:
num_class(int): number of classes to classify
feature_dim(int): feature dimension for each example
num_value(int): number of possible values for each pixel
"""
self.num_value = num_value
self.num_class = num_class
self.feature_dim = feature_dim
self.prior = np.zeros((num_class))
self.likelihood = np.zeros((feature_dim, num_value, num_class))
def train(self, train_set, train_label):
""" Train naive bayes model (self.prior and self.likelihood) with training dataset.
self.prior(numpy.ndarray): training set class prior (in log) with a dimension of (# of class,),
self.likelihood(numpy.ndarray): traing set likelihood (in log) with a dimension of
(# of features/pixels per image, # of possible values per pixel, # of class).
You should apply Laplace smoothing to compute the likelihood.
Args:
train_set(numpy.ndarray): training examples with a dimension of (# of examples, feature_dim)
train_label(numpy.ndarray): training labels with a dimension of (# of examples, )
"""
self.calculate_prior_probability(train_set, train_label)
# print("prior calculated")
class_count = self.calculate_class_counts(train_label)
# print("count calculated")
self.calculate_likelihood(train_set, train_label, class_count)
def test(self, test_set, test_label):
""" Test the trained naive bayes model (self.prior and self.likelihood) on testing dataset,
by performing maximum a posteriori (MAP) classification.
The accuracy is computed as the average of correctness
by comparing between predicted label and true label.
Args:
test_set(numpy.ndarray): testing examples with a dimension of (# of examples, feature_dim)
test_label(numpy.ndarray): testing labels with a dimension of (# of examples, )
Returns:
accuracy(float): average accuracy value
pred_label(numpy.ndarray): predicted labels with a dimension of (# of examples, )
"""
accuracy = 0
pred_label = np.zeros((len(test_set)))
count = 0
for i in range(len(test_set)):
test_image = test_set[i]
actual_label = test_label[i]
posterior_probabilites = np.zeros(self.num_class)
for c in range(self.num_class):
probability = 0
for j in range(len(test_image)):
current_likelihood = self.likelihood[j][test_image[j]][c]
probability += current_likelihood
probability += np.log(self.prior[c])
posterior_probabilites[c] = probability
predicted_label = np.argmax(posterior_probabilites)
pred_label[i] = predicted_label
if actual_label == predicted_label:
count = count + 1
accuracy = count / len(test_set)
return accuracy, pred_label
def intensity_feature_likelihoods(self, likelihood):
"""
Get the feature likelihoods for high intensity pixels for each of the classes,
by sum the probabilities of the top 128 intensities at each pixel location,
sum k<-128:255 P(F_i = k | c).
This helps generate visualization of trained likelihood images.
Args:
likelihood(numpy.ndarray): likelihood (in log) with a dimension of
(# of features/pixels per image, # of possible values per pixel, # of class)
Returns:
feature_likelihoods(numpy.ndarray): feature likelihoods for each class with a dimension of
(# of features/pixels per image, # of class)
"""
feature_likelihoods = np.zeros((likelihood.shape[0], likelihood.shape[2]))
# of features/pixels per image, # of possible values per pixel, # of class
for i in range(likelihood.shape[0]):
for j in range(likelihood.shape[2]):
sum = 0
for k in range(128,255):
sum = sum + likelihood[i][k][j]
feature_likelihoods[i][j] = sum
return feature_likelihoods
def calculate_prior_probability(self, train_set, train_label):
unique, counts = np.unique(train_label, return_counts=True)
prior_counts = dict(zip(unique, counts))
length_set = len(train_label)
for i in range(self.num_class):
self.prior[i] = prior_counts[i] / length_set
def calculate_likelihood(self, train_set, train_label, class_counts):
laplace = 0.1
for i in range(len(train_set)):
for j in range(self.feature_dim):
self.likelihood[j][train_set[i][j]][train_label[i]] += 1
for i, value in np.ndenumerate(self.likelihood):
counts = self.likelihood[i[0]][i[1]][i[2]]
self.likelihood[i[0]][i[1]][i[2]] = np.log(((counts + laplace) / (laplace * self.num_value + class_counts[i[2]])))
def calculate_class_counts(self, train_label):
unique, counts = np.unique(train_label, return_counts=True)
class_counts = dict(zip(unique, counts))
return class_counts
In [3]:
def load_dataset(data_dir=''):
"""Load the train and test examples
"""
x_train = np.load("./data/x_train.npy")
y_train = np.load("./data/y_train.npy")
x_test = np.load("./data/x_test.npy")
y_test = np.load("./data/y_test.npy")
return x_train, y_train, x_test, y_test
In [4]:
def plot_visualization(images, classes, cmap):
"""Plot the visualizations
"""
fig, ax = plt.subplots(2, 5, figsize=(12, 5))
for i in range(10):
ax[i%2, i//2].imshow(images[:, i].reshape((28, 28)), cmap=cmap)
ax[i%2, i//2].set_xticks([])
ax[i%2, i//2].set_yticks([])
ax[i%2, i//2].set_title(classes[i])
plt.show()
In [5]:
def plot_confusion_matrix(y_true, y_pred, classes,
normalize=False,
title=None,
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
if not title:
if normalize:
title = 'Normalized confusion matrix'
else:
title = 'Confusion matrix, without normalization'
# Compute confusion matrix
cm = confusion_matrix(y_true, y_pred)
# Only use the labels that appear in the data
classes = classes[unique_labels(y_true, y_pred)]
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
fig, ax = plt.subplots()
im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
ax.figure.colorbar(im, ax=ax)
# We want to show all ticks...
ax.set(xticks=np.arange(cm.shape[1]),
yticks=np.arange(cm.shape[0]),
# ... and label them with the respective list entries
xticklabels=classes, yticklabels=classes,
title=title,
ylabel='True label',
xlabel='Predicted label')
# Rotate the tick labels and set their alignment.
plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
rotation_mode="anchor")
# Loop over data dimensions and create text annotations.
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i in range(cm.shape[0]):
for j in range(cm.shape[1]):
ax.text(j, i, format(cm[i, j], fmt),
ha="center", va="center",
color="white" if cm[i, j] > thresh else "black")
fig.tight_layout()
return ax
In [6]:
# Load dataset.
x_train, y_train, x_test, y_test = load_dataset()
In [7]:
# Initialize naive bayes model.
num_class = len(np.unique(y_train))
feature_dim = len(x_train[0])
num_value = 256
NB = NaiveBayes(num_class,feature_dim,num_value)
In [8]:
# Train model.
NB.train(x_train,y_train)
In [9]:
# Feature likelihood for high intensity pixels.
feature_likelihoods = NB.intensity_feature_likelihoods(NB.likelihood)
# Visualize the feature likelihoods for high intensity pixels.
class_names = np.array(["T-shirt/top","Trouser","Pullover","Dress",
"Coat","Sandal","Shirt","Sneaker","Bag","Ankle boot"])
plot_visualization(feature_likelihoods, class_names, "Greys")
Out[9]:
In [10]:
# Classify the test sets.
accuracy, y_pred = NB.test(x_test,y_test)
# Plot confusion matrix.
plot_confusion_matrix(y_test, y_pred, classes=class_names, normalize=True,
title='Confusion matrix, with normalization')
plt.show()
Out[10]:
Average Classification Rate: 74.7%. The Laplace factor that yielded the best accuracy rate was interestingly 0.1.
Class : Classification Rate
T-shirt : 0.762
Trouser : 0.894
Pullover : 0.455
Dress : 0.842
Coat : 0.634
Sandal : 0.765
Shirt : 0.445
Sneaker : 0.905
Bag : 0.885
Ankle Boot : 0.883
