Machine Learning Programming Workshop
4.1 Tensorflow/Keras (Natural Language)
Prepared By: Cheong Shiu Hong (FTFNCE)
In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
In [2]:
import tensorflow as tf
import tensorflow.keras as K
Version of Tensorflow and Keras
In [3]:
tf.__version__
Out[3]:
In [4]:
K.__version__ # tf means Tensorflow Backend
Out[4]:
Load Dataset
In [5]:
data = K.datasets.imdb
In [6]:
(train_text, train_labels), (val_text, val_labels) = data.load_data(num_words=20000)
In [7]:
train_text.shape, train_labels.shape # Notice Text Shape don't include Seq. Length (In List Form - Unstandardized Lengths)
Out[7]:
In [8]:
val_text.shape, val_labels.shape
Out[8]:
Visualize an Example
In [9]:
print(train_text[0]) # Train Data is in Numbers (Each number maps to a word)
In [10]:
print(train_labels[:10]) # Train Labels are Binary (Positive or Negative Review)
Get the mapping for converting the numbers to words
In [11]:
word_index = data.get_word_index()
In [12]:
len(word_index)
Out[12]:
In [13]:
word_index
Out[13]:
Normally, we will have to build our own 'Voacbulary Dictionary' with words from the corpus
In [14]:
word_index = {k: (v+2) for (k,v) in word_index.items()}
In [15]:
word_index["<PAD>"] = 0 # Used to fill sentences to make Sequence Lengths the same
word_index["<START>"] = 1 # To show the start of a sequence
word_index["UNK"] = 2 # Used to fill in the gap for unknown words
In [16]:
num_to_word = {v: k for (k,v) in word_index.items()}
In [17]:
for i in range(10):
print(num_to_word[i+1])
Convert numerical sentences into word sentences
In [18]:
def decode(numbers):
sentence = ""
for number in numbers:
sentence += num_to_word[number] + " "
return sentence
In [19]:
decode(train_text[0])
Out[19]:
In [20]:
decode(train_text[1]) # Contains words like 'BEST' and 'GOOD', but we can easily tell this is a negative review
Out[20]:
In [21]:
decode(train_text[2])
Out[21]:
Notice the lengths of sentences are different
How can we pass the data into the model?
Process Data
In [22]:
train_data = K.preprocessing.sequence.pad_sequences(train_text, value=0, padding='post', maxlen=256)
In [23]:
decode(train_data[0])
Out[23]:
In [24]:
decode(train_data[1])
Out[24]:
In [25]:
decode(train_data[2])
Out[25]:
In [26]:
train_data.shape
Out[26]:
Remember to Do it for Our Validation Data Too!
In [27]:
val_data = K.preprocessing.sequence.pad_sequences(val_text, value=0, padding='post', maxlen=256)
Let's Build a Neural Network with Keras
What is an Embedding Layer?
Word2Vec, GloVe, ELMo, Tf-idf
In [28]:
model = K.Sequential([
K.layers.Embedding(len(word_index), 8),
K.layers.GlobalAveragePooling1D(), # Averaging the Sentiment across the Embedding Dimensions
K.layers.Dense(32, activation='relu'),
K.layers.Dense(16, activation='relu'),
K.layers.Dense(1, activation='sigmoid')
])
In [29]:
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
In [30]:
model.fit(train_data, train_labels, epochs=5, batch_size=128)
Out[30]:
In [31]:
model.evaluate(val_data, val_labels)
Out[31]:
In [32]:
model.summary()
Taking the "Average Sentiment" of our sentence dont seem quite good enough does it?
Simple Recurrent Layers
In [33]:
rec_model = K.Sequential([
K.layers.Embedding(len(word_index), 8),
K.layers.SimpleRNN(4, return_sequences=False), # Don't Specify Activation Layer - Why?
K.layers.Dense(32, activation='relu'),
K.layers.Dense(16, activation='relu'),
K.layers.Dense(1, activation='sigmoid')
])
In [34]:
rec_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
In [35]:
rec_model.fit(train_data, train_labels, epochs=5, batch_size=128)
Out[35]:
In [36]:
rec_model.evaluate(val_data, val_labels)
Out[36]:
In [37]:
rec_model.summary()
Gated-Recurrent-Unit Layers
In [38]:
gru_model = K.Sequential([
K.layers.Embedding(len(word_index), 8),
K.layers.GRU(4, return_sequences=False), # Don't Specify Activation Layer - Why?
K.layers.Dense(32, activation='relu'),
K.layers.Dense(16, activation='relu'),
K.layers.Dense(1, activation='sigmoid')
])
In [39]:
gru_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
In [40]:
gru_model.fit(train_data, train_labels, epochs=5, batch_size=128)
Out[40]:
In [41]:
gru_model.evaluate(val_data, val_labels)
Out[41]:
In [42]:
gru_model.summary()
Long-Short-Term-Memory Layers
In [43]:
lstm_model = K.Sequential([
K.layers.Embedding(len(word_index), 8),
K.layers.LSTM(4, return_sequences=False), # Don't Specify Activation Layer - Why?
K.layers.Dense(32, activation='relu'),
K.layers.Dense(16, activation='relu'),
K.layers.Dense(1, activation='sigmoid')
])
In [44]:
lstm_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
In [45]:
lstm_model.fit(train_data, train_labels, epochs=5, batch_size=128)
Out[45]:
In [46]:
lstm_model.evaluate(val_data, val_labels)
Out[46]:
In [47]:
lstm_model.summary()
Outputs
In [48]:
gru_model.predict([[val_data[0]]])
Out[48]:
In [49]:
decode(val_text[0])
Out[49]:
In [50]:
print('Models Predicted:',
model.predict([[val_data[0]]]),
rec_model.predict([[val_data[0]]]),
gru_model.predict([[val_data[0]]]),
lstm_model.predict([[val_data[0]]]))
print('True Value:',val_labels[0])
In [51]:
decode(val_text[1])
Out[51]:
In [52]:
print('Models Predicted:',
model.predict([[val_data[1]]]),
rec_model.predict([[val_data[1]]]),
gru_model.predict([[val_data[1]]]),
lstm_model.predict([[val_data[1]]]))
print('True Value:',val_labels[1])
In [53]:
decode(val_text[2])
Out[53]:
In [54]:
print('Models Predicted:',
model.predict([[val_data[2]]]),
rec_model.predict([[val_data[2]]]),
gru_model.predict([[val_data[2]]]),
lstm_model.predict([[val_data[2]]]))
print('True Value:',val_labels[2])
Try our own sentences
In [55]:
def encode(sentence):
words = sentence.lower().split()
numbers = []
for word in words:
try:
numbers.append(word_index[word])
except KeyError:
numbers.append(2)
if len(numbers) < 256:
numbers += ([0] * (256 - len(numbers)))
return numbers
In [57]:
my_sentence = encode('Things were good at the start but it only got worse, even though i still enjoyed the movie')
In [58]:
print(model.predict([[my_sentence]]),
rec_model.predict([[my_sentence]]),
gru_model.predict([[my_sentence]]),
lstm_model.predict([[my_sentence]]))
Notice that even though the ANN performed better in training & validating, the Complex Neural Networks work better for Complex Sentences
In [59]:
shopee_data = pd.read_csv('./sources/shopee_beauty_data.csv', index_col=0)
In [60]:
shopee_data.head()
Out[60]:
Product Texture
In [61]:
data = shopee_data[['title', 'Product_texture']].dropna()
In [62]:
data.head()
Out[62]:
In [63]:
X = data['title']
Y = data['Product_texture']
Load Label Names from JSON File
In [64]:
import json
with open('./sources/beauty_profile_train.json') as f:
beauty_profiles = json.load(f)
In [65]:
class_names = [pair[0] for pair in sorted(beauty_profiles['Product_texture'].items(), key=lambda x: x[1])]
In [66]:
num_classes = len(class_names)
print(class_names)
Process Text
In [67]:
tokenizer = K.preprocessing.text.Tokenizer(num_words=1000)
In [68]:
tokenizer.fit_on_texts(X)
In [69]:
word_index = {k: v+2 for k,v in tokenizer.word_index.items()}
In [70]:
word_index["<PAD>"] = 0 # Used to fill sentences to make Sequence Lengths the same
word_index["<START>"] = 1 # To show the start of a sequence
word_index["UNK"] = 2 # Used to fill in the gap for unknown words
In [71]:
int_data = data['title'].apply(lambda x: [1] + [word_index.get(xi, 2) for xi in x.split()])
In [72]:
padded_data = K.preprocessing.sequence.pad_sequences(int_data, value=0, padding='post', maxlen=30)
In [73]:
print(padded_data)
In [74]:
print(padded_data[0])
Create Number to Word Dictionary for Decoder Function to Work
In [75]:
num_to_word = {v: k for (k,v) in word_index.items()}
In [76]:
print(decode(padded_data[0]))
In [77]:
print(decode(padded_data[1]))
Split Data
In [78]:
padded_data.shape
Out[78]:
In [79]:
split_ratio = 0.2
split_idx = int(split_ratio*len(padded_data))
X_train = padded_data[split_idx:]
Y_train = Y[split_idx:]
X_val = padded_data[:split_idx]
Y_val = Y[:split_idx]
Build Model
In [80]:
gru_model = K.Sequential([
K.layers.Embedding(len(word_index), 8),
K.layers.GRU(4, return_sequences=False),
K.layers.Dense(32, activation='relu'),
K.layers.Dense(16, activation='relu'),
K.layers.Dense(num_classes, activation='softmax')
])
In [81]:
gru_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
Train Model
In [82]:
gru_model.fit(X_train, Y_train, epochs=3, batch_size=64)
Out[82]:
Evaluate Model
In [83]:
gru_model.evaluate(X_val, Y_val)
Out[83]:
In [84]:
class_names
Out[84]:
In [85]:
preds = gru_model.predict(X_val)
class_preds = np.argmax(preds,1)
In [86]:
val_text = data['title'].iloc[:split_idx]
In [87]:
for i in range(20):
print(val_text.iloc[i])
print('True Value: {} | Predicted: {}'.format(class_names[int(Y_val.iloc[i])], class_names[class_preds[i]]))
print()
Predictor Function
In [88]:
def predictor(text):
int_data = [1] + [word_index.get(xi, 2) for xi in text.lower().split()]
padded_data = K.preprocessing.sequence.pad_sequences([int_data], value=0, padding='post', maxlen=30)
pred = gru_model.predict(padded_data)
idx = np.argmax(pred)
class_pred = class_names[idx]
return class_pred
In [89]:
print(class_names)
Try to input a Product Description
In [90]:
text = "suss special invincible amazing super delicious unbelievable wet jelly of immortality"
predictor(text)
Out[90]:
Because the Data included Indonesian Product Listings, Indonesian Descriptions like "Krim" (Cream) and "Bubuk" (Powder) will work too
In [91]:
text = "dijamin terlihat lebih muda dan lebih indah bubuk super luar biasa dengan aditif kecantikan"
predictor(text)
Out[91]:
In [92]:
text = "dijamin terlihat lebih muda dan mousse super luar biasa lebih indah dengan aditif kecantikan"
predictor(text)
Out[92]:
Previous: