1. Hand written digits classification using MLP and CNN using keras.

Using Multi Layer Perceptrons

import numpy
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.utils import np_utils
# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
# load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# flatten 28*28 images to a 784 vector for each image
num_pixels = X_train.shape[1] * X_train.shape[2]
X_train = X_train.reshape(X_train.shape[0], num_pixels).astype('float32')
X_test = X_test.reshape(X_test.shape[0], num_pixels).astype('float32')
# normalize inputs from 0-255 to 0-1
X_train = X_train / 255
X_test = X_test / 255
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
# define baseline model
def baseline_model():
# create model
model = Sequential()
 model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal',
activation='relu'))
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
# build the model
model = baseline_model()
# Fit the model
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200,
verbose=2)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Baseline Error: %.2f%%" % (100-scores[1]*100))
OUTPUT

Using CNN
import numpy
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from keras import backend as K
K.set_image_dim_ordering('th')
# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
# load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# reshape to be [samples][pixels][width][height]
X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')
X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')
# normalize inputs from 0-255 to 0-1
X_train = X_train / 255
X_test = X_test / 255
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
model.add(Conv2D(32, (5, 5), input_shape=(1, 28, 28), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
# build the model
model = baseline_model()
# Fit the model
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200,
verbose=2)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("CNN Error: %.2f%%" % (100-scores[1]*100))

OUTPUT
2. Hand written digits classification using MLP and CNN using Tensorflow

import numpy as np
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
# Model function for CNN
def cnn_model_fn(features, labels, mode):
# Input Layer
input_layer = tf.reshape(features["x"], [-1, 28, 28, 1])
# Convolutional Layer #1
conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
# Pooling Layer #1
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
# Convolutional Layer #2 and Pooling Layer #2
conv2 = tf.layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
# Dense Layer
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense, rate=0.4, training=mode ==
tf.estimator.ModeKeys.TRAIN)
# Logits Layer
logits = tf.layers.dense(inputs=dropout, units=10)
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT and by the
`logging_hook`
 "probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

# Calculate Loss (for both TRAIN and EVAL modes)

loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)

# Configure the Training Op (for TRAIN mode)

if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(labels=labels, predictions=predictions["classes"])
}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss,
eval_metric_ops=eval_metric_ops)
# Load training and eval data
mnist = tf.contrib.learn.datasets.load_dataset("mnist")
train_data = mnist.train.images # Returns np.array
train_labels = np.asarray(mnist.train.labels, dtype=np.int32)
eval_data = mnist.test.images # Returns np.array
eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)

# Create the Estimator

mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
model_dir="/tmp/mnist_convnet_model")
# Set up logging for predictions
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=50)
# Train the model
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": train_data}, y=train_labels,
batch_size=100, num_epochs=None, shuffle=True)
mnist_classifier.train(input_fn=train_input_fn, steps=20000, hooks=[logging_hook])
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": eval_data}, y=eval_labels,
num_epochs=1, shuffle=False)
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
 OUTPUT

3. An Interactive Character-Level Language Model tensorflow implentaion using

LSTM.

import tensorflow as tf

import matplotlib.pyplot as plt

from tensorlm import Vocabulary, Dataset, GeneratingLSTM

TEXT_PATH = "tinytrain.txt"

DEV_PATH = "tinyvalid.txt"

BATCH_SIZE =20
NUM_TIMESTEPS =30

with tf.Session() as session:

# Generate a vocabulary based on the text

vocab = Vocabulary.create_from_text(TEXT_PATH, max_vocab_size=96,

level="char")

# Obtain input and target batches from the text file

dataset = Dataset(TEXT_PATH, vocab, BATCH_SIZE, NUM_TIMESTEPS)

# Create the model in a TensorFlow graph

model = GeneratingLSTM(vocab_size=vocab.get_size(),

neurons_per_layer=100,

num_layers=2,

max_batch_size=BATCH_SIZE,

output_keep_prob= 0.8)

# Initialize all defined TF Variables

session.run(tf.global_variables_initializer())

# Do the training

epoch =1
step = 1
arr1 = []
arr2 = []
for epoch in range(101):
for inputs, targets in dataset:
loss = model.train_step(session, inputs, targets)
step+=1
if epoch%10==0:

# Evaluate from time to time

print("Evaluating")
 dev_dataset = Dataset(DEV_PATH, vocab, batch_size=1,

num_timesteps=NUM_TIMESTEPS)

dev_loss = model.evaluate(session, dev_dataset)

arr1.append(loss)

arr2.append(dev_loss)

print("Epoch: %d, Step: %d, Train Loss: %f, Dev Loss: %f" % ( epoch, step,
loss, dev_loss))

# Sample from the model from time to time

print("The " + model.sample_text(session, vocab, "The "))

OUTPUT
4. Predicting reviews using tensorflow

import pandas as pd

import numpy as np

import tensorflow as tf

import nltk, re, time

from nltk.corpus import stopwords

from collections import defaultdict

from tqdm import tqdm

from sklearn.model_selection import train_test_split

from keras.preprocessing.text import Tokenizer

from keras.preprocessing.sequence import pad_sequences

from collections import namedtuple

path = r'\Movie Dataset\\'

train = pd.read_csv(path + "labeledTrainData.tsv", delimiter="\t")

test = pd.read_csv(path + "testData.tsv", delimiter="\t")

train.head(5)

def clean_text(text, remove_stopwords=True):

'''Clean the text, with the option to remove stopwords'''

# Convert words to lower case and split them

text = text.lower().split()

# Optionally, remove stop words

if remove_stopwords:

stops = set(stopwords.words("english"))
 text = [w for w in text if not w in stops]

text = " ".join(text)

# Clean the text

text = re.sub(r"<br />", " ", text)

text = re.sub(r"[^a-z]", " ", text)

text = re.sub(r" ", " ", text) # Remove any extra spaces

text = re.sub(r" ", " ", text)

# Return a list of words

return(text)

train_clean = train

test_clean = test

train_clean['review'] = train_clean['review'].apply(lambda x : clean_text(x))

test_clean['review'] = test_clean['review'].apply(lambda x : clean_text(x))

test_clean.head(5)

# Tokenize the reviews

all_reviews = list(train_clean['review']) + list(test_clean['review'])

tokenizer = Tokenizer()

tokenizer.fit_on_texts(all_reviews)

print("Fitting is complete.")

train_seq = tokenizer.texts_to_sequences(list(train_clean['review']))

print("train_seq is complete.")

test_seq = tokenizer.texts_to_sequences(list(test_clean['review']))

print("test_seq is complete")
max_review_length = 200

train_pad = pad_sequences(train_seq, maxlen = max_review_length)

print("train_pad is complete.")

test_pad = pad_sequences(test_seq, maxlen = max_review_length)

print("test_pad is complete.")

x_train, x_valid, y_train, y_valid = train_test_split(train_pad, train.sentiment, test_size =

0.15, random_state = 2)

def get_batches(x, y, batch_size):

'''Create the batches for the training and validation data'''

n_batches = len(x)//batch_size

x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]

for ii in range(0, len(x), batch_size):

yield x[ii:ii+batch_size], y[ii:ii+batch_size]

def get_test_batches(x, batch_size):

'''Create the batches for the testing data'''

n_batches = len(x)//batch_size

x = x[:n_batches*batch_size]

for ii in range(0, len(x), batch_size):

yield x[ii:ii+batch_size]

def build_rnn(n_words, embed_size, batch_size, lstm_size, num_layers, dropout,

learning_rate, multiple_fc, fc_units):

'''Build the Recurrent Neural Network'''

tf.reset_default_graph()
# Declare placeholders we'll feed into the graph

with tf.name_scope('inputs'):

inputs = tf.placeholder(tf.int32, [None, None], name='inputs')

print("Input = ", inputs.shape)

with tf.name_scope('labels'):

labels = tf.placeholder(tf.int32, [None, None], name='labels')

print("labels = ", labels.shape)

keep_prob = tf.placeholder(tf.float32, name='keep_prob')

# Create the embeddings

with tf.name_scope("embeddings"):

embedding = tf.Variable(tf.random_uniform((n_words,embed_size), -1, 1))

print("embeddings = ", embedding.shape)

embed = tf.nn.embedding_lookup(embedding, inputs)

print("embed = ", embed.shape)

# Build the RNN layers

with tf.name_scope("RNN_layers"):

lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)

#print(lstm.output_size)

drop = tf.contrib.rnn.DropoutWrapper(lstm,output_keep_prob=keep_prob)

#print(drop.output_shape)

#cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)

cell = drop

#print(cell.shape)
 # Set the initial state

with tf.name_scope("RNN_init_state"):

initial_state = cell.zero_state(batch_size, tf.float32)

# Run the data through the RNN layers

with tf.name_scope("RNN_forward"):

outputs, final_state = tf.nn.dynamic_rnn(cell, embed,initial_state=initial_state)

# Create the fully connected layers

with tf.name_scope("fully_connected"):

# Initialize the weights and biases

weights = tf.truncated_normal_initializer(stddev=0.1)

biases = tf.zeros_initializer()

dense = tf.contrib.layers.fully_connected(outputs[:, -1], num_outputs = fc_units,

activation_fn = tf.sigmoid,weights_initializer = weights,biases_initializer = biases)

dense = tf.contrib.layers.dropout(dense, keep_prob)

# Depending on the iteration, use a second fully connected layer

if multiple_fc == True:

dense = tf.contrib.layers.fully_connected(dense,

num_outputs = fc_units,

activation_fn = tf.sigmoid,

weights_initializer = weights,

biases_initializer = biases)

dense = tf.contrib.layers.dropout(dense, keep_prob)

# Make the predictions

with tf.name_scope('predictions'):
 predictions = tf.contrib.layers.fully_connected(dense,

num_outputs = 1,

activation_fn=tf.sigmoid,

weights_initializer = weights,

biases_initializer = biases)

tf.summary.histogram('predictions', predictions)

# Calculate the cost

with tf.name_scope('cost'):

cost = tf.losses.mean_squared_error(labels, predictions)

tf.summary.scalar('cost', cost)

# Train the model

with tf.name_scope('train'):

optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)

# Determine the accuracy

with tf.name_scope("accuracy"):

correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels)

accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

tf.summary.scalar('accuracy', accuracy)

# Merge all of the summaries

merged = tf.summary.merge_all()

# Export the nodes

export_nodes = ['inputs', 'labels', 'keep_prob','initial_state', 'final_state','accuracy',

'predictions', 'cost', 'optimizer', 'merged']

Graph = namedtuple('Graph', export_nodes)

 local_dict = locals()

graph = Graph(*[local_dict[each] for each in export_nodes])

return graph

def train(model, epochs, log_string):

'''Train the RNN'''

saver = tf.train.Saver()

with tf.Session() as sess:

sess.run(tf.global_variables_initializer())

# Used to determine when to stop the training early

valid_loss_summary = []

# Keep track of which batch iteration is being trained

iteration = 0

print()

print("Training Model: {}".format(log_string))

train_writer = tf.summary.FileWriter('./logs/3/train/{}'.format(log_string),
sess.graph)

valid_writer = tf.summary.FileWriter('./logs/3/valid/{}'.format(log_string))

for e in range(epochs):

state = sess.run(model.initial_state)

# Record progress with each epoch

train_loss = []

train_acc = []

val_acc = []
 val_loss = []

with tqdm(total=len(x_train)) as pbar:

for _, (x, y) in enumerate(get_batches(x_train, y_train,batch_size), 1):

feed = {model.inputs: x, model.labels: y[:, None], model.keep_prob:

dropout, model.initial_state: state}

summary, loss, acc, state, _ = sess.run([model.merged,model.cost,

model.accuracy, model.final_state, model.optimizer], feed_dict=feed)

# Record the loss and accuracy of each training batch

train_loss.append(loss)

train_acc.append(acc)

# Record the progress of training

train_writer.add_summary(summary, iteration)

iteration += 1

pbar.update(batch_size)

# Average the training loss and accuracy of each epoch

avg_train_loss = np.mean(train_loss)

avg_train_acc = np.mean(train_acc)

val_state = sess.run(model.initial_state)

with tqdm(total=len(x_valid)) as pbar:

for x, y in get_batches(x_valid,y_valid,batch_size):

feed = {model.inputs: x,

model.labels: y[:, None],

model.keep_prob: 1,

model.initial_state: val_state}
 summary, batch_loss, batch_acc, val_state = sess.run([model.merged,
model.cost, model.accuracy, model.final_state], feed_dict=feed)

# Record the validation loss and accuracy of each epoch

val_loss.append(batch_loss)

val_acc.append(batch_acc)

pbar.update(batch_size)

# Average the validation loss and accuracy of each epoch

avg_valid_loss = np.mean(val_loss)

avg_valid_acc = np.mean(val_acc)

valid_loss_summary.append(avg_valid_loss)

# Record the validation data's progress

valid_writer.add_summary(summary, iteration)

# Print the progress of each epoch

print("Epoch: {}/{}".format(e, epochs),

"Train Loss: {:.3f}".format(avg_train_loss),

"Train Acc: {:.3f}".format(avg_train_acc),

"Valid Loss: {:.3f}".format(avg_valid_loss),

"Valid Acc: {:.3f}".format(avg_valid_acc))

# Stop training if the validation loss does not decrease after 3 epochs

if avg_valid_loss > min(valid_loss_summary):

print("No Improvement.")

stop_early += 1

if stop_early == 3:

break
 # Reset stop_early if the validation loss finds a new low

# Save a checkpoint of the model

else:

print("New Record!")

stop_early = 0

checkpoint ="./sentiment_{}.ckpt".format(log_string)

saver.save(sess, checkpoint)

n_words = len(tokenizer.word_index)

embed_size = 50

batch_size = 250

lstm_size = 64

num_layers = 2

dropout = 0.5

learning_rate = 0.001

epochs = 5

multiple_fc = False

fc_units = 250

for lstm_size in [64]:

for multiple_fc in [True]:

for fc_units in [128]:

log_string = 'ru={},fcl={},fcu={}'.format(lstm_size,multiple_fc,fc_units)

model = build_rnn(n_words = n_words, embed_size = embed_size,batch_size =

batch_size,lstm_size = lstm_size,num_layers = num_layers,dropout = dropout,
learning_rate = learning_rate, multiple_fc = multiple_fc,fc_units = fc_units)
 train(model, epochs, log_string)

def make_predictions(lstm_size, multiple_fc, fc_units, checkpoint):

'''Predict the sentiment of the testing data'''

# Record all of the predictions

all_preds = []

model = build_rnn(n_words = n_words,

embed_size = embed_size,

batch_size = batch_size,

lstm_size = lstm_size,

num_layers = num_layers,

dropout = dropout,

learning_rate = learning_rate,

multiple_fc = multiple_fc,

fc_units = fc_units)

with tf.Session() as sess:

saver = tf.train.Saver()

# Load the model

saver.restore(sess, checkpoint)

test_state = sess.run(model.initial_state)

for _, x in enumerate(get_test_batches(x_valid,

batch_size), 1):

feed = {model.inputs: x,
 model.keep_prob: 1,

model.initial_state: test_state}

predictions = sess.run(model.predictions,feed_dict=feed)

for pred in predictions:

all_preds.append(float(pred))

return all_preds

pred = make_predictions(64,True,128,"sentiment_ru=64,fcl=True,fcu=128.ckpt")

5. Sentiment analysis on twitter date using keras

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

from keras.layers import Lambda, Input, Dense

from keras.models import Model

from keras.datasets import mnist

from keras.losses import mse, binary_crossentropy

from keras.utils import plot_model

from keras import backend as K

import numpy as np

import matplotlib.pyplot as plt

import argparse

import os

# reparameterization trick

# instead of sampling from Q(z|X), sample eps = N(0,I)

# z = z_mean + sqrt(var)*eps

def sampling(args):

"""Reparameterization trick by sampling fr an isotropic unit Gaussian.

# Arguments:

args (tensor): mean and log of variance of Q(z|X)

# Returns:

z (tensor): sampled latent vector

"""

z_mean, z_log_var = args

batch = K.shape(z_mean)[0]

dim = K.int_shape(z_mean)[1]

# by default, random_normal has mean=0 and std=1.0

epsilon = K.random_normal(shape=(batch, dim))

return z_mean + K.exp(0.5 * z_log_var) * epsilon

def plot_results(models,
 data,

batch_size=128,

model_name="vae_mnist"):

"""Plots labels and MNIST digits as function of 2-dim latent vector

# Arguments:

models (tuple): encoder and decoder models

data (tuple): test data and label

batch_size (int): prediction batch size

model_name (string): which model is using this function

"""

encoder, decoder = models

x_test, y_test = data

os.makedirs(model_name, exist_ok=True)

filename = os.path.join(model_name, "vae_mean.png")

# display a 2D plot of the digit classes in the latent space

z_mean, _, _ = encoder.predict(x_test, batch_size=batch_size)

plt.figure(figsize=(12, 10))

plt.scatter(z_mean[:, 0], z_mean[:, 1], c=y_test)

plt.colorbar()

plt.xlabel("z[0]")

plt.ylabel("z[1]")

plt.savefig(filename)

plt.show()
filename = os.path.join(model_name, "digits_over_latent.png")

# display a 30x30 2D manifold of digits

n = 30

digit_size = 28

figure = np.zeros((digit_size * n, digit_size * n))

# linearly spaced coordinates corresponding to the 2D plot

# of digit classes in the latent space

grid_x = np.linspace(-4, 4, n)

grid_y = np.linspace(-4, 4, n)[::-1]

for i, yi in enumerate(grid_y):

for j, xi in enumerate(grid_x):

z_sample = np.array([[xi, yi]])

x_decoded = decoder.predict(z_sample)

digit = x_decoded[0].reshape(digit_size, digit_size)

figure[i * digit_size: (i + 1) * digit_size,

j * digit_size: (j + 1) * digit_size] = digit

plt.figure(figsize=(10, 10))

start_range = digit_size // 2

end_range = n * digit_size + start_range + 1

pixel_range = np.arange(start_range, end_range, digit_size)

sample_range_x = np.round(grid_x, 1)

sample_range_y = np.round(grid_y, 1)
 plt.xticks(pixel_range, sample_range_x)

plt.yticks(pixel_range, sample_range_y)

plt.xlabel("z[0]")

plt.ylabel("z[1]")

plt.imshow(figure, cmap='Greys_r')

plt.savefig(filename)

plt.show()

# MNIST dataset

(x_train, y_train), (x_test, y_test) = mnist.load_data()

image_size = x_train.shape[1]

original_dim = image_size * image_size

x_train = np.reshape(x_train, [-1, original_dim])

x_test = np.reshape(x_test, [-1, original_dim])

x_train = x_train.astype('float32') / 255

x_test = x_test.astype('float32') / 255

# network parameters

input_shape = (original_dim, )

intermediate_dim = 512

batch_size = 128

latent_dim = 2

epochs = 50

# VAE model = encoder + decoder

# build encoder model

inputs = Input(shape=input_shape, name='encoder_input')

x = Dense(intermediate_dim, activation='relu')(inputs)

z_mean = Dense(latent_dim, name='z_mean')(x)

z_log_var = Dense(latent_dim, name='z_log_var')(x)

# use reparameterization trick to push the sampling out as input

# note that "output_shape" isn't necessary with the TensorFlow backend

z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])

# instantiate encoder model

encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')

encoder.summary()

plot_model(encoder, to_file='vae_mlp_encoder.png', show_shapes=True)

# build decoder model

latent_inputs = Input(shape=(latent_dim,), name='z_sampling')

x = Dense(intermediate_dim, activation='relu')(latent_inputs)

outputs = Dense(original_dim, activation='sigmoid')(x)

# instantiate decoder model

decoder = Model(latent_inputs, outputs, name='decoder')

decoder.summary()

plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)

# instantiate VAE model

outputs = decoder(encoder(inputs)[2])

vae = Model(inputs, outputs, name='vae_mlp')

if __name__ == '__main__':

models = (encoder, decoder)

data = (x_test, y_test)

# VAE loss = mse_loss or xent_loss + kl_loss

reconstruction_loss = mse(inputs, outputs)

reconstruction_loss *= original_dim

kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)

kl_loss = K.sum(kl_loss, axis=-1)

kl_loss *= -0.5

vae_loss = K.mean(reconstruction_loss + kl_loss)

vae.add_loss(vae_loss)

vae.compile(optimizer='adam')

vae.summary()

plot_model(vae, to_file='vae_mlp.png', show_shapes=True)

# train the autoencoder

vae.fit(x_train,epochs=epochs,batch_size=batch_size,validation_data=(x_test, None))

vae.save_weights('vae_mlp_mnist.h5')

plot_results(models, data, batch_size=batch_size, model_name="vae_mlp")

 5. Variational auto encoder to reconstruct the image keras

from __future__ import absolute_import

from __future__ import division
from __future__ import print_function
from keras.layers import Lambda, Input, Dense
from keras.models import Model
from keras.datasets import mnist
from keras.losses import mse, binary_crossentropy
from keras.utils import plot_model
from keras import backend as K
import numpy as np
import matplotlib.pyplot as plt
import argparse
import os
import pydot
# reparameterization trick
# instead of sampling from Q(z|X), sample eps = N(0,I)
# z = z_mean + sqrt(var)*eps
def sampling(args):
"""Reparameterization trick by sampling fr an isotropic unit Gaussian.
# Arguments:
args (tensor): mean and log of variance of Q(z|X)
# Returns:
z (tensor): sampled latent vector
"""
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
# by default, random_normal has mean=0 and std=1.0
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var) * epsilon
def plot_results(models, data, batch_size=128, model_name="vae_mnist"):
"""Plots labels and MNIST digits as function of 2-dim latent vector
# Arguments:
models (tuple): encoder and decoder models
data (tuple): test data and label
batch_size (int): prediction batch size
model_name (string): which model is using this function
 """
encoder, decoder = models
x_test, y_test = data
os.makedirs(model_name, exist_ok=True)
filename = os.path.join(model_name, "vae_mean.png")
# display a 2D plot of the digit classes in the latent space
z_mean, _, _ = encoder.predict(x_test,
batch_size=batch_size)
plt.figure(figsize=(12, 10))
plt.scatter(z_mean[:, 0], z_mean[:, 1], c=y_test)
plt.colorbar()
plt.xlabel("z[0]")
plt.ylabel("z[1]")
plt.savefig(filename)
plt.show()
filename = os.path.join(model_name, "digits_over_latent.png")
# display a 30x30 2D manifold of digits
n = 30
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# linearly spaced coordinates corresponding to the 2D plot
# of digit classes in the latent space
grid_x = np.linspace(-4, 4, n)
grid_y = np.linspace(-4, 4, n)[::-1]
for i, yi in enumerate(grid_y):
for j, xi in enumerate(grid_x):
z_sample = np.array([[xi, yi]])
x_decoded = decoder.predict(z_sample)
 digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size: (i + 1) * digit_size,
j * digit_size: (j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
start_range = digit_size // 2
end_range = n * digit_size + start_range + 1
pixel_range = np.arange(start_range, end_range, digit_size)
sample_range_x = np.round(grid_x, 1)
sample_range_y = np.round(grid_y, 1)
plt.xticks(pixel_range, sample_range_x)
plt.yticks(pixel_range, sample_range_y)
plt.xlabel("z[0]")
plt.ylabel("z[1]")
plt.imshow(figure, cmap='Greys_r')
plt.savefig(filename)
plt.show()

# MNIST dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()

image_size = x_train.shape[1]
original_dim = image_size * image_size
x_train = np.reshape(x_train, [-1, original_dim])
x_test = np.reshape(x_test, [-1, original_dim])
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

# network parameters
input_shape = (original_dim, )
intermediate_dim = 512
batch_size = 128
latent_dim = 2
epochs = 50

# VAE model = encoder + decoder

# build encoder model
inputs = Input(shape=input_shape, name='encoder_input')
x = Dense(intermediate_dim, activation='relu')(inputs)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)

# use reparameterization trick to push the sampling out as input

# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])

# instantiate encoder model

encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
plot_model(encoder, to_file='vae_mlp_encoder.png', show_shapes=True)

# build decoder model

latent_inputs = Input(shape=(latent_dim,), name='z_sampling')
x = Dense(intermediate_dim, activation='relu')(latent_inputs)
outputs = Dense(original_dim, activation='sigmoid')(x)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)

# instantiate VAE model

outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')

if __name__ == '__main__':
parser = argparse.ArgumentParser()
help_ = "Load h5 model trained weights"
parser.add_argument("-w", "--weights", help=help_)
help_ = "Use mse loss instead of binary cross entropy (default)"
parser.add_argument("-m",
"--mse",
help=help_, action='store_true')
args = parser.parse_args()
models = (encoder, decoder)
data = (x_test, y_test)

# VAE loss = mse_loss or xent_loss + kl_loss

if args.mse:
reconstruction_loss = mse(inputs, outputs)
else:
reconstruction_loss = binary_crossentropy(inputs,outputs)
reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
vae.summary()
plot_model(vae,to_file='vae_mlp.png', show_shapes=True)
if args.weights:
vae.load_weights(args.weights)
else:
# train the autoencoder
vae.fit(x_train, epochs=epochs,batch_size=batch_size,validation_data=(x_test, None))
vae.save_weights('vae_mlp_mnist.h5')
plot_results(models, data,batch_size=batch_size,model_name="vae_mlp")
7. Variational auto encoder to reconstruct the image using tensorflow

import numpy as np

import tensorflow as tf

import matplotlib.pyplot as plt

%matplotlib inline

np.random.seed(0)

tf.set_random_seed(0)

# Load MNIST data in a format suited for tensorflow.

# The script input_data is available under this URL:

#
https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/examples/tut
orials/mnist/input_data.py

from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

n_samples = mnist.train.num_examples

def xavier_init(fan_in, fan_out, constant=1):

""" Xavier initialization of network weights"""

# https://stackoverflow.com/questions/33640581/how-to-do-xavier-initialization-on-
tensorflow

low = -constant*np.sqrt(6.0/(fan_in + fan_out))

high = constant*np.sqrt(6.0/(fan_in + fan_out))

return tf.random_uniform((fan_in, fan_out),

minval=low, maxval=high,

dtype=tf.float32)
class VariationalAutoencoder(object):

""" Variation Autoencoder (VAE) with an sklearn-like interface implemented using

TensorFlow.

This implementation uses probabilistic encoders and decoders using Gaussian

distributions and realized by multi-layer perceptrons. The VAE can be learned

end-to-end.

See "Auto-Encoding Variational Bayes" by Kingma and Welling for more details.

"""

def __init__(self, network_architecture, transfer_fct=tf.nn.softplus,

learning_rate=0.001, batch_size=100):

self.network_architecture = network_architecture

self.transfer_fct = transfer_fct

self.learning_rate = learning_rate

self.batch_size = batch_size

# tf Graph input

self.x = tf.placeholder(tf.float32, [None, network_architecture["n_input"]])

# Create autoencoder network

self._create_network()

# Define loss function based variational upper-bound and

# corresponding optimizer
 self._create_loss_optimizer()

# Initializing the tensor flow variables

init = tf.global_variables_initializer()

# Launch the session

self.sess = tf.InteractiveSession()

self.sess.run(init)

def _create_network(self):

# Initialize autoencode network weights and biases

network_weights = self._initialize_weights(**self.network_architecture)

# Use recognition network to determine mean and

# (log) variance of Gaussian distribution in latent

# space

self.z_mean, self.z_log_sigma_sq = \

self._recognition_network(network_weights["weights_recog"],

network_weights["biases_recog"])

# Draw one sample z from Gaussian distribution

n_z = self.network_architecture["n_z"]

eps = tf.random_normal((self.batch_size, n_z), 0, 1, dtype=tf.float32)

 # z = mu + sigma*epsilon

self.z = tf.add(self.z_mean, tf.multiply(tf.sqrt(tf.exp(self.z_log_sigma_sq)), eps))

# Use generator to determine mean of

# Bernoulli distribution of reconstructed input

self.x_reconstr_mean = \

self._generator_network(network_weights["weights_gener"],network_weights["biases_g
ener"])

def _initialize_weights(self, n_hidden_recog_1, n_hidden_recog_2,

n_hidden_gener_1, n_hidden_gener_2,

n_input, n_z):

all_weights = dict()

all_weights['weights_recog'] = {

'h1': tf.Variable(xavier_init(n_input, n_hidden_recog_1)),

'h2': tf.Variable(xavier_init(n_hidden_recog_1, n_hidden_recog_2)),

'out_mean': tf.Variable(xavier_init(n_hidden_recog_2, n_z)),

'out_log_sigma': tf.Variable(xavier_init(n_hidden_recog_2, n_z))}

all_weights['biases_recog'] = {

'b1': tf.Variable(tf.zeros([n_hidden_recog_1], dtype=tf.float32)),

'b2': tf.Variable(tf.zeros([n_hidden_recog_2], dtype=tf.float32)),

'out_mean': tf.Variable(tf.zeros([n_z], dtype=tf.float32)),

'out_log_sigma': tf.Variable(tf.zeros([n_z], dtype=tf.float32))}

 all_weights['weights_gener'] = {

'h1': tf.Variable(xavier_init(n_z, n_hidden_gener_1)),

'h2': tf.Variable(xavier_init(n_hidden_gener_1, n_hidden_gener_2)),

'out_mean': tf.Variable(xavier_init(n_hidden_gener_2, n_input)),

'out_log_sigma': tf.Variable(xavier_init(n_hidden_gener_2, n_input))}

all_weights['biases_gener'] = {

'b1': tf.Variable(tf.zeros([n_hidden_gener_1], dtype=tf.float32)),

'b2': tf.Variable(tf.zeros([n_hidden_gener_2], dtype=tf.float32)),

'out_mean': tf.Variable(tf.zeros([n_input], dtype=tf.float32)),

'out_log_sigma': tf.Variable(tf.zeros([n_input], dtype=tf.float32))}

return all_weights

def _recognition_network(self, weights, biases):

# Generate probabilistic encoder (recognition network), which

# maps inputs onto a normal distribution in latent space.

# The transformation is parametrized and can be learned.

layer_1 = self.transfer_fct(tf.add(tf.matmul(self.x, weights['h1']),

biases['b1']))

layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights['h2']),

biases['b2']))

z_mean = tf.add(tf.matmul(layer_2, weights['out_mean']),

biases['out_mean'])

z_log_sigma_sq = \
 tf.add(tf.matmul(layer_2, weights['out_log_sigma']),

biases['out_log_sigma'])

return (z_mean, z_log_sigma_sq)

def _generator_network(self, weights, biases):

# Generate probabilistic decoder (decoder network), which

# maps points in latent space onto a Bernoulli distribution in data space.

# The transformation is parametrized and can be learned.

layer_1 = self.transfer_fct(tf.add(tf.matmul(self.z, weights['h1']),

biases['b1']))

layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights['h2']),

biases['b2']))

x_reconstr_mean = \

tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['out_mean']),

biases['out_mean']))

return x_reconstr_mean

def _create_loss_optimizer(self):

# The loss is composed of two terms:

# 1.) The reconstruction loss (the negative log probability

# of the input under the reconstructed Bernoulli distribution

# induced by the decoder in the data space).

# This can be interpreted as the number of "nats" required

 # for reconstructing the input when the activation in latent

# is given.

# Adding 1e-10 to avoid evaluation of log(0.0)

reconstr_loss = \

-tf.reduce_sum(self.x * tf.log(1e-10 + self.x_reconstr_mean)

+ (1-self.x) * tf.log(1e-10 + 1 - self.x_reconstr_mean),

1)

# 2.) The latent loss, which is defined as the Kullback Leibler divergence

## between the distribution in latent space induced by the encoder on

# the data and some prior. This acts as a kind of regularizer.

# This can be interpreted as the number of "nats" required

# for transmitting the the latent space distribution given

# the prior.

latent_loss = -0.5 * tf.reduce_sum(1 + self.z_log_sigma_sq

- tf.square(self.z_mean)

- tf.exp(self.z_log_sigma_sq), 1)

self.cost = tf.reduce_mean(reconstr_loss + latent_loss) # average over batch

# Use ADAM optimizer

self.optimizer = \

tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.cost)

def partial_fit(self, X):

"""Train model based on mini-batch of input data.

 Return cost of mini-batch.

"""

opt, cost = self.sess.run((self.optimizer, self.cost),

feed_dict={self.x: X})

return cost

def transform(self, X):

"""Transform data by mapping it into the latent space."""

# Note: This maps to mean of distribution, we could alternatively

# sample from Gaussian distribution

return self.sess.run(self.z_mean, feed_dict={self.x: X})

def generate(self, z_mu=None):

""" Generate data by sampling from latent space.

If z_mu is not None, data for this point in latent space is

generated. Otherwise, z_mu is drawn from prior in latent

space.

"""

if z_mu is None:

z_mu = np.random.normal(size=self.network_architecture["n_z"])

# Note: This maps to mean of distribution, we could alternatively

 # sample from Gaussian distribution

return self.sess.run(self.x_reconstr_mean,

feed_dict={self.z: z_mu})

def reconstruct(self, X):

""" Use VAE to reconstruct given data. """

return self.sess.run(self.x_reconstr_mean,

feed_dict={self.x: X})

def train(network_architecture, learning_rate=0.001,

batch_size=100, training_epochs=10, display_step=5):

vae = VariationalAutoencoder(network_architecture,

learning_rate=learning_rate,

batch_size=batch_size)

# Training cycle

for epoch in range(training_epochs):

avg_cost = 0.

total_batch = int(n_samples / batch_size)

# Loop over all batches

for i in range(total_batch):

batch_xs, _ = mnist.train.next_batch(batch_size)

# Fit training using batch data

cost = vae.partial_fit(batch_xs)
 # Compute average loss

avg_cost += cost / n_samples * batch_size

# Display logs per epoch step

if epoch % display_step == 0:

print("Epoch:", '%04d' % (epoch+1),

"cost=", "{:.9f}".format(avg_cost))

return vae

network_architecture = \

dict(n_hidden_recog_1=500, # 1st layer encoder neurons

n_hidden_recog_2=500, # 2nd layer encoder neurons

n_hidden_gener_1=500, # 1st layer decoder neurons

n_hidden_gener_2=500, # 2nd layer decoder neurons

n_input=784, # MNIST data input (img shape: 28*28)

n_z=20) # dimensionality of latent space

vae = train(network_architecture, training_epochs=75)

x_sample = mnist.test.next_batch(100)[0]

x_reconstruct = vae.reconstruct(x_sample)

plt.figure(figsize=(8, 12))
for i in range(5):

plt.subplot(5, 2, 2*i + 1)

plt.imshow(x_sample[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")

plt.title("Test input")

plt.colorbar()

plt.subplot(5, 2, 2*i + 2)

plt.imshow(x_reconstruct[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")

plt.title("Reconstruction")

plt.colorbar()

plt.tight_layout()

plt.savefig('vae_tf_reconstruct.png')

network_architecture = \

dict(n_hidden_recog_1=500, # 1st layer encoder neurons

n_hidden_recog_2=500, # 2nd layer encoder neurons

n_hidden_gener_1=500, # 1st layer decoder neurons

n_hidden_gener_2=500, # 2nd layer decoder neurons

n_input=784, # MNIST data input (img shape: 28*28)

n_z=2) # dimensionality of latent space

vae_2d = train(network_architecture, training_epochs=75)

x_sample, y_sample = mnist.test.next_batch(5000)

z_mu = vae_2d.transform(x_sample)

plt.figure(figsize=(8, 6))

plt.scatter(z_mu[:, 0], z_mu[:, 1], c=np.argmax(y_sample, 1))

plt.colorbar()

plt.grid()

plt.savefig('vae_tf_mean.png')

nx = ny = 20

x_values = np.linspace(-3, 3, nx)

y_values = np.linspace(-3, 3, ny)

canvas = np.empty((28*ny, 28*nx))

for i, yi in enumerate(x_values):

for j, xi in enumerate(y_values):

z_mu = np.array([[xi, yi]]*vae.batch_size)

x_mean = vae_2d.generate(z_mu)

canvas[(nx-i-1)*28:(nx-i)*28, j*28:(j+1)*28] = x_mean[0].reshape(28, 28)

plt.figure(figsize=(8, 10))

Xi, Yi = np.meshgrid(x_values, y_values)

plt.imshow(canvas, origin="upper", cmap="gray")

plt.tight_layout()
 plt.savefig('vae_tf_reconstruct.png')

OUTPUT
8. Gated recurrent unit implementation using tensorflow and keras for next character
prediction.

from char_pridect import predict_words,get_top_words

import numpy as np, sys, os
from GRU import gru
from vanilla_rnn import vrnn
from rnn_interface import train, test
def usage():
print "Usage : python main.py [options]"
print "options:"
print "\t -rnn vrnn or gru: define type recurrent model use (by default 'gru')"
print "\t -text file name: input file name use for trainning model for character prediction"
print "\t -npredict number of pridection: given npredict number of words predicted"
exit()
def parse_argv(argv):
rnn = ''; ftext=''; npredict=10
if len(argv) > 1:
 if argv[1] == '-help' or argv[1] == '--help' or argv[1] == '-h':
usage()
if '-rnn' in argv:
ind=argv.index('-rnn')
rnn = sys.argv[ind+1]
if '-text' in argv:
ind=argv.index('-text')
ftext = sys.argv[ind+1]
if '-npredict' in sys.argv:
ind=sys.argv.index('-npredict')
npredict = int(sys.argv[ind+1])
if not os.path.isfile(ftext):
print 'Enter a correct file path !!'
exit()
rnn = vrnn if rnn == 'vrnn' else gru
fname = ftext if ftext else 'pg.txt'
return rnn, fname, npredict
if __name__ == "__main__":
rnn, fname, npredict = parse_argv(sys.argv)
chars = list(set(open(fname, 'r').read()))
i2c_map = {i: chars[i] for i in range(len(chars))} c2i_map = {chars[i]: i for i in
range(len(chars))} v_size = len(chars)
map_vect = {}
for i in range(len(chars)):
map_vect[chars[i]] = np.zeros((v_size, 1))
map_vect[chars[i]][i] = 1.0
# training recurrent model
if os.path.isfile('./weights.pickle'):
model = rnn(v_size, 250, v_size, wb='./weights.pickle')
 else:
model = train(fname, rnn, map_vect)
# testing model accuracy test = 0
if test == 1:
test(fname, model, map_vect)
# console for word prediction
print "Enter partial sentence after '>'"
while True:
text = raw_input('>').strip(' |\n|\t')
if text == 'q' or text == 'Q':
exit()
elif text == '' or text == '\n':
continue
ind = 1
itext = text[-25:]
words = []
count = 0
words = predict_words(itext, model, map_vect, i2c_map, count)
twords = get_top_words(words, npredict)
print "partial sentence:",text
print "Possible endings:"
for word in twords:
print '\t', word, '\t', text+word
OUTPUT