restructured nn for mnist number recognition, together with saved nn weights, diagrams and photos

mnist_restructured/mnist_eval.py

@@ -0,0 +1,106 @@
+# -*- coding: utf8 -*-
+import time
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+import matplotlib.pyplot as plt
+import mnist_inference
+import mnist_train
+from numpy.random import RandomState
+import os
+
+# generate new random dataset for test in 3 secs after close figure window manually
+EVAL_INTERVAL_SECS = 3
+NUMBER_OF_SAMPLES = 12
+FIG_ROWS = 3
+
+
+# display images and recognition result rather than accuracy diagram
+def evaluation(mnist):
+    with tf.Graph().as_default() as g:
+        x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='input-x')
+        y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='input-y')
+
+        # move sample picking into each cycle
+        # rdm = RandomState(int(time.time()))
+        # sample_index = rdm.randint(0, mnist.validation.num_examples)
+        # validation_feed = {
+        #     x: mnist.validation.images[sample_index:sample_index + 6],
+        #     y_: mnist.validation.labels[sample_index:sample_index + 6]}
+
+        # replace accuracy with actual recognition result
+        y = mnist_inference.inference(x, None)
+        indices = tf.argmax(y, 1)
+        correct_indices = tf.argmax(y_, 1)
+        # correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
+        # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+        variable_averages = tf.train.ExponentialMovingAverage(mnist_train.MOVING_AVERAGE_DECAY)
+        variables_to_restore = variable_averages.variables_to_restore()
+        saver = tf.train.Saver(variables_to_restore)
+
+        while True:
+            # configure TF to allocate mem properly, rather than consume all GPU mem
+            config = tf.ConfigProto(allow_soft_placement=True)
+            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
+            config.gpu_options.allow_growth = True
+            with tf.Session(config=config) as sess:
+                ckpt = tf.train.get_checkpoint_state(
+                    mnist_train.MODEL_SAVE_PATH
+                )
+                if ckpt and ckpt.model_checkpoint_path:
+                    saver.restore(sess, ckpt.model_checkpoint_path)
+                    rdm = RandomState(int(time.time()))
+                    sample_index = rdm.randint(0, mnist.validation.num_examples - NUMBER_OF_SAMPLES)
+                    validation_feed = {
+                        x: mnist.validation.images[sample_index:sample_index + NUMBER_OF_SAMPLES],
+                        y_: mnist.validation.labels[sample_index:sample_index + NUMBER_OF_SAMPLES]}
+
+                    # get global step from file name
+                    global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
+                    indices_score, correct_indices_score = sess.run(
+                        [indices, correct_indices], feed_dict=validation_feed)
+                    # accuracy_score = sess.run(accuracy, feed_dict=validation_feed)
+                    # print "after %s training step(s), validation accuracy = %g" % (global_step, accuracy_score)
+                    print "after %s training step(s), validation result = %s, correct answer: %s" \
+                          % (global_step, indices_score, correct_indices_score)
+                    fig = plt.figure(1)
+                    for n in range(1, NUMBER_OF_SAMPLES + 1):
+                        fig.add_subplot(FIG_ROWS, (NUMBER_OF_SAMPLES / FIG_ROWS + 1), n)
+                        plt.title("predict: [%s]\nanswer: [%s]"
+                                  % (indices_score[n - 1], correct_indices_score[n - 1]))
+                        plt.imshow(mnist.validation.images[sample_index + n - 1].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 1)
+                    # plt.imshow(mnist.validation.images[sample_index].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 2)
+                    # plt.imshow(mnist.validation.images[sample_index + 1].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 3)
+                    # plt.imshow(mnist.validation.images[sample_index + 2].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 4)
+                    # plt.imshow(mnist.validation.images[sample_index + 3].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 5)
+                    # plt.imshow(mnist.validation.images[sample_index + 4].reshape(28, 28))
+                    # fig.add_subplot(2, 3, 6)
+                    # plt.imshow(mnist.validation.images[sample_index + 5].reshape(28, 28))
+                    plt.subplots_adjust(
+                        top=0.9, bottom=0.1, left=0.1, right=0.9, hspace=0.5, wspace=0.55)
+                    try:
+                        os.mkdir('images/')
+                    except:
+                        print("directory already exist")
+                    plt.savefig('images/mnist_result_evaluation.jpg', format='jpg')
+                    plt.show()
+
+                else:
+                    print "no checkpoint file found"
+                    return
+
+            time.sleep(EVAL_INTERVAL_SECS)
+
+
+def main(argv=None):
+    mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
+    evaluation(mnist)
+
+
+if __name__ == '__main__':
+    tf.app.run()

mnist_restructured/mnist_inference.py

@@ -0,0 +1,33 @@
+# -*- coding: utf8 -*-
+import tensorflow as tf
+
+# define basic params
+INPUT_NODE = 784
+OUTPUT_NODE = 10
+LAYER1_NODE = 500
+
+
+def get_weight_variable(shape, regularizer):
+    # init and get weights
+    weights = tf.get_variable("weights", shape, initializer=tf.truncated_normal_initializer(stddev=0.1))
+    # save regularization loss
+    if regularizer is not None:
+        tf.add_to_collection('losses', regularizer(weights))
+    return weights
+
+
+def inference(input_tensor, regularizer):
+    # define layer1 forward propagation
+    with tf.variable_scope('layer1'):
+        weights = get_weight_variable([INPUT_NODE, LAYER1_NODE], regularizer)
+        biases = tf.get_variable("biases", [LAYER1_NODE], initializer=tf.constant_initializer(0.0))
+        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights) + biases)
+
+    # define layer2 forward propagation
+    with tf.variable_scope('layer2'):
+        weights = get_weight_variable([LAYER1_NODE, OUTPUT_NODE], regularizer)
+        biases = tf.get_variable("biases", [OUTPUT_NODE], initializer=tf.constant_initializer(0.0))
+        layer2 = tf.matmul(layer1, weights) + biases
+
+    return layer2
+

mnist_restructured/mnist_train.py

@@ -0,0 +1,81 @@
+# -*- coding: utf8 -*-
+import os
+
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+
+import mnist_inference
+
+# define input, output, batch and training params
+
+BATCH_SIZE = 1000
+LEARNING_RATE_BASE = 0.8
+LEARNING_RATE_DECAY = 0.99
+REGULARIZATION_RATE = 0.0001
+TRAINING_STEPS = 50000
+MOVING_AVERAGE_DECAY = 0.99
+
+MODEL_SAVE_PATH = "model/"
+MODEL_NAME = "model.ckpt"
+
+
+def train(mnist):
+    x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')
+    y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')
+
+    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
+    y = mnist_inference.inference(x, regularizer)
+    global_step = tf.Variable(0, trainable=False)
+
+    # moving average, cross entropy, loss function with regularization and learning rate
+    variable_average = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
+    variable_average_op = variable_average.apply(tf.trainable_variables())
+    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
+    cross_entropy_mean = tf.reduce_mean(cross_entropy)
+    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
+    learning_rate = tf.train.exponential_decay(
+        LEARNING_RATE_BASE,
+        global_step,
+        mnist.train.num_examples / BATCH_SIZE,
+        LEARNING_RATE_DECAY
+    )
+
+    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
+    with tf.control_dependencies([train_step, variable_average_op]):
+        train_op = tf.no_op(name='train')
+
+    # initialize persistence class
+    saver = tf.train.Saver()
+
+    config = tf.ConfigProto(allow_soft_placement=True)
+    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
+    config.gpu_options.allow_growth = True
+    with tf.Session(config=config) as sess:
+        sess.run(tf.global_variables_initializer())
+        # create directory
+        try:
+            os.mkdir(MODEL_SAVE_PATH)
+        except:
+            print("directory already exist")
+
+        for i in range(TRAINING_STEPS):
+            xs, ys = mnist.train.next_batch(BATCH_SIZE)
+
+            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
+
+            if i % 2500 == 0:
+                print "after %d training step(s), loss on training batch is %g " % (step, loss_value)
+                saver.save(
+                    sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step
+                )
+
+
+def main(argv=None):
+    print "start"
+    mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
+    print "start"
+    train(mnist)
+
+
+if __name__ == '__main__':
+    tf.app.run()

mnist_restructured/model/checkpoint

@@ -0,0 +1,6 @@
+model_checkpoint_path: "model.ckpt-47501"
+all_model_checkpoint_paths: "model.ckpt-37501"
+all_model_checkpoint_paths: "model.ckpt-40001"
+all_model_checkpoint_paths: "model.ckpt-42501"
+all_model_checkpoint_paths: "model.ckpt-45001"
+all_model_checkpoint_paths: "model.ckpt-47501"

tests/forward_propagation.py

@@ -20,3 +20,6 @@ with tf.Session() as sess:
     # sess.run(w2.initializer)
     sess.run(tf.global_variables_initializer())
     print (sess.run(y, feed_dict={x: [[0.7, 0.9], [0.1, 0.4], [0.5, 0.8]]}))
+
+    saver = tf.train.Saver()
+    saver.export_meta_graph("model.ckpt.meda.json", as_text=True)