tensorboard usage. RNN structure

RNN/rnn_example.py

@@ -0,0 +1,207 @@
+# -*- codeing: utf-8 -*-
+import numpy as np
+import tensorflow as tf
+# 以下两行用于解决错误： _tkinter.TclError: no display name and no $DISPLAY environment variable
+# import matplotlib as mpl
+# mpl.use("Agg")
+import matplotlib.pyplot as plt
+
+
+def rnn_forward_prop():
+    # define input
+    x = [1, 2]
+    state = [0.0, 0.0]
+
+    # define inner weight and bias
+    w_cell_state = np.asarray([[0.1, 0.2], [0.3, 0.4]])
+    w_cell_input = np.asarray([0.5, 0.6])
+    b_cell = np.asarray([0.1, -0.1])
+
+    # define output weight and bias
+    w_output = np.asarray([1.0, 2.0])
+    b_outoput = 0.1
+
+    for i in range(len(x)):
+        before_activation = np.dot(state, w_cell_state) + x[i] * w_cell_input + b_cell
+        # tanh as activation function
+        state = np.tanh(before_activation)
+
+        final_output = np.dot(state, w_output) + b_outoput
+
+        print("before activation: ", before_activation)
+        print("state: ", state)
+        print("output: ", final_output, "\n")
+
+
+def lstm_structure():
+    lstm_hidden_size = 1
+    batch_size = 10
+    num_steps = 10  # the length of data
+    num_of_layers = 5
+    lstm = tf.nn.rnn_cell.BasicLSTMCell(lstm_hidden_size)
+    # 带dropout的多层循环神经网络
+    stacked_lstm = tf.nn.rnn_cell.MultiRNNCell(
+        [tf.nn.rnn_cell.DropoutWrapper(lstm(lstm_hidden_size)) for _ in range(num_of_layers)]
+    )
+    # state is a LSTEMStateTuple instance with two Tensors, state.c and state.h
+    state = lstm.zero_state(batch_size, tf.float32)
+    loss = 0.0
+    for i in range(num_steps):
+        # assign variables in the first time step, and reuse variables in the following steps
+        if i > 0:
+            tf.get_variable_scope().reuse_variables()
+
+        # current_input represent xt, input prev state ht-1 and ct-1
+        # lstm_output can be sent to other layers, state can be used for the next time step
+        lstm_output, state = lstm(current_input, state)
+
+        # connect to a fc layer to generate the final output
+        final_output = fully_connected(lstm_output)
+
+        # calc current loss
+        loss += calc_loss(final_output, expected_output)
+
+
+HIDDEN_SIZE = 30  # No. of hidden node
+NUM_LAYERS = 2
+
+TIMESTEPS = 10
+TRAINING_STEPS = 10000
+BATCH_SIZE = 32
+
+TRAINING_EXAMPLES = 10000
+TEST_EXAMPLES = 1000
+SAMPlE_GAP = 0.01  # 采样间隔
+
+
+def generate_data(seq):
+    x = []
+    y = []
+    # input from i to i+TIMESTEPS-1
+    # output i+TIMESTEPS, which uses TIMESTEPS samples to predict the TIMESTEPSth result
+    for i in range(len(seq) - TIMESTEPS):
+        x.append([seq[i:i + TIMESTEPS]])
+        y.append([seq[i + TIMESTEPS]])
+    return np.array(x, dtype=np.float32), np.array(y, dtype=np.float32)
+
+
+def lstm_model(x, y, is_training):
+    with tf.name_scope("rnn"):
+        cell = tf.nn.rnn_cell.MultiRNNCell([
+            tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE) for _ in range(NUM_LAYERS)
+        ])
+        # output for every timestep, shape=[batch_size, time, HIDDEN_SIZE]
+        outputs, _ = tf.nn.dynamic_rnn(cell, x, dtype=tf.float32)
+        output = outputs[:, -1, :]
+        # add a fc layer for output
+        predictions = tf.contrib.layers.fully_connected(output, 1, activation_fn=None)
+
+    if not is_training:
+        return predictions, None, None
+
+    with tf.name_scope("loss"):
+        loss = tf.losses.mean_squared_error(labels=y, predictions=predictions)
+
+    # create model optimizer
+    with tf.name_scope("train"):
+        train_op = tf.contrib.layers.optimize_loss(
+            loss, tf.train.get_global_step(),
+            optimizer="Adagrad", learning_rate=0.1
+        )
+
+    return predictions, loss, train_op
+
+
+def train(sess, train_x, train_y, writer):
+    # generate dataset
+    dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y))
+    dataset = dataset.repeat().shuffle(1000).batch(BATCH_SIZE)
+    x, y = dataset.make_one_shot_iterator().get_next()
+
+    with tf.variable_scope("model"):
+        predictions, loss, train_op = lstm_model(x, y, True)
+
+    sess.run(tf.global_variables_initializer())
+    merged = tf.summary.merge_all()
+    for i in range(TRAINING_STEPS):
+
+        if i % 500 == 0:
+            # config necessary info when training
+            run_options = tf.RunOptions(
+                trace_level=tf.RunOptions.FULL_TRACE
+            )
+            # record proto when training
+            run_metadata = tf.RunMetadata()
+
+            summary, _, l = sess.run([merged, train_op, loss], options=run_options, run_metadata=run_metadata)
+            writer.add_run_metadata(run_metadata, 'step%03d' % i)
+            writer.add_summary(summary, i)
+            print("train step: " + str(i) + ", loss: " + str(l))
+        else:
+            _, l = sess.run([train_op, loss])
+
+
+def run_eval(sess, test_x, test_y):
+    # generate dataset
+    dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y))
+    dataset = dataset.batch(1)
+    x, y = dataset.make_one_shot_iterator().get_next()
+
+    with tf.variable_scope("model", reuse=True):
+        # unnecessary to input real y value
+        prediction, _, _ = lstm_model(x, [0.0], False)
+
+    predictions = []
+    labels = []
+    for i in range(TEST_EXAMPLES):
+        p, l = sess.run([prediction, y])
+        predictions.append(p)
+        labels.append(l)
+
+    predictions = np.array(predictions).squeeze()
+    labels = np.array(labels).squeeze()
+    # root mean square error
+    rmse = np.sqrt(((predictions - labels) ** 2).mean(axis=0))
+    print("MSE: %.4f" % rmse)
+
+    # draw fitting lines
+    plt.figure(1)
+    plt.plot(predictions, label="predictions")
+    plt.plot(labels, label="real_sin")
+    plt.legend()
+    plt.show()
+
+
+def sample_generator(data):
+    amps = [0.6, 0.2, 0.8]
+    phases = [0.1, 0.4, 0.7]
+    return amps[0] * np.sin(data + np.pi * phases[0]) + amps[1] * np.sin(data + np.pi * phases[1]) + amps[2] * np.sin(
+        data + np.pi * phases[2])  # + np.random.rand(len(data))*0.02
+
+
+def rnn_example():
+    test_start = (TRAINING_EXAMPLES + TIMESTEPS) * SAMPlE_GAP
+    test_end = test_start * 2
+    train_x, train_y = generate_data(
+        sample_generator(np.linspace(0, test_start, TRAINING_EXAMPLES + TIMESTEPS, dtype=np.float32)))
+    test_x, test_y = generate_data(
+        sample_generator(np.linspace(test_start, test_end, TRAINING_EXAMPLES + TIMESTEPS, dtype=np.float32)))
+
+    # writer = tf.summary.FileWriter("log", tf.get_default_graph())
+    # with tf.device("/gpu:0"):
+    with tf.Session() as sess:
+        writer = tf.summary.FileWriter("log", sess.graph)
+        print("train")
+        train(sess, train_x, train_y, writer)
+        print("eval")
+        run_eval(sess, test_x, test_y)
+        writer.close()
+
+
+def main():
+    # rnn_forward_prop()
+    rnn_example()
+
+
+if __name__ == "__main__":
+    main()

img_proc/mnist_inference.py

@@ -18,6 +18,18 @@ CONV2_SIZE = 5
 FC_SIZE = 84
 
 
+def variable_summaries(var, name):
+    with tf.name_scope('summaries'):
+        tf.summary.histogram(name, var)
+
+        # calc mean
+        mean = tf.reduce_mean(var)
+        tf.summary.scalar('mean/' + name, mean)
+        # calc standard deviation
+        stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
+        tf.summary.scalar('stddev/' + name, stddev)
+
+
 def inference(input_tensor, train, regularizer):
     # print(input_tensor.get_shape())
     # define layer1 forward propagation
@@ -30,6 +42,9 @@ def inference(input_tensor, train, regularizer):
         # strides 中间两项表示长宽方向步长1
         conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')
         relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
+
+        variable_summaries(conv1_weights, 'layer1-conv1' + '/weights')
+        variable_summaries(conv1_biases, 'layer1-conv1' + '/biases')
     # define layer2 forward propagation, max pooling, size 2*2, step 2*2, all 0 filling
     with tf.variable_scope('layer2-pool1'):
         pool1 = tf.nn.max_pool(relu1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
@@ -44,6 +59,8 @@ def inference(input_tensor, train, regularizer):
         conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')
         relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
 
+        variable_summaries(conv2_weights, 'layer3-conv2' + '/weights')
+        variable_summaries(conv2_biases, 'layer3-conv2' + '/biases')
     with tf.variable_scope('layer4-poll2'):
         pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
     # print(pool2.get_shape())
@@ -67,6 +84,8 @@ def inference(input_tensor, train, regularizer):
         if train:
             fc1 = tf.nn.dropout(fc1, 0.5)
 
+        variable_summaries(fc1_weights, 'layer5-fc1' + '/weights')
+        variable_summaries(fc1_biases, 'layer5-fc1' + '/biases')
     with tf.variable_scope('layer6-fc2'):
         fc2_weight = tf.get_variable(
             'weight',
@@ -79,4 +98,6 @@ def inference(input_tensor, train, regularizer):
 
         logit = tf.matmul(fc1, fc2_weight) + fc2_biases
 
+        variable_summaries(fc2_weight, 'layer6-fc2' + '/weights')
+        variable_summaries(fc2_biases, 'layer6-fc2' + '/biases')
     return logit

img_proc/model/checkpoint

@@ -1,3 +1,2 @@
-model_checkpoint_path: "model.ckpt-55000"
-all_model_checkpoint_paths: "model.ckpt-5500"
-all_model_checkpoint_paths: "model.ckpt-55000"
+model_checkpoint_path: "model.ckpt-2750"
+all_model_checkpoint_paths: "model.ckpt-2750"

img_proc/multiThread.py

@@ -268,7 +268,7 @@ def process_data(doTrain=True):
     min_after_dequeue = 2000
     shuffle_buffer = 10000
     num_epochs = 50  # same effect as training_rounds
-    batch_size = 500
+    batch_size = 1000
     training_rounds = 5000
     training_images = 55000  # 362
     validation_images = 5000  # 367
@@ -287,34 +287,41 @@ def process_data(doTrain=True):
                                     image_size,
                                     num_channels], name='x-input')
     y_ = tf.placeholder(tf.float32, [None], name='y-input')
+
     regularizer = tf.contrib.layers.l2_regularizer(regularization_rate)
     y = mnist_inference.inference(x, True, regularizer)
-
     global_step = tf.Variable(0, trainable=False)
 
     # moving average, cross entropy, loss function with regularization and learning rate
-    variable_average = tf.train.ExponentialMovingAverage(moving_average_decay, global_step)
-    variable_average_op = variable_average.apply(tf.trainable_variables())
-    # calc loss
-    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.cast(y_, tf.int32))
-    cross_entropy_mean = tf.reduce_mean(cross_entropy)
-    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
-    learning_rate = tf.train.exponential_decay(
-        learning_rate_base,
-        global_step,
-        training_images / batch_size,
-        learning_rate_decay
-    )
+    with tf.name_scope("moving_average"):
+        variable_average = tf.train.ExponentialMovingAverage(moving_average_decay, global_step)
+        variable_average_op = variable_average.apply(tf.trainable_variables())
 
-    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
-    with tf.control_dependencies([train_step, variable_average_op]):
-        train_op = tf.no_op(name='train')
+    # calc loss
+    with tf.name_scope("loss_function"):
+        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.cast(y_, tf.int32))
+        cross_entropy_mean = tf.reduce_mean(cross_entropy)
+        tf.summary.scalar('cross_entropy', cross_entropy_mean)
+        loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
+
+    with tf.name_scope("train_step"):
+        learning_rate = tf.train.exponential_decay(
+            learning_rate_base,
+            global_step,
+            training_images / batch_size,
+            learning_rate_decay
+        )
+        train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
+        with tf.control_dependencies([train_step, variable_average_op]):
+            train_op = tf.no_op(name='train')
 
     # define accuracy
-    prediction = tf.argmax(y, 1)
-    answer = tf.cast(y_, tf.int64)
-    correct_prediction = tf.equal(tf.argmax(y, 1), tf.cast(y_, tf.int64))
-    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+    with tf.name_scope("accuracy_calc"):
+        prediction = tf.argmax(y, 1)
+        answer = tf.cast(y_, tf.int64)
+        correct_prediction = tf.equal(tf.argmax(y, 1), tf.cast(y_, tf.int64))
+        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+        tf.summary.scalar('accuracy', accuracy)
     # test_result = list(range(int(training_rounds / 500)))
 
     # # ********** original tfrecord data operator **********
@@ -339,7 +346,7 @@ def process_data(doTrain=True):
         lambda image, label: (
             preprocessing.process_for_train(tf.image.convert_image_dtype(image, dtype=tf.float32), image_size,
                                             image_size, None, 1), label
-        # tf.image.resize_images(tf.image.convert_image_dtype(image, dtype=tf.float32), [image_size, image_size]), label
+            # tf.image.resize_images(tf.image.convert_image_dtype(image, dtype=tf.float32), [image_size, image_size]), label
         ))
     dataset = dataset.shuffle(shuffle_buffer).batch(batch_size)
     dataset = dataset.repeat(num_epochs)
@@ -378,7 +385,10 @@ def process_data(doTrain=True):
     saver = tf.train.Saver()
     config = tf.ConfigProto(allow_soft_placement=True)
     config.gpu_options.allow_growth = True
+
+    merged = tf.summary.merge_all()
     with tf.Session(config=config) as sess:
+        writer = tf.summary.FileWriter("log", sess.graph)
         sess.run(tf.global_variables_initializer())
         sess.run(tf.local_variables_initializer())
 
@@ -394,6 +404,7 @@ def process_data(doTrain=True):
             sess.run(validation_iterator.initializer)
             while True:
                 i += 1
+
                 try:
                     # img = sess.run(distorted_img)
                     # plt.imshow(img)
@@ -402,13 +413,28 @@ def process_data(doTrain=True):
                     xs, ys = sess.run([image_batch, label_batch])
                     # print(xs.shape)
                     # print(ys.shape)
-                    _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
 
                     if i % 200 == 0:
+
+                        # config necessary info when training
+                        run_options = tf.RunOptions(
+                            trace_level=tf.RunOptions.FULL_TRACE
+                        )
+                        # record proto when training
+                        run_metadata = tf.RunMetadata()
+                        summary, _, loss_value, step = sess.run([merged, train_op, loss, global_step],
+                                                                feed_dict={x: xs, y_: ys},
+                                                                options=run_options, run_metadata=run_metadata)
+                        writer.add_run_metadata(run_metadata, 'step%03d' % i)
+                        writer.add_summary(summary, i)
                         vxs, vys = sess.run([validation_image_batch, validation_label_batch])
                         p, a, accuracy_score = sess.run([prediction, answer, accuracy], feed_dict={x: vxs, y_: vys})
                         print("prediction: \t%s, \nanswer: \t\t%s" % (p[0:10], a[0:10]))
                         print("after %d steps, loss: %.3f, accuracy: %.3f" % (step, loss_value, accuracy_score))
+                    else:
+                        summary, _, loss_value, step = sess.run([merged, train_op, loss, global_step],
+                                                                feed_dict={x: xs, y_: ys})
+                        writer.add_summary(summary, i)
                 except tf.errors.OutOfRangeError:
                     # i = step
                     break
@@ -436,15 +462,15 @@ def process_data(doTrain=True):
             )
 
         else:
-
             ckpt = tf.train.get_checkpoint_state("model/")
             if ckpt and ckpt.model_checkpoint_path:
                 sess.run(test_iterator.initializer)
                 saver.restore(sess, ckpt.model_checkpoint_path)
-                start = np.random.randint(int(test_images/3), int(test_images/2))
+                start = np.random.randint(int(test_images / 3), int(test_images / 2))
                 length = 10
                 txs, tys = sess.run([test_image_batch, test_label_batch])
-                p, a = sess.run([prediction, answer], feed_dict={x: txs[start:start+length], y_: tys[start:start+length]})
+                p, a = sess.run([prediction, answer],
+                                feed_dict={x: txs[start:start + length], y_: tys[start:start + length]})
                 print("prediction: \t%s, \nanswer: \t\t%s" % (p, a))
 
             else:
@@ -452,6 +478,9 @@ def process_data(doTrain=True):
         coord.request_stop()
         coord.join(threads)
 
+    # writer = tf.summary.FileWriter("log", tf.get_default_graph())
+    writer.close()
+
 
 # ************* dataset operation **************
 def parser(record):
@@ -525,6 +554,8 @@ def dataset_basic_test():
     dataset = dataset.repeat(N)
 
 
+# open tensorboard cmd:
+# tensorboard --logdir=/path/to/log --port=6006
 if __name__ == '__main__':
     # threads_mgmt()
     # generate_files()
@@ -532,5 +563,5 @@ if __name__ == '__main__':
     # batch_example()
     # process_data()
     # generate_record()
-    process_data(doTrain=False)
+    process_data(doTrain=True)
     # dataset_basic_test()

img_proc/preprocessing.py

@@ -109,7 +109,7 @@ def process_for_train(image, height, width, bbox, channels=3):
         image = tf.image.convert_image_dtype(image, dtype=tf.float32)
 
     bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(
-        tf.shape(image), bounding_boxes=bbox
+        tf.shape(image), bounding_boxes=bbox, min_object_covered=0.1
     )
     distorted_img = tf.slice(image, bbox_begin, bbox_size)
     # resize input image for train, all kinds of interpolation