04_modern_net

import sys
sys.path.append('../')

#!/usr/bin/env python

import tensorflow as tf
import numpy as np
import input_data


def init_weights(shape):
    return tf.Variable(tf.random_normal(shape, stddev=0.01))


def model(X, w_h, w_h2, w_o, p_keep_input, p_keep_hidden): # this network is the same as the previous one except with an extra hidden layer + dropout
    X = tf.nn.dropout(X, p_keep_input)
    h = tf.nn.relu(tf.matmul(X, w_h))

    h = tf.nn.dropout(h, p_keep_hidden)
    h2 = tf.nn.relu(tf.matmul(h, w_h2))

    h2 = tf.nn.dropout(h2, p_keep_hidden)

    return tf.matmul(h2, w_o)


mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels

X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])

w_h = init_weights([784, 625])
w_h2 = init_weights([625, 625])
w_o = init_weights([625, 10])

p_keep_input = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w_h, w_h2, w_o, p_keep_input, p_keep_hidden)

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y))
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)
predict_op = tf.argmax(py_x, 1)

# Launch the graph in a session
with tf.Session() as sess:
    # you need to initialize all variables
    tf.initialize_all_variables().run()

    for i in range(100):
        for start, end in zip(range(0, len(trX), 128), range(128, len(trX)+1, 128)):
            sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end],
                                          p_keep_input: 0.8, p_keep_hidden: 0.5})
        print(i, np.mean(np.argmax(teY, axis=1) ==
                         sess.run(predict_op, feed_dict={X: teX, Y: teY,
                                                         p_keep_input: 1.0,
                                                         p_keep_hidden: 1.0})))

('Extracting', 'MNIST_data/train-images-idx3-ubyte.gz')
('Extracting', 'MNIST_data/train-labels-idx1-ubyte.gz')
('Extracting', 'MNIST_data/t10k-images-idx3-ubyte.gz')
('Extracting', 'MNIST_data/t10k-labels-idx1-ubyte.gz')
(0, 0.93589999999999995)
(1, 0.96489999999999998)
(2, 0.97099999999999997)
(3, 0.97319999999999995)
(4, 0.97570000000000001)
(5, 0.97640000000000005)
(6, 0.97840000000000005)
(7, 0.97650000000000003)
(8, 0.97940000000000005)
(9, 0.9788)
(10, 0.97909999999999997)
(11, 0.98099999999999998)
(12, 0.98070000000000002)
(13, 0.98109999999999997)
(14, 0.98160000000000003)
(15, 0.98019999999999996)
(16, 0.98019999999999996)
(17, 0.98060000000000003)
(18, 0.98080000000000001)
(19, 0.98150000000000004)
(20, 0.98219999999999996)
(21, 0.98119999999999996)
(22, 0.98199999999999998)
(23, 0.98229999999999995)
(24, 0.9839)
(25, 0.98329999999999995)
(26, 0.98340000000000005)
(27, 0.98460000000000003)
(28, 0.98319999999999996)
(29, 0.98270000000000002)
(30, 0.98350000000000004)
(31, 0.98399999999999999)
(32, 0.98309999999999997)
(33, 0.98160000000000003)
(34, 0.98329999999999995)
(35, 0.98309999999999997)
(36, 0.98340000000000005)
(37, 0.98260000000000003)
(38, 0.98450000000000004)
(39, 0.98380000000000001)
(40, 0.98319999999999996)
(41, 0.98319999999999996)
(42, 0.98440000000000005)
(43, 0.98350000000000004)
(44, 0.98380000000000001)
(45, 0.98309999999999997)
(46, 0.98380000000000001)
(47, 0.98419999999999996)
(48, 0.98380000000000001)
(49, 0.98409999999999997)
(50, 0.9839)
(51, 0.98319999999999996)
(52, 0.98419999999999996)
(53, 0.98360000000000003)
(54, 0.98340000000000005)
(55, 0.98480000000000001)
(56, 0.98360000000000003)
(57, 0.98440000000000005)
(58, 0.9849)
(59, 0.98380000000000001)
(60, 0.98399999999999999)
(61, 0.98360000000000003)
(62, 0.98409999999999997)
(63, 0.98580000000000001)
(64, 0.98460000000000003)
(65, 0.98399999999999999)
(66, 0.98460000000000003)
(67, 0.9849)
(68, 0.98529999999999995)
(69, 0.98370000000000002)
(70, 0.98450000000000004)
(71, 0.98450000000000004)
(72, 0.98570000000000002)
(73, 0.98429999999999995)
(74, 0.98529999999999995)
(75, 0.98540000000000005)
(76, 0.98409999999999997)
(77, 0.98409999999999997)
(78, 0.98519999999999996)
(79, 0.98440000000000005)
(80, 0.98529999999999995)
(81, 0.98480000000000001)
(82, 0.98540000000000005)
(83, 0.98550000000000004)
(84, 0.98370000000000002)
(85, 0.98460000000000003)
(86, 0.98619999999999997)
(87, 0.98580000000000001)
(88, 0.98460000000000003)
(89, 0.98470000000000002)
(90, 0.98509999999999998)
(91, 0.98609999999999998)
(92, 0.98419999999999996)
(93, 0.98529999999999995)
(94, 0.98670000000000002)
(95, 0.98550000000000004)
(96, 0.98529999999999995)
(97, 0.98540000000000005)
(98, 0.98480000000000001)
(99, 0.98499999999999999)