4-CNN-TensorFlow

Score：0.90929

结构如下：

  • x_image=tf.reshape(x,[-1,28,28,1])
  • h_conv1=tf.nn.relu(conv2d(x_image,W_conv1)+b_conv1)
  • h_pool1=max_pool_2x2(h_conv1)
  • h_conv2=tf.nn.relu(conv2d(h_pool1,W_conv2)+b_conv2)
  • h_pool2=max_pool_2x2(h_conv2)
  • h_pool2_flat=tf.reshape(h_pool2,[-1,7764])
  • h_fc1=tf.nn.relu(tf.matmul(h_pool2_flat,W_fc1)+b_fc1)
  • h_fc1_drop=tf.nn.dropout(h_fc1,keep_prob)
  • y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2)+b_fc2)
  • 代价函数：交叉熵
  • 最小化代价函数：AdamOptimizer，学习率1e-4

详细分析见：TensorFlow (4): 卷积神经网络识别手写数字

In [1]:
import pandas as pd
import numpy as np

train_data=pd.read_csv('train.csv')
test_data=pd.read_csv('test.csv')
train_data.shape,test_data.shape

/usr/lib/pymodules/python2.7/matplotlib/__init__.py:907: UserWarning: Duplicate key in file "/etc/matplotlibrc", line #32
  (fname, cnt))
Out[1]:
((42000, 785), (28000, 784))
In [2]:
train_data.head()
Out[2]:
label pixel0 pixel1 pixel2 pixel3 pixel4 pixel5 pixel6 pixel7 pixel8 ... pixel774 pixel775 pixel776 pixel777 pixel778 pixel779 pixel780 pixel781 pixel782 pixel783
0 1 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
2 1 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
3 4 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0

5 rows × 785 columns

In [3]:
images=train_data.drop(['label'],axis=1).values
images.shape
Out[3]:
(42000, 784)
In [4]:
labels=train_data['label']
labels=pd.get_dummies(labels)
print labels.shape
labels.head()

(42000, 10)
Out[4]:
0 1 2 3 4 5 6 7 8 9
0 0 1 0 0 0 0 0 0 0 0
1 1 0 0 0 0 0 0 0 0 0
2 0 1 0 0 0 0 0 0 0 0
3 0 0 0 0 1 0 0 0 0 0
4 1 0 0 0 0 0 0 0 0 0
In [5]:
labels=labels.values
labels.shape
Out[5]:
(42000, 10)
In [6]:
images=np.multiply(images,1.0/255.0)
images.shape
Out[6]:
(42000, 784)
In [7]:
TRAIN_SIZE=40000
train_images=images[:TRAIN_SIZE]
train_labels=labels[:TRAIN_SIZE]
val_images=images[TRAIN_SIZE:]
val_labels=labels[TRAIN_SIZE:]

train_images.shape,len(train_labels),val_images.shape,len(val_labels)
Out[7]:
((40000, 784), 40000, (2000, 784), 2000)
In [8]:
train_images[:5]
Out[8]:
array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])
In [9]:
train_labels[:5]
Out[9]:
array([[0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
       [1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
       [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=uint8)

参数

weight 和 bias

In [10]:
import tensorflow as tf

sess=tf.InteractiveSession()
In [11]:
def weight_variable(shape):
    initial=tf.truncated_normal(shape,stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    initial=tf.constant(0.1,shape=shape)
    return tf.Variable(initial)

卷积和池化

In [12]:
def conv2d(x,W):
    return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')

def max_pool_2x2(x):
    return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
In [13]:
x=tf.placeholder(tf.float32,[None,784])
y_=tf.placeholder(tf.float32,[None,10])
x_image=tf.reshape(x,[-1,28,28,1])
In [14]:
W_conv1=weight_variable([5,5,1,32])
b_conv1=bias_variable([32])

h_conv1=tf.nn.relu(conv2d(x_image,W_conv1)+b_conv1)
h_pool1=max_pool_2x2(h_conv1)
In [15]:
W_conv2=weight_variable([5,5,32,64])
b_conv2=bias_variable([64])

h_conv2=tf.nn.relu(conv2d(h_pool1,W_conv2)+b_conv2)
h_pool2=max_pool_2x2(h_conv2)
In [16]:
W_fc1=weight_variable([7*7*64,1024])
b_fc1=bias_variable([1024])

h_pool2_flat=tf.reshape(h_pool2,[-1,7*7*64])
h_fc1=tf.nn.relu(tf.matmul(h_pool2_flat,W_fc1)+b_fc1)
In [17]:
keep_prob=tf.placeholder(tf.float32)
h_fc1_drop=tf.nn.dropout(h_fc1,keep_prob)
In [18]:
W_fc2=weight_variable([1024,10])
b_fc2=bias_variable([10])

y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2)+b_fc2)
In [19]:
cross_entropy=-tf.reduce_sum(y_*tf.log(y_conv))
train_step=tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

训练模型

In [20]:
BATCH_SIZE=50
index_in_epoch=0
num_examples=train_images.shape[0]

def next_batch(batch_size):
    global train_images
    global train_labels
    global index_in_epoch
    
    start=index_in_epoch
    index_in_epoch+=batch_size
    
    if index_in_epoch>num_examples:
        print 'epoches copleted!'
        # 一轮完毕，shuffle数据
        perm=np.arange(num_examples) # 生成一个序列
        np.random.shuffle(perm)
        train_images=train_images[perm]
        train_labels=train_labels[perm]
        # 开始下个epoch
        start=0
        index_in_epoch=batch_size
        assert batch_size<=num_examples
    end=index_in_epoch
    return train_images[start:end],train_labels[start:end]
In [21]:
correct_prediction=tf.equal(tf.argmax(y_conv,1),tf.argmax(y_,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,'float'))
In [22]:
tf.global_variables_initializer().run()
for i in range(16000):
    batch_xs,batch_ys=next_batch(BATCH_SIZE)
    if i%100==0:
        train_accuracy=accuracy.eval({x:batch_xs,y_:batch_ys,keep_prob:1.0})
        test_accuracy=accuracy.eval({x:val_images[:BATCH_SIZE],y_:val_labels[:BATCH_SIZE],keep_prob:1.0})
        print 'step:{}, train_accuracy={},test_accuracy={}'.format(i,train_accuracy,test_accuracy)
    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys,keep_prob:0.75})

step:0, train_accuracy=0.0399999991059,test_accuracy=0.10000000149
step:100, train_accuracy=0.759999990463,test_accuracy=0.740000009537
step:200, train_accuracy=0.939999997616,test_accuracy=0.839999973774
step:300, train_accuracy=1.0,test_accuracy=0.800000011921
step:400, train_accuracy=0.939999997616,test_accuracy=0.860000014305
step:500, train_accuracy=0.939999997616,test_accuracy=0.879999995232
step:600, train_accuracy=0.980000019073,test_accuracy=0.879999995232
step:700, train_accuracy=1.0,test_accuracy=0.899999976158
epoches copleted!
step:800, train_accuracy=0.920000016689,test_accuracy=0.860000014305
step:900, train_accuracy=0.920000016689,test_accuracy=0.920000016689
step:1000, train_accuracy=0.959999978542,test_accuracy=0.879999995232
step:1100, train_accuracy=0.939999997616,test_accuracy=0.899999976158
step:1200, train_accuracy=0.980000019073,test_accuracy=0.920000016689
step:1300, train_accuracy=0.899999976158,test_accuracy=0.959999978542
step:1400, train_accuracy=1.0,test_accuracy=0.939999997616
step:1500, train_accuracy=0.959999978542,test_accuracy=0.920000016689
epoches copleted!
step:1600, train_accuracy=0.959999978542,test_accuracy=0.899999976158
step:1700, train_accuracy=0.980000019073,test_accuracy=0.899999976158
step:1800, train_accuracy=0.959999978542,test_accuracy=0.920000016689
step:1900, train_accuracy=1.0,test_accuracy=0.939999997616
step:2000, train_accuracy=0.980000019073,test_accuracy=0.939999997616
step:2100, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:2200, train_accuracy=1.0,test_accuracy=0.959999978542
step:2300, train_accuracy=1.0,test_accuracy=0.959999978542
epoches copleted!
step:2400, train_accuracy=1.0,test_accuracy=0.939999997616
step:2500, train_accuracy=0.980000019073,test_accuracy=0.939999997616
step:2600, train_accuracy=1.0,test_accuracy=0.920000016689
step:2700, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:2800, train_accuracy=1.0,test_accuracy=0.939999997616
step:2900, train_accuracy=1.0,test_accuracy=0.939999997616
step:3000, train_accuracy=0.980000019073,test_accuracy=0.939999997616
step:3100, train_accuracy=1.0,test_accuracy=0.939999997616
epoches copleted!
step:3200, train_accuracy=1.0,test_accuracy=0.939999997616
step:3300, train_accuracy=0.959999978542,test_accuracy=0.939999997616
step:3400, train_accuracy=1.0,test_accuracy=0.959999978542
step:3500, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:3600, train_accuracy=1.0,test_accuracy=0.939999997616
step:3700, train_accuracy=0.980000019073,test_accuracy=0.939999997616
step:3800, train_accuracy=1.0,test_accuracy=0.980000019073
step:3900, train_accuracy=0.959999978542,test_accuracy=0.980000019073
epoches copleted!
step:4000, train_accuracy=0.980000019073,test_accuracy=1.0
step:4100, train_accuracy=1.0,test_accuracy=0.959999978542
step:4200, train_accuracy=1.0,test_accuracy=0.959999978542
step:4300, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:4400, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:4500, train_accuracy=0.959999978542,test_accuracy=0.980000019073
step:4600, train_accuracy=1.0,test_accuracy=0.939999997616
step:4700, train_accuracy=1.0,test_accuracy=0.959999978542
epoches copleted!
step:4800, train_accuracy=1.0,test_accuracy=0.959999978542
step:4900, train_accuracy=0.980000019073,test_accuracy=0.939999997616
step:5000, train_accuracy=0.980000019073,test_accuracy=1.0
step:5100, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:5200, train_accuracy=1.0,test_accuracy=0.980000019073
step:5300, train_accuracy=1.0,test_accuracy=0.959999978542
step:5400, train_accuracy=1.0,test_accuracy=0.980000019073
step:5500, train_accuracy=0.980000019073,test_accuracy=1.0
epoches copleted!
step:5600, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:5700, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:5800, train_accuracy=1.0,test_accuracy=0.980000019073
step:5900, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:6000, train_accuracy=1.0,test_accuracy=0.959999978542
step:6100, train_accuracy=1.0,test_accuracy=0.959999978542
step:6200, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:6300, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:6400, train_accuracy=1.0,test_accuracy=1.0
step:6500, train_accuracy=0.980000019073,test_accuracy=0.959999978542
step:6600, train_accuracy=1.0,test_accuracy=1.0
step:6700, train_accuracy=1.0,test_accuracy=0.980000019073
step:6800, train_accuracy=0.980000019073,test_accuracy=1.0
step:6900, train_accuracy=1.0,test_accuracy=0.980000019073
step:7000, train_accuracy=1.0,test_accuracy=0.980000019073
step:7100, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:7200, train_accuracy=1.0,test_accuracy=0.980000019073
step:7300, train_accuracy=1.0,test_accuracy=0.980000019073
step:7400, train_accuracy=1.0,test_accuracy=0.980000019073
step:7500, train_accuracy=1.0,test_accuracy=0.959999978542
step:7600, train_accuracy=1.0,test_accuracy=0.980000019073
step:7700, train_accuracy=1.0,test_accuracy=0.980000019073
step:7800, train_accuracy=1.0,test_accuracy=1.0
step:7900, train_accuracy=1.0,test_accuracy=0.980000019073
epoches copleted!
step:8000, train_accuracy=1.0,test_accuracy=1.0
step:8100, train_accuracy=1.0,test_accuracy=1.0
step:8200, train_accuracy=1.0,test_accuracy=1.0
step:8300, train_accuracy=0.980000019073,test_accuracy=0.980000019073
step:8400, train_accuracy=1.0,test_accuracy=1.0
step:8500, train_accuracy=1.0,test_accuracy=0.980000019073
step:8600, train_accuracy=1.0,test_accuracy=0.980000019073
step:8700, train_accuracy=0.980000019073,test_accuracy=0.980000019073
epoches copleted!
step:8800, train_accuracy=1.0,test_accuracy=0.980000019073
step:8900, train_accuracy=1.0,test_accuracy=0.959999978542
step:9000, train_accuracy=1.0,test_accuracy=0.980000019073
step:9100, train_accuracy=1.0,test_accuracy=1.0
step:9200, train_accuracy=1.0,test_accuracy=0.959999978542
step:9300, train_accuracy=1.0,test_accuracy=0.980000019073
step:9400, train_accuracy=1.0,test_accuracy=1.0
step:9500, train_accuracy=0.980000019073,test_accuracy=0.980000019073
epoches copleted!
step:9600, train_accuracy=1.0,test_accuracy=1.0
step:9700, train_accuracy=1.0,test_accuracy=0.980000019073
step:9800, train_accuracy=1.0,test_accuracy=0.980000019073
step:9900, train_accuracy=1.0,test_accuracy=1.0
step:10000, train_accuracy=1.0,test_accuracy=1.0
step:10100, train_accuracy=1.0,test_accuracy=0.959999978542
step:10200, train_accuracy=1.0,test_accuracy=0.980000019073
step:10300, train_accuracy=1.0,test_accuracy=0.980000019073
epoches copleted!
step:10400, train_accuracy=1.0,test_accuracy=0.959999978542
step:10500, train_accuracy=1.0,test_accuracy=1.0
step:10600, train_accuracy=1.0,test_accuracy=1.0
step:10700, train_accuracy=1.0,test_accuracy=1.0
step:10800, train_accuracy=1.0,test_accuracy=1.0
step:10900, train_accuracy=1.0,test_accuracy=1.0
step:11000, train_accuracy=1.0,test_accuracy=1.0
step:11100, train_accuracy=1.0,test_accuracy=0.980000019073
epoches copleted!
step:11200, train_accuracy=1.0,test_accuracy=0.980000019073
step:11300, train_accuracy=1.0,test_accuracy=0.980000019073
step:11400, train_accuracy=1.0,test_accuracy=0.980000019073
step:11500, train_accuracy=1.0,test_accuracy=0.980000019073
step:11600, train_accuracy=1.0,test_accuracy=1.0
step:11700, train_accuracy=1.0,test_accuracy=1.0
step:11800, train_accuracy=1.0,test_accuracy=0.980000019073
step:11900, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:12000, train_accuracy=1.0,test_accuracy=1.0
step:12100, train_accuracy=1.0,test_accuracy=1.0
step:12200, train_accuracy=1.0,test_accuracy=1.0
step:12300, train_accuracy=1.0,test_accuracy=1.0
step:12400, train_accuracy=1.0,test_accuracy=1.0
step:12500, train_accuracy=1.0,test_accuracy=1.0
step:12600, train_accuracy=1.0,test_accuracy=1.0
step:12700, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:12800, train_accuracy=1.0,test_accuracy=1.0
step:12900, train_accuracy=1.0,test_accuracy=0.980000019073
step:13000, train_accuracy=1.0,test_accuracy=1.0
step:13100, train_accuracy=1.0,test_accuracy=0.959999978542
step:13200, train_accuracy=1.0,test_accuracy=1.0
step:13300, train_accuracy=1.0,test_accuracy=1.0
step:13400, train_accuracy=1.0,test_accuracy=1.0
step:13500, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:13600, train_accuracy=1.0,test_accuracy=1.0
step:13700, train_accuracy=1.0,test_accuracy=0.980000019073
step:13800, train_accuracy=1.0,test_accuracy=0.980000019073
step:13900, train_accuracy=1.0,test_accuracy=1.0
step:14000, train_accuracy=1.0,test_accuracy=1.0
step:14100, train_accuracy=1.0,test_accuracy=1.0
step:14200, train_accuracy=1.0,test_accuracy=0.980000019073
step:14300, train_accuracy=1.0,test_accuracy=1.0
epoches copleted!
step:14400, train_accuracy=1.0,test_accuracy=0.980000019073
step:14500, train_accuracy=1.0,test_accuracy=1.0
step:14600, train_accuracy=1.0,test_accuracy=0.980000019073
step:14700, train_accuracy=1.0,test_accuracy=1.0
step:14800, train_accuracy=1.0,test_accuracy=1.0
step:14900, train_accuracy=1.0,test_accuracy=1.0
step:15000, train_accuracy=1.0,test_accuracy=1.0
step:15100, train_accuracy=1.0,test_accuracy=0.980000019073
epoches copleted!
step:15200, train_accuracy=1.0,test_accuracy=1.0
step:15300, train_accuracy=0.980000019073,test_accuracy=1.0
step:15400, train_accuracy=1.0,test_accuracy=1.0
step:15500, train_accuracy=1.0,test_accuracy=0.980000019073
step:15600, train_accuracy=1.0,test_accuracy=1.0
step:15700, train_accuracy=1.0,test_accuracy=1.0
step:15800, train_accuracy=1.0,test_accuracy=1.0
step:15900, train_accuracy=1.0,test_accuracy=1.0
In [23]:
val_accuracy=accuracy.eval({x:val_images,y_:val_labels,keep_prob:1.0})
val_accuracy
# 提交分数0.98825
Out[23]:
0.98699999
In [24]:
test_data.head()
Out[24]:
pixel0 pixel1 pixel2 pixel3 pixel4 pixel5 pixel6 pixel7 pixel8 pixel9 ... pixel774 pixel775 pixel776 pixel777 pixel778 pixel779 pixel780 pixel781 pixel782 pixel783
0 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0

5 rows × 784 columns

In [25]:
test_images=test_data.values
test_images=np.multiply(test_images,1.0/255.0)
predict = tf.argmax(y_conv,1)
predicted_labels=predict.eval(feed_dict={x:test_images,keep_prob:1.0})
predicted_labels[:5]
Out[25]:
array([2, 0, 9, 0, 3])
In [27]:
submissions=pd.DataFrame({"ImageId": list(range(1,len(predicted_labels)+1)),
                         "Label": predicted_labels})
submissions.to_csv('submission_4.csv',index=False)