# 滑动平均模型在Tensorflow中的应用

0.399字数 435阅读 3469

### 操作步骤

1 训练阶段：为每个可训练的权重维护影子变量，并随着迭代的进行更新；
2 预测阶段：使用影子变量替代真实变量值，进行预测。

### 结果

#### 不使用滑动平均模型

``````After 0 training steps, validation accuracy is 0.16740000247955322
After 1000 training steps, validation accuracy is 0.9747997522354126
After 2000 training steps, validation accuracy is 0.9775997400283813
After 3000 training steps, validation accuracy is 0.9811996817588806
After 4000 training steps, validation accuracy is 0.9805997014045715
after 5000 steps, test accuracy is 0.9790000915527344
``````

#### 使用滑动平均模型

``````After 0 training steps, validation accuracy is 0.16499999165534973
After 1000 training steps, validation accuracy is 0.9763997197151184
After 2000 training steps, validation accuracy is 0.9829997420310974
After 3000 training steps, validation accuracy is 0.9825997352600098
After 4000 training steps, validation accuracy is 0.9843996167182922
after 5000 steps, test accuracy is 0.9821001291275024
``````

### 代码

``````import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

BATCH_SIZE = 100
INPUT_NODE = 784
OUTPUT_NODE = 10
LAYER1_NODE = 500

LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99

REGULARIZATION_RATE = 0.0001

TRAINING_STEPS = 5000

MOVING_AVERAGE_DECAY = 0.9999

def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

def train(mnist):
x = tf.placeholder(tf.float32, shape=[None, INPUT_NODE], name='input_x')
y_ = tf.placeholder(tf.float32, shape=[None, OUTPUT_NODE], name='input_y')

weights1 = tf.Variable(tf.truncated_normal(shape=[INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))

weights2 = tf.Variable(tf.truncated_normal(shape=[LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

y = inference(x, None, weights1=weights1, biases1=biases1, weights2=weights2, biases2=biases2)

global_step = tf.Variable(0, trainable=False)

variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())

average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits= y,labels= tf.arg_max(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
regularization = regularizer(weights1) + regularizer(weights2)

loss = cross_entropy_mean + regularization

learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True
)

with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

with tf.Session() as sess:
tf.global_variables_initializer().run()

validate_feed = {
x: mnist.validation.images,
y_: mnist.validation.labels
}

test_feed = {
x: mnist.test.images,
y_: mnist.test.labels
}

for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print(f'After {i} training steps, validation accuracy is {validate_acc}')

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(f'after {TRAINING_STEPS} steps, test accuracy is {test_acc}')

def main(argv=None):
train(mnist)

if __name__ == '__main__':
main()

``````

### 总结

1 滑动平均模型在梯段下降算法上才会有好的结果，别的优化算法没有这个现象，没见到合理的解释。
2 优化的方法有很多，可能这个可以作为最后的提高健壮性的错失。