# TensorFlow 高效编程

## 一、TensorFlow 基础

TensorFlow 和其他数字计算库（如 numpy）之间最明显的区别在于 TensorFlow 中操作的是符号。这是一个强大的功能，这保证了 TensorFlow 可以做很多其他库（例如 numpy）不能完成的事情（例如自动区分）。这可能也是它更复杂的原因。今天我们来一步步探秘 TensorFlow，并为更有效地使用 TensorFlow 提供了一些指导方针和最佳实践。

``````import numpy as np
x = np.random.normal(size=[10, 10])
y = np.random.normal(size=[10, 10])
z = np.dot(x, y)
print(z)

``````

``````import TensorFlow as tf
x = tf.random_normal([10, 10])
y = tf.random_normal([10, 10])
z = tf.matmul(x, y)
sess = tf.Session()
z_val = sess.run(z)
print(z_val)

``````

``````Tensor("MatMul:0", shape=(10, 10), dtype=float32)
``````

``````import numpy as np
import TensorFlow as tf
x = tf.placeholder(tf.float32)
y = tf.placeholder(tf.float32)
w = tf.get_variable("w", shape=[3, 1])
f = tf.stack([tf.square(x), x, tf.ones_like(x)], 1)
yhat = tf.squeeze(tf.matmul(f, w), 1)
loss = tf.nn.l2_loss(yhat - y) + 0.1 * tf.nn.l2_loss(w)
def generate_data():
x_val = np.random.uniform(-10.0, 10.0, size=100)
y_val = 5 * np.square(x_val) + 3
return x_val, y_val
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for _ in range(1000):
x_val, y_val = generate_data()
_, loss_val = sess.run([train_op, loss], {x: x_val, y: y_val})
print(loss_val)
print(sess.run([w]))

``````

``````[4.9924135, 0.00040895029, 3.4504161]
``````

## 二、理解静态和动态形状

TensorFlow 中，`tensor`有一个在图构建过程中就被决定的静态形状属性， 这个静态形状可以是未规定的，比如，我们可以定一个具有形状`[None, 128]`大小的`tensor`

``````import TensorFlow as tf
a = tf.placeholder(tf.float32, [None, 128])
``````

``````static_shape = a.shape.as_list()  # returns [None, 128]
``````

``````dynamic_shape = tf.shape(a)
``````

`tensor`的静态形状可以通过方法`Tensor_name.set_shape()`设定，如：

``````a.set_shape([32, 128])  # static shape of a is [32, 128]
a.set_shape([None, 128])  # first dimension of a is determined dynamically
``````

``````a =  tf.reshape(a, [32, 128])
``````

``````def get_shape(tensor):
static_shape = tensor.shape.as_list()
dynamic_shape = tf.unstack(tf.shape(tensor))
dims = [s[1] if s[0] is None else s[0]
for s in zip(static_shape, dynamic_shape)]
return dims
``````

``````b = tf.placeholder(tf.float32, [None, 10, 32])
shape = get_shape(b)
b = tf.reshape(b, [shape[0], shape[1] * shape[2]])
``````

``````import TensorFlow as tf
import numpy as np

def reshape(tensor, dims_list):
shape = get_shape(tensor)
dims_prod = []
for dims in dims_list:
if isinstance(dims, int):
dims_prod.append(shape[dims])
elif all([isinstance(shape[d], int) for d in dims]):
dims_prod.append(np.prod([shape[d] for d in dims]))
else:
dims_prod.append(tf.prod([shape[d] for d in dims]))
tensor = tf.reshape(tensor, dims_prod)
return tensor
``````

``````b = tf.placeholder(tf.float32, [None, 10, 32])
b = reshape(b, [0, [1, 2]])
``````

## 三、作用域和何时使用它

``````a = tf.constant(1)
print(a.name)  # prints "Const:0"

b = tf.Variable(1)
print(b.name)  # prints "Variable:0"
``````

``````a = tf.constant(1, name="a")
print(a.name)  # prints "b:0"

b = tf.Variable(1, name="b")
print(b.name)  # prints "b:0"
``````

TF 引进了两个不同的上下文管理器，用于更改张量或者变量的名字，第一个就是`tf.name_scope`，如：

``````with tf.name_scope("scope"):
a = tf.constant(1, name="a")
print(a.name)  # prints "scope/a:0"

b = tf.Variable(1, name="b")
print(b.name)  # prints "scope/b:0"

c = tf.get_variable(name="c", shape=[])
print(c.name)  # prints "c:0"
``````

`tf.name_scope()`只会影响到通过调用`tf.Variable`创建的张量和变量的名字，而不会影响到通过调用`tf.get_variable()`创建的变量和张量。

`tf.name_scope()`不同，`tf.variable_scope()`也会修改，影响通过`tf.get_variable()`创建的变量和张量，如：

``````with tf.variable_scope("scope"):
a = tf.constant(1, name="a")
print(a.name)  # prints "scope/a:0"

b = tf.Variable(1, name="b")
print(b.name)  # prints "scope/b:0"

c = tf.get_variable(name="c", shape=[])
print(c.name)  # prints "scope/c:0"
with tf.variable_scope("scope"):
a1 = tf.get_variable(name="a", shape=[])
a2 = tf.get_variable(name="a", shape=[])  # Disallowed
``````

``````with tf.variable_scope("scope"):
a1 = tf.get_variable(name="a", shape=[])
with tf.variable_scope("scope", reuse=True):
a2 = tf.get_variable(name="a", shape=[])  # OK
This becomes handy for example when using built-in neural network layers:

features1 = tf.layers.conv2d(image1, filters=32, kernel_size=3)
# Use the same convolution weights to process the second image:
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
features2 = tf.layers.conv2d(image2, filters=32, kernel_size=3)
``````

``````conv3x32 = tf.make_template("conv3x32", lambda x: tf.layers.conv2d(x, 32, 3))
features1 = conv3x32(image1)
features2 = conv3x32(image2)  # Will reuse the convolution weights.
``````

## 四、广播的优缺点

TensorFlow 支持广播机制，可以广播逐元素操作。正常情况下，当你想要进行一些操作如加法，乘法时，你需要确保操作数的形状是相匹配的，如：你不能将一个具有形状`[3, 2]`的张量和一个具有`[3,4]`形状的张量相加。但是，这里有一个特殊情况，那就是当你的其中一个操作数是一个某个维度为一的张量的时候，TF 会隐式地填充它的单一维度方向，以确保和另一个操作数的形状相匹配。所以，对一个`[3,2]`的张量和一个`[3,1]`的张量相加在 TF 中是合法的。

``````import TensorFlow as tf

a = tf.constant([[1., 2.], [3., 4.]])
b = tf.constant([[1.], [2.]])
# c = a + tf.tile(b, [1, 2])
c = a + b
``````

``````a = tf.random_uniform([5, 3, 5])
b = tf.random_uniform([5, 1, 6])

# concat a and b and apply nonlinearity
tiled_b = tf.tile(b, [1, 3, 1])
c = tf.concat([a, tiled_b], 2)
d = tf.layers.dense(c, 10, activation=tf.nn.relu)
``````

``````pa = tf.layers.dense(a, 10, activation=None)
pb = tf.layers.dense(b, 10, activation=None)
d = tf.nn.relu(pa + pb)
``````

``````def merge(a, b, units, activation=tf.nn.relu):
pa = tf.layers.dense(a, units, activation=None)
pb = tf.layers.dense(b, units, activation=None)
c = pa + pb
if activation is not None:
c = activation(c)
return c
``````

``````a = tf.constant([[1.], [2.]])
b = tf.constant([1., 2.])
c = tf.reduce_sum(a + b)
``````

``````a = tf.constant([[1.], [2.]])
b = tf.constant([1., 2.])
c = tf.reduce_sum(a + b, 0)
``````

## 五、向 TensorFlow 投喂数据

TensorFlow 被设计可以在大规模的数据情况下高效地运行。所以你需要记住千万不要“饿着”你的 TF 模型，这样才能得到最好的表现。一般来说，一共有三种方法可以“投喂”你的模型。

### 常数方式（`tf.constant`）

``````import TensorFlow as tf
import numpy as np

actual_data = np.random.normal(size=[100])
data = tf.constant(actual_data)
``````

### 占位符方式（`tf.placeholder`）

``````import TensorFlow as tf
import numpy as np

data = tf.placeholder(tf.float32)
prediction = tf.square(data) + 1
actual_data = np.random.normal(size=[100])
tf.Session().run(prediction, feed_dict={data: actual_data})
``````

### python 操作（`tf.py_func`）

``````def py_input_fn():
actual_data = np.random.normal(size=[100])
return actual_data

data = tf.py_func(py_input_fn, [], (tf.float32))
``````

python 操作允许你将一个常规的 python 函数转换成一个 TF 的操作。

### 利用 TF 的自带数据集 API

``````actual_data = np.random.normal(size=[100])
dataset = tf.contrib.data.Dataset.from_tensor_slices(actual_data)
data = dataset.make_one_shot_iterator().get_next()
``````

``````dataset = tf.contrib.data.Dataset.TFRecordDataset(path_to_data)
``````

``````dataset = ...
dataset = dataset.cache()
if mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.repeat()
dataset = dataset.shuffle(batch_size * 5)
dataset = dataset.batch(batch_size)
``````

## 六、利用运算符重载

``````z = x[begin:end]  # z = tf.slice(x, [begin], [end-begin])
``````

``````import TensorFlow as tf
import time

x = tf.random_uniform([500, 10])

z = tf.zeros([10])
for i in range(500):
z += x[i]

sess = tf.Session()
start = time.time()
sess.run(z)
print("Took %f seconds." % (time.time() - start))
``````

``````z = tf.zeros([10])
for x_i in tf.unstack(x):
z += x_i
``````

``````z = tf.reduce_sum(x, axis=0)
``````

TensorFlow 除了切片操作，也重载了一系列的数学逻辑运算，如：

``````z = -x  # z = tf.negative(x)
z = x + y  # z = tf.add(x, y)
z = x - y  # z = tf.subtract(x, y)
z = x * y  # z = tf.mul(x, y)
z = x / y  # z = tf.div(x, y)
z = x // y  # z = tf.floordiv(x, y)
z = x % y  # z = tf.mod(x, y)
z = x ** y  # z = tf.pow(x, y)
z = x @ y  # z = tf.matmul(x, y)
z = x > y  # z = tf.greater(x, y)
z = x >= y  # z = tf.greater_equal(x, y)
z = x < y  # z = tf.less(x, y)
z = x <= y  # z = tf.less_equal(x, y)
z = abs(x)  # z = tf.abs(x)
z = x & y  # z = tf.logical_and(x, y)
z = x | y  # z = tf.logical_or(x, y)
z = x ^ y  # z = tf.logical_xor(x, y)
z = ~x  # z = tf.logical_not(x)
``````

TensorFlow 也不允许把张量当成`boolean`类型使用，因为这个很容易出错：

``````x = tf.constant(1.)
if x:  # 这个将会抛出TypeError错误
...
``````

## 七、理解执行顺序和控制依赖

``````import TensorFlow as tf

a = tf.constant(1)
b = tf.constant(2)
a = a + b

tf.Session().run(a)
``````

``````print(tf.contrib.graph_editor.get_tensors(tf.get_default_graph()))
``````

``````a = tf.Variable(1)
b = tf.constant(2)
assign = tf.assign(a, a + b)

sess = tf.Session()
sess.run(tf.global_variables_initializer())
print(sess.run(assign))
``````

``````a = tf.Variable(1)
b = tf.constant(2)
c = a + b

assign = tf.assign(a, 5)

sess = tf.Session()
for i in range(10):
sess.run(tf.global_variables_initializer())
print(sess.run([assign, c]))
``````

``````a = tf.Variable(1)
b = tf.constant(2)
c = a + b

with tf.control_dependencies([c]):
assign = tf.assign(a, 5)

sess = tf.Session()
for i in range(10):
sess.run(tf.global_variables_initializer())
print(sess.run([assign, c]))
``````

## 八、控制流操作：条件和循环

``````a = tf.constant(1)
b = tf.constant(2)

p = tf.constant(True)

x = tf.cond(p, lambda: a + b, lambda: a * b)

print(tf.Session().run(x))
``````

``````a = tf.constant([1, 1])
b = tf.constant([2, 2])

p = tf.constant([True, False])

x = tf.where(p, a + b, a * b)

print(tf.Session().run(x))
``````

``````n = tf.constant(5)

def cond(i, a, b):
return i < n

def body(i, a, b):
return i + 1, b, a + b

i, a, b = tf.while_loop(cond, body, (2, 1, 1))

print(tf.Session().run(b))
``````

``````n = tf.constant(5)

def cond(i, a, b, c):
return i < n

def body(i, a, b, c):
return i + 1, b, a + b, tf.concat([c, [a + b]], 0)

i, a, b, c = tf.while_loop(cond, body, (2, 1, 1, tf.constant([1, 1])))

print(tf.Session().run(c))
``````

``````i, a, b, c = tf.while_loop(
cond, body, (2, 1, 1, tf.constant([1, 1])),
shape_invariants=(tf.TensorShape([]),
tf.TensorShape([]),
tf.TensorShape([]),
tf.TensorShape([None])))
``````

``````n = tf.constant(5)

c = tf.TensorArray(tf.int32, n)
c = c.write(0, 1)
c = c.write(1, 1)

def cond(i, a, b, c):
return i < n

def body(i, a, b, c):
c = c.write(i, a + b)
return i + 1, b, a + b, c

i, a, b, c = tf.while_loop(cond, body, (2, 1, 1, c))

c = c.stack()

print(tf.Session().run(c))
``````

TensorFlow while 循环和张量数组是构建复杂的循环神经网络的基本工具。 作为练习，尝试使用`tf.while_loops`实现集束搜索（beam search）。 使用张量数组可以使效率更高吗？

## 九、使用 Python 操作设计核心和高级可视化

TensorFlow 中的操作核心完全用 C++ 编写，用于提高效率。 但是用 C++ 编写 TensorFlow 核心可能会非常痛苦。因此，在花费数小时实现核心之前，你可能希望快速创建原型，但效率低下。使用`tf.py_func()`，你可以将任何一段 python 代码转换为 TensorFlow 操作。

``````import numpy as np
import tensorflow as tf
import uuid

def relu(inputs):
# Define the op in python
def _relu(x):
return np.maximum(x, 0.)

# Define the op's gradient in python
return np.float32(x > 0)

x = op.inputs[0]

# Register the gradient with a unique id

# Override the gradient of the custom op
g = tf.get_default_graph()
output = tf.py_func(_relu, [inputs], tf.float32)
return output
``````

``````x = tf.random_normal([10])
y = relu(x * x)

with tf.Session():
diff = tf.test.compute_gradient_error(x, [10], y, [10])
print(diff)
``````

`compute_gradient_error()`以数值方式计算梯度，并返回提供的梯度的差。 我们想要的是非常低的差。

``````image = tf.placeholder(tf.float32)
tf.summary.image("image", image)
``````

``````import io
import matplotlib.pyplot as plt
import numpy as np
import PIL
import tensorflow as tf

def visualize_labeled_images(images, labels, max_outputs=3, name="image"):
def _visualize_image(image, label):
# Do the actual drawing in python
fig = plt.figure(figsize=(3, 3), dpi=80)
ax.imshow(image[::-1,...])
ax.text(0, 0, str(label),
horizontalalignment="left",
verticalalignment="top")
fig.canvas.draw()

# Write the plot as a memory file.
buf = io.BytesIO()
data = fig.savefig(buf, format="png")
buf.seek(0)

# Read the image and convert to numpy array
img = PIL.Image.open(buf)
return np.array(img.getdata()).reshape(img.size[0], img.size[1], -1)

def _visualize_images(images, labels):
# Only display the given number of examples in the batch
outputs = []
for i in range(max_outputs):
output = _visualize_image(images[i], labels[i])
outputs.append(output)
return np.array(outputs, dtype=np.uint8)

# Run the python op.
figs = tf.py_func(_visualize_images, [images, labels], tf.uint8)
return tf.summary.image(name, figs)
``````

## 十、多 GPU 和数据并行

`````` import tensorflow as tf

with tf.device(tf.DeviceSpec(device_type="CPU", device_index=0)):
a = tf.random_uniform([1000, 100])
b = tf.random_uniform([1000, 100])
c = a + b

tf.Session().run(c)
``````

GPU 上可以做相同的事情：

``````with tf.device(tf.DeviceSpec(device_type="GPU", device_index=0)):
a = tf.random_uniform([1000, 100])
b = tf.random_uniform([1000, 100])
c = a + b
``````

``````split_a = tf.split(a, 2)
split_b = tf.split(b, 2)

split_c = []
for i in range(2):
with tf.device(tf.DeviceSpec(device_type="GPU", device_index=i)):
split_c.append(split_a[i] + split_b[i])

c = tf.concat(split_c, axis=0)
``````

``````def make_parallel(fn, num_gpus, **kwargs):
in_splits = {}
for k, v in kwargs.items():
in_splits[k] = tf.split(v, num_gpus)

out_split = []
for i in range(num_gpus):
with tf.device(tf.DeviceSpec(device_type="GPU", device_index=i)):
with tf.variable_scope(tf.get_variable_scope(), reuse=i > 0):
out_split.append(fn(**{k : v[i] for k, v in in_splits.items()}))

return tf.concat(out_split, axis=0)

def model(a, b):
return a + b

c = make_parallel(model, 2, a=a, b=b)
``````

``````import numpy as np
import tensorflow as tf

def model(x, y):
w = tf.get_variable("w", shape=[3, 1])

f = tf.stack([tf.square(x), x, tf.ones_like(x)], 1)
yhat = tf.squeeze(tf.matmul(f, w), 1)

loss = tf.square(yhat - y)
return loss

x = tf.placeholder(tf.float32)
y = tf.placeholder(tf.float32)

loss = model(x, y)

tf.reduce_mean(loss))

def generate_data():
x_val = np.random.uniform(-10.0, 10.0, size=100)
y_val = 5 * np.square(x_val) + 3
return x_val, y_val

sess = tf.Session()
sess.run(tf.global_variables_initializer())
for _ in range(1000):
x_val, y_val = generate_data()
_, loss_val = sess.run([train_op, loss], {x: x_val, y: y_val})

_, loss_val = sess.run([train_op, loss], {x: x_val, y: y_val})
print(sess.run(tf.contrib.framework.get_variables_by_name("w")))
``````

``````loss = make_parallel(model, 2, x=x, y=y)

tf.reduce_mean(loss),
``````

## 十一、调试 TensorFlow 模型

``````a = tf.random_uniform([2, 3])
b = tf.random_uniform([3, 4])
c = tf.matmul(a, b)  # c is a tensor of shape [2, 4]
``````

``````a = tf.random_uniform([10, 2, 3])
b = tf.random_uniform([10, 3, 4])
tf.matmul(a, b)  # c is a tensor of shape [10, 2, 4]
``````

``````a = tf.constant([[1.], [2.]])
b = tf.constant([1., 2.])
c = a + b  # c is a tensor of shape [2, 2]
``````

### 使用`tf.assert*`操作验证你的张量

``````a = tf.constant([[1.], [2.]])
b = tf.constant([1., 2.])
check_a = tf.assert_rank(a, 1)  # This will raise an InvalidArgumentError exception
check_b = tf.assert_rank(b, 1)
with tf.control_dependencies([check_a, check_b]):
c = a + b  # c is a tensor of shape [2, 2]
``````

``````check_pos = tf.assert_positive(a)
``````

### 使用`tf.Print`记录张量的值

``````input_copy = tf.Print(input, tensors_to_print_list)
``````

``````a = ...
b = ...
a = tf.Print(a, [a, b])
c = a + b
``````

### 使用`tf.compute_gradient_error`检查梯度

TensorFlow 中并非所有操作都带有梯度，并且很容易在无意中构建 TensorFlow 无法计算梯度的图形。

``````import tensorflow as tf

def non_differentiable_entropy(logits):
probs = tf.nn.softmax(logits)
return tf.nn.softmax_cross_entropy_with_logits(labels=probs, logits=logits)

w = tf.get_variable("w", shape=[5])
y = -non_differentiable_entropy(w)

train_op = opt.minimize(y)

sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in range(10000):
sess.run(train_op)

print(sess.run(tf.nn.softmax(w)))
``````

``````[ 0.34081486  0.24287023  0.23465775  0.08935683  0.09230034]
``````

``````with tf.Session():
diff = tf.test.compute_gradient_error(w, [5], y, [])
print(diff)
``````

``````import tensorflow as tf
import numpy as np

def entropy(logits, dim=-1):
probs = tf.nn.softmax(logits, dim)
nplogp = probs * (tf.reduce_logsumexp(logits, dim, keep_dims=True) - logits)
return tf.reduce_sum(nplogp, dim)

w = tf.get_variable("w", shape=[5])
y = -entropy(w)

print(w.get_shape())
print(y.get_shape())

with tf.Session() as sess:
diff = tf.test.compute_gradient_error(w, [5], y, [])
print(diff)
``````

``````[ 0.2  0.2  0.2  0.2  0.2]
``````

TensorFlow 摘要tfdbg（TensorFlow 调试器）是可用于调试的其他工具。 请参阅官方文档来了解更多信息。

## 十二、TensorFlow 中的数值稳定性

``````import numpy as np

x = np.float32(1)

y = np.float32(1e-50)  # y would be stored as zero
z = x * y / y

print(z)  # prints nan
``````

``````y = np.float32(1e39)  # y would be stored as inf
z = x * y / y

print(z)  # prints 0
``````

`float32`类型可以表示的最小正值是`1.4013e-45`，低于该值的任何值都将存储为零。 此外，任何超过`3.40282e+38`的数字都将存储为`inf`

``````print(np.nextafter(np.float32(0), np.float32(1)))  # prints 1.4013e-45
print(np.finfo(np.float32).max)  # print 3.40282e+38
``````

``````import tensorflow as tf

def unstable_softmax(logits):
exp = tf.exp(logits)
return exp / tf.reduce_sum(exp)

tf.Session().run(unstable_softmax([1000., 0.]))  # prints [ nan, 0.]
``````

``````import tensorflow as tf

def softmax(logits):
exp = tf.exp(logits - tf.reduce_max(logits))
return exp / tf.reduce_sum(exp)

tf.Session().run(softmax([1000., 0.]))  # prints [ 1., 0.]
``````

``````def unstable_softmax_cross_entropy(labels, logits):
logits = tf.log(softmax(logits))
return -tf.reduce_sum(labels * logits)

labels = tf.constant([0.5, 0.5])
logits = tf.constant([1000., 0.])

xe = unstable_softmax_cross_entropy(labels, logits)

print(tf.Session().run(xe))  # prints inf
``````

``````def softmax_cross_entropy(labels, logits):
scaled_logits = logits - tf.reduce_max(logits)
normalized_logits = scaled_logits - tf.reduce_logsumexp(scaled_logits)
return -tf.reduce_sum(labels * normalized_logits)

labels = tf.constant([0.5, 0.5])
logits = tf.constant([1000., 0.])

xe = softmax_cross_entropy(labels, logits)

print(tf.Session().run(xe))  # prints 500.0
``````

``````g = tf.gradients(xe, logits)
print(tf.Session().run(g))  # prints [0.5, -0.5]
``````

## 十三、使用学习 API 构建神经网络训练框架

``````import tensorflow as tf

def model_fn(features, labels, mode, params):
predictions = ...
loss = ...
train_op = ...
metric_ops = ...
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=metric_ops)

params = ...
run_config = tf.contrib.learn.RunConfig(model_dir=FLAGS.output_dir)
estimator = tf.estimator.Estimator(
model_fn=model_fn, config=run_config, params=params)
``````

``````def input_fn():
features = ...
labels = ...
return features, labels

estimator.train(input_fn=input_fn, max_steps=...)
``````

``````estimator.evaluate(input_fn=input_fn)
``````

``````experiment = tf.contrib.learn.Experiment(
estimator=estimator,
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
eval_metrics=eval_metrics)
``````

``````experiment.train_and_evaluate()
``````

``````import tensorflow as tf

tf.flags.DEFINE_string("output_dir", "", "Optional output dir.")
tf.flags.DEFINE_string("schedule", "train_and_evaluate", "Schedule.")
tf.flags.DEFINE_string("hparams", "", "Hyper parameters.")

FLAGS = tf.flags.FLAGS

def experiment_fn(run_config, hparams):
estimator = tf.estimator.Estimator(
model_fn=make_model_fn(), config=run_config, params=hparams)
return tf.contrib.learn.Experiment(
estimator=estimator,
train_input_fn=make_input_fn(tf.estimator.ModeKeys.TRAIN, hparams),
eval_input_fn=make_input_fn(tf.estimator.ModeKeys.EVAL, hparams),
eval_metrics=eval_metrics_fn(hparams))

def main(unused_argv):
run_config = tf.contrib.learn.RunConfig(model_dir=FLAGS.output_dir)
hparams = tf.contrib.training.HParams()
hparams.parse(FLAGS.hparams)

estimator = tf.contrib.learn.learn_runner.run(
experiment_fn=experiment_fn,
run_config=run_config,
schedule=FLAGS.schedule,
hparams=hparams)

if __name__ == "__main__":
tf.app.run()
``````

`schedule`标志决定调用`Experiment`对象的哪个成员函数。 因此，如果你将`schedule`设置为`train_and_evaluate`，则会调用`experiment.train_and_evaluate()`

``````def input_fn():
features = ...
labels = ...
return features, labels
``````

## 十四、TensorFlow 秘籍

### 集束搜索

``````import tensorflow as tf

def get_shape(tensor):
"""Returns static shape if available and dynamic shape otherwise."""
static_shape = tensor.shape.as_list()
dynamic_shape = tf.unstack(tf.shape(tensor))
dims = [s[1] if s[0] is None else s[0]
for s in zip(static_shape, dynamic_shape)]
return dims

def log_prob_from_logits(logits, axis=-1):
"""Normalize the log-probabilities so that probabilities sum to one."""
return logits - tf.reduce_logsumexp(logits, axis=axis, keep_dims=True)

def batch_gather(tensor, indices):
"""Gather in batch from a tensor of arbitrary size.

In pseudocode this module will produce the following:
output[i] = tf.gather(tensor[i], indices[i])

Args:
tensor: Tensor of arbitrary size.
indices: Vector of indices.
Returns:
output: A tensor of gathered values.
"""
shape = get_shape(tensor)
flat_first = tf.reshape(tensor, [shape[0] * shape[1]] + shape[2:])
indices = tf.convert_to_tensor(indices)
offset_shape = [shape[0]] + [1] * (indices.shape.ndims - 1)
offset = tf.reshape(tf.range(shape[0]) * shape[1], offset_shape)
output = tf.gather(flat_first, indices + offset)
return output

def rnn_beam_search(update_fn, initial_state, sequence_length, beam_width,
begin_token_id, end_token_id, name="rnn"):
"""Beam-search decoder for recurrent models.

Args:
update_fn: Function to compute the next state and logits given the current
state and ids.
initial_state: Recurrent model states.
sequence_length: Length of the generated sequence.
beam_width: Beam width.
begin_token_id: Begin token id.
end_token_id: End token id.
name: Scope of the variables.
Returns:
ids: Output indices.
logprobs: Output log probabilities probabilities.
"""
batch_size = initial_state.shape.as_list()[0]

state = tf.tile(tf.expand_dims(initial_state, axis=1), [1, beam_width, 1])

sel_sum_logprobs = tf.log([[1.] + [0.] * (beam_width - 1)])

ids = tf.tile([[begin_token_id]], [batch_size, beam_width])
sel_ids = tf.expand_dims(ids, axis=2)

for i in range(sequence_length):
with tf.variable_scope(name, reuse=True if i > 0 else None):

state, logits = update_fn(state, ids)
logits = log_prob_from_logits(logits)

sum_logprobs = (
tf.expand_dims(sel_sum_logprobs, axis=2) +

num_classes = logits.shape.as_list()[-1]

sel_sum_logprobs, indices = tf.nn.top_k(
tf.reshape(sum_logprobs, [batch_size, num_classes * beam_width]),
k=beam_width)

ids = indices % num_classes

beam_ids = indices // num_classes

state = batch_gather(state, beam_ids)

sel_ids = tf.concat([batch_gather(sel_ids, beam_ids),
tf.expand_dims(ids, axis=2)], axis=2)

tf.to_float(tf.not_equal(ids, end_token_id)))

return sel_ids, sel_sum_logprobs
``````

### 合并

``````import tensorflow as tf

def merge(tensors, units, activation=tf.nn.relu, name=None, **kwargs):

This operation concatenates multiple features of varying length and applies
non-linear transformation to the outcome.

Example:
a = tf.zeros([m, 1, d1])
b = tf.zeros([1, n, d2])
c = merge([a, b], d3)  # shape of c would be [m, n, d3].

Args:
tensors: A list of tensor with the same rank.
units: Number of units in the projection function.
"""
with tf.variable_scope(name, default_name="merge"):
# Apply linear projection to input tensors.
projs = []
for i, tensor in enumerate(tensors):
proj = tf.layers.dense(
tensor, units, activation=None,
name="proj_%d" % i,
**kwargs)
projs.append(proj)

# Compute sum of tensors.
result = projs.pop()
for proj in projs:
result = result + proj

# Apply nonlinearity.
if activation:
result = activation(result)
return result
``````

### 熵

``````import tensorflow as tf

def softmax(logits, dims=-1):
"""Compute softmax over specified dimensions."""
exp = tf.exp(logits - tf.reduce_max(logits, dims, keep_dims=True))
return exp / tf.reduce_sum(exp, dims, keep_dims=True)

def entropy(logits, dims=-1):
"""Compute entropy over specified dimensions."""
probs = softmax(logits, dims)
nplogp = probs * (tf.reduce_logsumexp(logits, dims, keep_dims=True) - logits)
return tf.reduce_sum(nplogp, dims)
``````

### KL 散度

``````def gaussian_kl(q, p=(0., 0.)):
"""Computes KL divergence between two isotropic Gaussian distributions.

To ensure numerical stability, this op uses mu, log(sigma^2) to represent
the distribution. If q is not provided, it's assumed to be unit Gaussian.

Args:
q: A tuple (mu, log(sigma^2)) representing a multi-variatie Gaussian.
p: A tuple (mu, log(sigma^2)) representing a multi-variatie Gaussian.
Returns:
A tensor representing KL(q, p).
"""
mu1, log_sigma1_sq = q
mu2, log_sigma2_sq = p
return tf.reduce_sum(
0.5 * (log_sigma2_sq - log_sigma1_sq +
tf.exp(log_sigma1_sq - log_sigma2_sq) +
tf.square(mu1 - mu2) / tf.exp(log_sigma2_sq) -
1), axis=-1)
``````

### 并行化

``````def make_parallel(fn, num_gpus, **kwargs):
"""Parallelize given model on multiple gpu devices.

Args:
fn: Arbitrary function that takes a set of input tensors and outputs a
single tensor. First dimension of inputs and output tensor are assumed
to be batch dimension.
num_gpus: Number of GPU devices.
**kwargs: Keyword arguments to be passed to the model.
Returns:
A tensor corresponding to the model output.
"""
in_splits = {}
for k, v in kwargs.items():
in_splits[k] = tf.split(v, num_gpus)

out_split = []
for i in range(num_gpus):
with tf.device(tf.DeviceSpec(device_type="GPU", device_index=i)):
with tf.variable_scope(tf.get_variable_scope(), reuse=i > 0):
out_split.append(fn(**{k : v[i] for k, v in in_splits.items()}))

return tf.concat(out_split, axis=0)
``````