Tensorflow基本指南

使用Anaconda安装，运行更快，参见
会话注册sess=tf.InteractivateSession()
表示将是这个创建的session作为随后默认的session，之后的运行也运算也在这里面进行
列出当前每个节点使用的device，使用的是tf.Session(config=tf.ConfigProto(log_device_placement=True))

Tensorflow如何处理训练模式和测试模式

DropOut层和BN层都有训练模式、测试模式。
DropOut层：设立一个placeholder，设置训练时一个数，测试时设置为0
phase = tf.placeholder(tf.bool, name='phase')
BN层：未知

模型的导入导出

载入模型

trainable = tf.trainable_variables()
optim = optimizer.minimize(loss, global_step=global_step, var_list=trainable)
saver = tf.train.Saver(trainable)
saver.restore(sess, './logdir/model.ckpt')

载入多模型交替循环不同部分

trainable = tf.trainable_variables()
trainable_DNN = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,"DNN_part")
trainable_WaveNet = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,"wavenet_part")

optim = optimizer.minimize(loss, global_step=global_step, var_list=trainable)
optim_DNN = optimizer.minimize(loss, global_step=global_step, var_list=trainable_DNN)
optim_WaveNet = optimizer.minimize(loss, global_step=global_step, var_list=trainable_WaveNet)

saver = tf.train.Saver(trainable)
saver.restore(sess, './logdir/model.ckpt')
DNN_saver = tf.train.Saver(trainable_DNN)
DNN_saver .restore(sess, './logdir/DNN_part/Unit2Vec.ckpt')
WaveNet_saver = tf.train.Saver(trainable_WaveNet)
WaveNet_saver.restore(sess, './logdir/WaveNet_part/Unit2Vec.ckpt')

#part 1->3->2->3->2->3...
if step < 10: #turn round every 10 steps  #part 1
    summary, loss_value, acc_value, _ = sess.run([summaries, loss, acc, optim])
elif int(step/10) % 2 == 0:               #part 2
    summary, loss_value, acc_value, _ = sess.run([summaries, loss, acc, optim_WaveNet])
elif int(step/10) % 2 == 1:               #part 3
    summary, loss_value, acc_value, _ = sess.run([summaries, loss, acc, optim_DNN])
else:
    raise ValueError(x)

数据操作

Boardcasting机制

可以看到b的维度竟然和a的最后一维一致，使用了boardcasting

import tensorflow as tf
a=tf.constant(1,shape=[2,3,5])
b=tf.constant(range(5))
with tf.Session() as sess:
    tf.global_variables_initializer().run()
    print sess.run(a+b)

[[[1 2 3 4 5]
[1 2 3 4 5]
[1 2 3 4 5]]
[[1 2 3 4 5]
[1 2 3 4 5]
[1 2 3 4 5]]]

tf.one_hot()

tf.one_hot([[0,1],[2,3]],depth=5)得到的数据维度是(2,2,5)
tf.one_hot([[[0,1],[2,3]],[[0,1],[2,3]]],depth=5)得到的数据维度是(2,2,2,5)

tf.unique和tf.unique_with_counts函数

类似于mma的{1, 1, 1, 1, 5, 5, 3, 3, 3, 2, 2, 2, 2, 4} // Counts功能

import tensorflow as tf
import numpy as np

Y1 = tf.constant([5,5,1,2,2,4,4,4,3,3])
Y2 = tf.unique(Y1)
Y3 = tf.unique_with_counts(Y1) #Can Count value

with tf.Session() as sess:
    [Y2,Y3] = sess.run([Y2,Y3])
    print Y2
    # Unique(y=array([5, 1, 2, 4, 3], dtype=int32),idx=array([0, 0, 1, 2, 2, 3, 3, 3, 4, 4], dtype=int32))
    print Y3
    # UniqueWithCounts(y=array([5, 1, 2, 4, 3], dtype=int32), idx=array([0, 0, 1, 2, 2, 3, 3, 3, 4, 4], dtype=int32), count=array([2, 1, 2, 3, 2], dtype=int32))

实现左移一个数，最右补零

将[[0],[1],[2],[3]]->[[1],[2],[3],[0]]

import tensorflow as tf

encoded = tf.reshape(tf.constant(range(4)),[1,-1,1])
shifted = tf.slice(encoded, [0, 1, 0],[-1, tf.shape(encoded)[1] - 1, -1])
shifted = tf.pad(shifted, [[0, 0], [0, 1], [0, 0]])

with tf.Session() as sess:
    print sess.run([encoded,shifted])

各种loss函数

tf.losses.mean_squared_error理解

import tensorflow as tf
import numpy as np

shape_obj = (5,6,3)
Y1 = tf.random_normal(shape=shape_obj)
Y2 = tf.random_normal(shape=shape_obj)

loss1 = tf.reduce_sum(tf.pow(Y1 - Y2, 2)) / (reduce(lambda x, y: x*y, shape_obj))
loss2 = tf.reduce_mean(tf.squared_difference(Y1, Y2))
loss3 = tf.losses.mean_squared_error(predictions=Y1, labels=Y2)
loss4 = tf.nn.l2_loss(Y1 - Y2)

with tf.Session() as sess:
    #不需要初始化
    lis = sess.run([Y1, Y2, loss1, loss2, loss3, loss4])
    Y1, Y2 = lis[:2] #Y1, Y2
    print lis[2:]  #loss1, loss2, loss3, loss4
    lis=[]
    for i in range(shape_obj[-1]):
        lis.append(np.mean(np.square((Y1[:,:,i]-Y2[:,:,i]))))
    print np.mean(lis)

[2.0185342, 2.0185342, 2.0185342, 90.83404]
2.018534

可以看到tf把最后一维（shape_obj 是二维也是一样）当做batch_axis，而且loss没有除以2
在Gluon里就相当于：loss1=gluon.loss.L2Loss(batch_axis=-1)*2
还发现一个问题就是sess.run(想要的必须写在一起)，如果sess.run(loss1);sess.run(loss1)结果也会不一样，因为每运行一次run计算图就运行一次，因为每次运行的随机数不一样，结果自然也不一样

import mxnet as mx
from mxnet import gluon
import numpy as np
#Indicate 2-th dims as batch_axis
loss1=gluon.loss.L2Loss(batch_axis=1)
a=mx.nd.random.uniform(0, 10,shape=(3,5,4))
b=mx.nd.random.uniform(0, 10,shape=(3,5,4))

print loss1(a,b)
#[  4.95132017  11.42089748   7.28211212   6.8867259    6.93179274]
#<NDArray 5 @cpu(0)>
lis=[]
for i in range(5):
    lis.append(1./2*np.mean(np.square((a[:,i,:]-b[:,i,:]).asnumpy())))
print lis
#[4.9513201713562012, 11.420897483825684, 7.282111644744873, 6.886725902557373, 6.9317927360534668]

tf.losses.sparse_softmax_cross_entropy理解

sparse_softmax_cross_entropy要求labels是一个整数列表形式，范围是[0, num_classes], logits是一个浮点数据，存储的是网络输出的值

MXNet和tf结果一样，说明在二维数据的时候，都是将第一维作为batch_axis,类别总数可以从Y1.shape[1]知道

import tensorflow as tf
import numpy as np

Y1 = tf.constant(np.arange(50).reshape((5,10)),dtype=tf.float32)
Y2 = tf.constant([5,3,2,9,1])
Y3 = tf.one_hot(Y2, depth = Y1.shape[1])

loss1 = tf.losses.sparse_softmax_cross_entropy(logits=Y1, labels=Y2)
loss2 = tf.losses.softmax_cross_entropy(logits=Y1, onehot_labels=Y3)
loss3 = tf.nn.softmax_cross_entropy_with_logits(logits=Y1, labels=Y3)
loss4 = tf.reduce_mean(loss3)
with tf.Session() as sess:
    [loss1,loss2,loss3,loss4] = sess.run([loss1,loss2,loss3,loss4])
    print loss1,loss2    #5.4586296, 5.4586296
    print loss3.shape    #(5,)
    print np.mean(loss3) #5.4586296
    print loss4          #5.4586296
'''

用MXNet来验证

import mxnet as mx
import numpy as np
loss=mx.gluon.loss.SoftmaxCrossEntropyLoss()
a=mx.nd.array(np.arange(50).reshape((5,10)))
b=mx.nd.array([5,3,2,9,1])
print loss(a,b)
print mx.nd.mean(loss(a,b))#5.45862961

图像处理

完整范例

import tensorflow as tf
import numpy as np
from PIL import Image

img = Image.open("1620.jpg")
img.show()
w, h = img.size
img = np.expand_dims(img, 0)
bi_image_bilinear = tf.image.resize_bilinear(img, size=(int(h*3), int(w*3)))

bi_image_bilinear = tf.squeeze(bi_image_bilinear)
with tf.Session() as sess:
    bi_result = sess.run(bi_image_bilinear)
Image.fromarray(np.uint8(bi_result)).show()

tf.image.resize*

tf.image.resize_nearest_neighbor [0 1 2] -> [0 0 1 1 2 2]
tf.image.resize_bilinear [0 1 2] -> [0 0.5 1 1.5 2 2 ]
tf.image.resize_bicubic [0 0.40625 1 1.59375 2 2.09375]

import tensorflow as tf
import numpy as np

img = np.arange(3).reshape((1,1,3,1))
bi_image_bilinear = tf.image.resize_bilinear(img, size=(1,6))

bi_image_bilinear = tf.squeeze(bi_image_bilinear)
with tf.Session() as sess:
    print sess.run(bi_image_bilinear)

高阶函数

tf.map_fn 类似于map函数，一个迭代器，将函数作用于指定变量上
这是例子1：
因为elems是(a,b)，lambda的变量必须与其保持一致，所以也是tuple的。因为a,b分别是int32和float32，返回值也是int32和float32的tuple，与elms一致，所以dtype可以省略。

import tensorflow as tf
import numpy as np

def mean(lis,num):
    return (tf.reduce_mean(lis[:num]), num)

a = tf.constant(np.arange(30).reshape((3,10)), dtype='float32')
b = tf.constant([3,6,2],dtype='int32')
c = tf.map_fn(fn=lambda (i,j):mean(i,j), elems=(a,b))
with tf.Session() as sess:
    print sess.run(c)

例子2：此时返回值与elms结构不一样，需要明确指明返回值类型

import tensorflow as tf
import numpy as np

def mean(lis,num):
    return tf.reduce_mean(lis[:num])

a = tf.constant(np.arange(30).reshape((3,10)), dtype='float32')
b = tf.constant([3,6,2],dtype='int32')
c = tf.map_fn(fn=lambda (i,j):mean(i,j), elems=(a,b), dtype='float32')
with tf.Session() as sess:
    print sess.run(c)

网络

tf.conv1D

输入的shape是（batch_size, width1, in_channel）
卷积核的shape是（filter_width, in_channel, out_channel）
输出是（batch_size, width2, out_channel）

import tensorflow as tf
import numpy as np

x = tf.Variable(np.arange(320).reshape(8,10,4).astype(np.float32))
k = tf.Variable(tf.contrib.layers.xavier_initializer_conv2d()(shape=[5,4,16]))
conv = tf.nn.conv1d(x, k, stride=1, padding='VALID', data_format='NWC')

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    [a, b] = sess.run([x, conv])
    print a.shape, b.shape
    #(8, 10, 4) -> (8, 6, 16)

队列机制

入队的方式有两种，enqueue和enqueue_many。一般采取第一种
出队的方式有两种，dequeue和dequeue_many。一般采取第二种，这相当于dequeue出来的数据是一个batch

RandomShuffleQueue

创建一个随机队列，表示出队的是随机的，但是当使用RandomShuffleQueue时如果shape不明确指定具体大小，dequeue_many是禁用的。

enqueue

这里首先创建了一个10个箱子（每个箱子可以放两个物品，第一种物品是5*2的矩阵，第二种是5*3的矩阵）的队列，要求min_after_dequeue为7个箱子，也就是最多只可以deque出三个箱子。循环中入队了十次，使得队列为满箱状态，所以dequeue_many(3)刚好不阻塞。

import tensorflow as tf
import numpy as np

input_queue = tf.RandomShuffleQueue(capacity=10, min_after_dequeue=7, 
                                dtypes=[tf.int32, tf.int32], shapes=[(5,2),(5,3)])

placeholder1 = tf.placeholder(dtype=tf.int32, shape=[5,2])
placeholder2 = tf.placeholder(dtype=tf.int32, shape=[5,3])

enqueue_op = input_queue.enqueue([placeholder1, placeholder2])
dequeue = input_queue.dequeue_many(3)
with tf.Session() as sess:
    #filled the queue
    for i in range(10):
        data1 = np.arange(i,i+10).reshape(-1,2)
        data2 = np.arange(i,i+15).reshape(-1,3)
        sess.run(enqueue_op,feed_dict={placeholder1:data1, 
                                       placeholder2:data2})
    print sess.run(dequeue)

[array([[[ 3, 4], [ 5, 6],[ 7, 8], [ 9, 10], [11, 12]],
[[ 9, 10],[11, 12],[13, 14], [15, 16], [17, 18]],
[[ 5, 6],[ 7, 8], [ 9, 10], [11, 12], [13, 14]]], dtype=int32),
array([[[ 3, 4, 5],[ 6, 7, 8],[ 9, 10, 11],[12, 13, 14],[15, 16, 17]],
[[ 9, 10, 11],[12, 13, 14],[15, 16, 17],[18, 19, 20],[21, 22, 23]],
[[ 5, 6, 7],[ 8, 9, 10],[11, 12, 13],[14, 15, 16],[17, 18, 19]]], dtype=int32)]

可以看到返回两种箱子，一般我们会将这个返回值赋值给tuples,比如把代码最后一句改为(col2,col3) = sess.run(dequeue)。这样col2和col3就可以作为数据供给网络使用

enqueue_many

这里首先创建了一个10个箱子（每个箱子放两种物品，第一个物品是一个长度为2的数组，第二个物品是长度为3的数组）的随机队列。使用enqueue_many相当于一次性填充多个箱子。现在一次性填充10个箱子让队列满箱，因为min_after_dequeue是7所以最多只能弹出3个箱子,否则阻塞。

import tensorflow as tf
import numpy as np

input_queue = tf.RandomShuffleQueue(capacity=10, min_after_dequeue=7, 
                          dtypes=[tf.int32, tf.int32], shapes=[(2,),(3,)])

placeholder1 = tf.placeholder(dtype=tf.int32, shape=[10,2])
placeholder2 = tf.placeholder(dtype=tf.int32, shape=[10,3])

enqueue_op = input_queue.enqueue_many([placeholder1, placeholder2])
dequeue = input_queue.dequeue_many(3)
with tf.Session() as sess:
    #filled the queue
    data1 = np.arange(20).reshape(10,2)
    data2 = np.arange(30).reshape(10,3)
    sess.run(enqueue_op,feed_dict={placeholder1:data1, 
                                   placeholder2:data2})
    print sess.run(dequeue)

PaddingFIFOQueue

支持变化的shape，同时支持dequeue_many!

import tensorflow as tf
import numpy as np

input_queue = tf.PaddingFIFOQueue(capacity=10,
                                  dtypes=[tf.int32, tf.int32], shapes=[(None,2),(None,3)])

placeholder1 = tf.placeholder(dtype=tf.int32, shape=[None,2])
placeholder2 = tf.placeholder(dtype=tf.int32, shape=[None,3])

enqueue_op = input_queue.enqueue([placeholder1, placeholder2])
dequeue = input_queue.dequeue_many(3)
with tf.Session() as sess:
    #filled the queue
    for i in range(10):
        data1 = np.arange(2*(i+1)).reshape(-1,2)+i
        data2 = np.arange(3*(i+1)).reshape(-1,3)+i
        sess.run(enqueue_op,feed_dict={placeholder1:data1, 
                                       placeholder2:data2})
    print sess.run(dequeue)

[array([[[0, 1],[0, 0],[0, 0]],
[[1, 2],[3, 4],[0, 0]],
[[2, 3],[4, 5],[6, 7]]], dtype=int32),
array([[[ 0, 1, 2],[ 0, 0, 0],[ 0, 0, 0]],
[[ 1, 2, 3],[ 4, 5, 6],[ 0, 0, 0]],
[[ 2, 3, 4],[ 5, 6, 7],[ 8, 9, 10]]], dtype=int32)]

发现了什么吧，就是它补零了。小小说些题外话，你见过np.array([[1],[1,2],[1,2,3]])的返回值吗？没吧，是Object而不是int64。因为numpy不支持，但是python本身支持[[1],[1,2],[1,2,3]]。这就不如Mathematica了,表扬一下顺便推荐这个语言！

如果改成dequeue = input_queue.dequeue_many(1)，一次只出一个箱子那么就不会出现补零情况了。所以补零发生在sess.run(dequeue)的时候

甚至它还支持shape完全不定的情况

import tensorflow as tf
import numpy as np

input_queue = tf.PaddingFIFOQueue(capacity=10,dtypes=[tf.int32],
                                  shapes=[(None,None)])

placeholder = tf.placeholder(dtype=tf.int32, shape=[None,None])

enqueue_op = input_queue.enqueue([placeholder])
dequeue = input_queue.dequeue_many(3)
with tf.Session() as sess:
    #filled the queue
    for i in range(10):
        data = np.arange((i+1)**2).reshape(i+1,i+1)+i
        sess.run(enqueue_op,feed_dict={placeholder:data})
    print sess.run(dequeue)

[[[ 0 0 0] [ 0 0 0] [ 0 0 0]]
[[ 1 2 0] [ 3 4 0] [ 0 0 0]]
[[ 2 3 4] [ 5 6 7] [ 8 9 10]]]

Tensorflow与其他框架联动

Keras与Tensorflow联动

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist_data = input_data.read_data_sets('MNIST_data/', one_hot=True)

# Building model
img = tf.placeholder(tf.float32, shape=(None, 784))
labels = tf.placeholder(tf.float32, shape=(None, 10))

x = tf.keras.layers.Dense(128, activation='relu')(img)
x = tf.keras.layers.Dense(128, activation='relu')(x)
prediction = tf.keras.layers.Dense(10)(x)

loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels))
train_optim = tf.train.AdamOptimizer().minimize(loss)

#Print model name and shape
for i in tf.trainable_variables():
    print '{}\t{}\t{}'.format(i.name,i.shape,i.dtype)
#dense/kernel:0 (784, 128)  <dtype: 'float32_ref'>
#dense/bias:0   (128,)  <dtype: 'float32_ref'>
#dense_1/kernel:0   (128, 128)  <dtype: 'float32_ref'>
#dense_1/bias:0 (128,)  <dtype: 'float32_ref'>
#dense_2/kernel:0   (128, 10)   <dtype: 'float32_ref'>
#dense_2/bias:0 (10,)   <dtype: 'float32_ref'>

# Train model
with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)
    for _ in range(1000):
        batch_x, batch_y = mnist_data.train.next_batch(50)
        sess.run(train_optim, feed_dict={img: batch_x, labels: batch_y})
        acc_pred = tf.keras.metrics.categorical_accuracy(labels, prediction)
        pred = sess.run(acc_pred, feed_dict={labels: mnist_data.test.labels, img: mnist_data.test.images})
        print('accuracy: %.3f' % (sum(pred)/len(mnist_data.test.labels)))