From 86cb1dc8bbecd67d609bbb4882710c50e5b1da36 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 23 Aug 2016 22:54:51 -0500 Subject: [PATCH 1/6] adds transpose_conv to layers --- aiutils/tftools/layers.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index 9a47319..3c2eeaa 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -100,6 +100,41 @@ def conv2d(input, tf.add_to_collection('to_regularize', w) return output +def conv2d_transpose(input, + filter_size, + out_dim, + name, + strides=[1, 1, 1, 1], + gain=np.sqrt(2), + func=tf.nn.relu, + reuse_vars=False): + padding = 'SAME' + in_shape = input.get_shape().as_list() + in_dim = in_shape[-1] + out_shape = [x*y for x,y in zip(in_shape, strides)] + out_shape[-1] = out_dim + + stddev = 1.0 * gain / np.sqrt(filter_size * filter_size * in_dim) + with tf.variable_scope(name, reuse=reuse_vars): + w_init = tf.random_normal_initializer(stddev=stddev) + b_init = tf.constant_initializer() + w = tf.get_variable('w', + shape=[filter_size, filter_size, out_dim, in_dim], + initializer=w_init) + b = tf.get_variable('b', shape=[out_dim], initializer=b_init) + + print "in_shape" + print in_shape + print "out_shape" + print out_shape + + output = tf.nn.conv2d_transpose(input, w, out_shape, strides=strides, padding=padding) + b + if func is not None: + output = func(output) + + tf.add_to_collection('to_regularize', w) + return output + def atrous_conv2d(input, filter_size, From 2301380aabb1114a4fec7c6b32eafa7b2d2fa3de Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 31 Aug 2016 21:10:40 -0500 Subject: [PATCH 2/6] remove print statements in conv2d_transpose --- aiutils/tftools/layers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index 3c2eeaa..c625c55 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -123,11 +123,6 @@ def conv2d_transpose(input, initializer=w_init) b = tf.get_variable('b', shape=[out_dim], initializer=b_init) - print "in_shape" - print in_shape - print "out_shape" - print out_shape - output = tf.nn.conv2d_transpose(input, w, out_shape, strides=strides, padding=padding) + b if func is not None: output = func(output) From b2f10d70d9e054a50ed7c28f2267279775a64fc1 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 26 Sep 2016 12:06:48 -0500 Subject: [PATCH 3/6] add padding param to conv2d_transpose --- aiutils/tftools/layers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index c625c55..125d232 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -105,14 +105,16 @@ def conv2d_transpose(input, out_dim, name, strides=[1, 1, 1, 1], + padding='SAME', gain=np.sqrt(2), func=tf.nn.relu, reuse_vars=False): - padding = 'SAME' in_shape = input.get_shape().as_list() in_dim = in_shape[-1] - out_shape = [x*y for x,y in zip(in_shape, strides)] - out_shape[-1] = out_dim + if padding=='SAME': + out_shape = [in_shape[0]] + [x*y for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] + if padding=='VALID': + out_shape = [in_shape[0]] + [(x - 1) * y + filter_size for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] stddev = 1.0 * gain / np.sqrt(filter_size * filter_size * in_dim) with tf.variable_scope(name, reuse=reuse_vars): From 21861c9f3d87fc052d859f38de9fe0f81e149832 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 10 Oct 2016 22:49:07 -0500 Subject: [PATCH 4/6] add out_shape praam to layers.conv2d_transpose --- aiutils/tftools/layers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index 125d232..e3f0b06 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -108,13 +108,16 @@ def conv2d_transpose(input, padding='SAME', gain=np.sqrt(2), func=tf.nn.relu, - reuse_vars=False): + reuse_vars=False, + out_shape=None): in_shape = input.get_shape().as_list() in_dim = in_shape[-1] - if padding=='SAME': - out_shape = [in_shape[0]] + [x*y for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] - if padding=='VALID': - out_shape = [in_shape[0]] + [(x - 1) * y + filter_size for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] + + if out_shape == None: + if padding=='SAME': + out_shape = [in_shape[0]] + [x*y for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] + if padding=='VALID': + out_shape = [in_shape[0]] + [(x - 1) * y + filter_size for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] stddev = 1.0 * gain / np.sqrt(filter_size * filter_size * in_dim) with tf.variable_scope(name, reuse=reuse_vars): From ff503a47513097a31334a5715fb90ad4160924e1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 14 Feb 2017 23:29:19 -0600 Subject: [PATCH 5/6] adds testing --- tests/test_tftools.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_tftools.py b/tests/test_tftools.py index 427ec65..a4200ef 100644 --- a/tests/test_tftools.py +++ b/tests/test_tftools.py @@ -55,6 +55,22 @@ def test_conv2d(): sess.close() tf.reset_default_graph() +def test_conv2d_transpose(): + batch = 1 + height = 16 + width = 16 + filter_size = 3 + in_dim = 4 + out_dim = 5 + + input_shape = [batch, height, width, in_dim] + output_shape = [batch, height/2, width/2, out_dim] + + x = tf.placeholder(tf.float32, input_shape) + y = layers.conv2d(x, filter_size, out_dim, 'conv2d', strides=[1,2,2,1]) + y2 = layers.conv2d_transpose(y, filter_size, in_dim, 'conv2d_tr', strides=[1,2,2,1]) + + assert(np.all(y2.get_shape().as_list() == input_shape)) def test_atrous_conv2d(): From 5083a113a0aed81be151564901f5ec7553e81d57 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 14 Feb 2017 23:45:41 -0600 Subject: [PATCH 6/6] cleanup with yapf, add a bit more testing --- aiutils/tftools/layers.py | 62 ++++++++++++++------------ tests/test_tftools.py | 94 ++++++++++++++++++++++++++------------- 2 files changed, 97 insertions(+), 59 deletions(-) diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index e3f0b06..809b499 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -100,24 +100,29 @@ def conv2d(input, tf.add_to_collection('to_regularize', w) return output + def conv2d_transpose(input, - filter_size, - out_dim, - name, - strides=[1, 1, 1, 1], - padding='SAME', - gain=np.sqrt(2), - func=tf.nn.relu, - reuse_vars=False, - out_shape=None): + filter_size, + out_dim, + name, + strides=[1, 1, 1, 1], + padding='SAME', + gain=np.sqrt(2), + func=tf.nn.relu, + reuse_vars=False, + out_shape=None): in_shape = input.get_shape().as_list() in_dim = in_shape[-1] if out_shape == None: - if padding=='SAME': - out_shape = [in_shape[0]] + [x*y for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] - if padding=='VALID': - out_shape = [in_shape[0]] + [(x - 1) * y + filter_size for x,y in zip(in_shape[1:-1], strides[1:-1])] + [out_dim] + out_shape = in_shape + out_shape[3] = out_dim + if padding == 'SAME': + out_shape[1] = in_shape[1] * strides[1] + out_shape[2] = in_shape[2] * strides[2] + if padding == 'VALID': + out_shape[1] = (in_shape[1] - 1) * strides[1] + filter_size + out_shape[2] = (in_shape[2] - 1) * strides[2] + filter_size stddev = 1.0 * gain / np.sqrt(filter_size * filter_size * in_dim) with tf.variable_scope(name, reuse=reuse_vars): @@ -128,7 +133,10 @@ def conv2d_transpose(input, initializer=w_init) b = tf.get_variable('b', shape=[out_dim], initializer=b_init) - output = tf.nn.conv2d_transpose(input, w, out_shape, strides=strides, padding=padding) + b + output = tf.nn.conv2d_transpose( + input, w, out_shape, + strides=strides, + padding=padding) + b if func is not None: output = func(output) @@ -193,7 +201,6 @@ def atrous_conv2d(input, return output - def batch_norm(input, training=tf.constant(True), decay=0.95, @@ -225,18 +232,17 @@ def batch_norm(input, else: raise ValueError('Input tensor must have rank 2 or 4.') - output = tf.contrib.layers.batch_norm( - input, - decay=decay, - is_training=training, - scale=True, - epsilon=epsilon, - updates_collections=None, - scope=name, - reuse=reuse_vars) + output = tf.contrib.layers.batch_norm(input, + decay=decay, + is_training=training, + scale=True, + epsilon=epsilon, + updates_collections=None, + scope=name, + reuse=reuse_vars) if rank == 2: - return tf.squeeze(output, [1,2]) + return tf.squeeze(output, [1, 2]) return output @@ -271,8 +277,8 @@ def dropout(input, training=True, keep_prob=.8, noise_shape=None, seed=None): return tf.cond( training, lambda: tf.nn.dropout( - input, - keep_prob, - noise_shape=noise_shape, + input, + keep_prob, + noise_shape=noise_shape, seed=seed), lambda: input) diff --git a/tests/test_tftools.py b/tests/test_tftools.py index a4200ef..be89bdd 100644 --- a/tests/test_tftools.py +++ b/tests/test_tftools.py @@ -55,6 +55,7 @@ def test_conv2d(): sess.close() tf.reset_default_graph() + def test_conv2d_transpose(): batch = 1 height = 16 @@ -64,13 +65,43 @@ def test_conv2d_transpose(): out_dim = 5 input_shape = [batch, height, width, in_dim] - output_shape = [batch, height/2, width/2, out_dim] + output_shape = [batch, height / 2, width / 2, out_dim] - x = tf.placeholder(tf.float32, input_shape) - y = layers.conv2d(x, filter_size, out_dim, 'conv2d', strides=[1,2,2,1]) - y2 = layers.conv2d_transpose(y, filter_size, in_dim, 'conv2d_tr', strides=[1,2,2,1]) + with tf.Graph().as_default(): + x = tf.placeholder(tf.float32, input_shape) + y = layers.conv2d( + x, filter_size, + out_dim, 'conv2d', + strides=[1, 2, 2, 1]) + z = layers.conv2d_transpose( + y, filter_size, + in_dim, 'conv2d_tr', + strides=[1, 2, 2, 1]) + z_1 = layers.conv2d_transpose(y, + filter_size, + in_dim, + 'conv2d_tr_1', + padding='VALID', + strides=[1, 2, 2, 1]) + + y2 = layers.conv2d(x, + filter_size, + out_dim, + 'conv2d2', + padding='VALID', + strides=[1, 2, 2, 1]) + z2 = layers.conv2d_transpose(y2, + filter_size, + in_dim, + 'conv2d_tr2', + padding='VALID', + strides=[1, 2, 2, 1], + out_shape=input_shape) + + assert (np.all(z.get_shape().as_list() == input_shape)) + assert (z_1.get_shape()[1] == 17) + assert (np.all(z2.get_shape().as_list() == input_shape)) - assert(np.all(y2.get_shape().as_list() == input_shape)) def test_atrous_conv2d(): @@ -102,7 +133,7 @@ def test_batch_norm_2d(): y = layers.batch_norm(x, training) sess = tf.Session() x_ = np.float32(np.random.randn(*input_shape)) - + sess.run(tf.global_variables_initializer()) y_hat = sess.run(y, feed_dict={x: x_, training: True}) @@ -150,6 +181,7 @@ def test_batch_norm_3d(): tf.reset_default_graph() + def test_batchnorm_train_mode_2d(): batch = 5 width = 2 @@ -184,6 +216,7 @@ def test_batchnorm_train_mode_2d(): 'during training with batch norm' assert (np.all(y_eval2 == np.zeros(input_shape))), assert_str + def test_batchnorm_train_mode(): batch = 5 width = 2 @@ -216,7 +249,10 @@ def test_batchnorm_train_mode(): assert (np.all(y_eval1 == np.zeros(input_shape))), assert_str assert_str = 'batch mean and var are not used correctly' + \ 'during training with batch norm' - np.testing.assert_allclose(y_eval2, np.zeros(input_shape), atol=1e-4, err_msg=assert_str) + np.testing.assert_allclose(y_eval2, + np.zeros(input_shape), + atol=1e-4, + err_msg=assert_str) def test_batchnorm_test_mode(): @@ -229,7 +265,7 @@ def test_batchnorm_test_mode(): g = tf.Graph() with g.as_default(): - training = tf.placeholder(tf.bool, shape = (), name='is_train') + training = tf.placeholder(tf.bool, shape=(), name='is_train') x = tf.placeholder(tf.float32, input_shape) y = layers.batch_norm(x, training, name='bn') initializer = tf.global_variables_initializer() @@ -312,21 +348,18 @@ def test_placeholder_management(): plh_mgr.add_placeholder('sp_ids', tf.int64, [2, 4], sparse=True) plh_mgr.add_placeholder('weights', tf.float64, [2, 4], sparse=True) plh_mgr.add_placeholder('list_of_tensors', tf.int64, [4], list_len=3) - plh_mgr.add_placeholder( - 'list_of_sparse_tensors', - tf.int64, - [2,2], - list_len=2, - sparse=True) - + plh_mgr.add_placeholder('list_of_sparse_tensors', + tf.int64, [2, 2], + list_len=2, + sparse=True) + # Get a dictionary of placeholders plhs = plh_mgr # Define computation graph - y = tf.nn.embedding_lookup_sparse( - plhs['word_embed'], - plhs['sp_ids']['tensor'], - plhs['weights']['tensor']) + y = tf.nn.embedding_lookup_sparse(plhs['word_embed'], + plhs['sp_ids']['tensor'], + plhs['weights']['tensor']) z = dict() for t in range(len(plhs['list_of_tensors'])): @@ -334,8 +367,8 @@ def test_placeholder_management(): w = dict() for t in range(len(plhs['list_of_sparse_tensors'])): - w[t] = tf.sparse_tensor_to_dense( - plhs['list_of_sparse_tensors'][t]['tensor']) + w[t] = tf.sparse_tensor_to_dense(plhs['list_of_sparse_tensors'][t][ + 'tensor']) # Create data to be fed into the graph I = np.array([0, 0, 1, 1]) @@ -346,20 +379,19 @@ def test_placeholder_management(): weights = sps.coo_matrix((W, (I, J)), shape=(2, 4), dtype=np.float64) word_embed = np.eye(10, 10, dtype=np.float64) list_of_arrays = [ - np.array([1,2,3,-1]), - np.array([4,5,6,-2]), - np.array([7,8,9,-3]) + np.array([1, 2, 3, -1]), np.array([4, 5, 6, -2]), + np.array([7, 8, 9, -3]) ] list_of_sparse_matrices = [ - sps.eye(2,2,dtype=np.int64), - 2*sps.eye(2,2,dtype=np.int64), + sps.eye(2, 2, dtype=np.int64), + 2 * sps.eye(2, 2, dtype=np.int64), ] # Create input dict inputs = { - 'word_embed': word_embed, - 'sp_ids': sp_ids, - 'weights': weights, + 'word_embed': word_embed, + 'sp_ids': sp_ids, + 'weights': weights, 'list_of_tensors': list_of_arrays, 'list_of_sparse_tensors': list_of_sparse_matrices } @@ -379,7 +411,7 @@ def test_placeholder_management(): assert_str = 'passing list of lists failed' for t in range(len(list_of_arrays)): z_t_value = z[t].eval(feed_dict) - gt_z_t_value = list_of_arrays[t]+1 + gt_z_t_value = list_of_arrays[t] + 1 assert (np.array_equal(z_t_value, gt_z_t_value)), assert_str assert_str = 'passing list of sparse matrices failed' @@ -387,7 +419,7 @@ def test_placeholder_management(): w_t_value = w[t].eval(feed_dict) gt_w_t_value = list_of_sparse_matrices[t].todense() # convert to dense to compare - assert(np.array_equal(w_t_value,gt_w_t_value)), assert_str + assert (np.array_equal(w_t_value, gt_w_t_value)), assert_str tf.reset_default_graph()