diff --git a/aiutils/tftools/layers.py b/aiutils/tftools/layers.py index 9a47319..809b499 100644 --- a/aiutils/tftools/layers.py +++ b/aiutils/tftools/layers.py @@ -101,6 +101,49 @@ def conv2d(input, return output +def conv2d_transpose(input, + filter_size, + out_dim, + name, + strides=[1, 1, 1, 1], + padding='SAME', + gain=np.sqrt(2), + func=tf.nn.relu, + reuse_vars=False, + out_shape=None): + in_shape = input.get_shape().as_list() + in_dim = in_shape[-1] + + if out_shape == None: + out_shape = in_shape + out_shape[3] = out_dim + if padding == 'SAME': + out_shape[1] = in_shape[1] * strides[1] + out_shape[2] = in_shape[2] * strides[2] + if padding == 'VALID': + out_shape[1] = (in_shape[1] - 1) * strides[1] + filter_size + out_shape[2] = (in_shape[2] - 1) * strides[2] + filter_size + + stddev = 1.0 * gain / np.sqrt(filter_size * filter_size * in_dim) + with tf.variable_scope(name, reuse=reuse_vars): + w_init = tf.random_normal_initializer(stddev=stddev) + b_init = tf.constant_initializer() + w = tf.get_variable('w', + shape=[filter_size, filter_size, out_dim, in_dim], + initializer=w_init) + b = tf.get_variable('b', shape=[out_dim], initializer=b_init) + + output = tf.nn.conv2d_transpose( + input, w, out_shape, + strides=strides, + padding=padding) + b + if func is not None: + output = func(output) + + tf.add_to_collection('to_regularize', w) + return output + + def atrous_conv2d(input, filter_size, out_dim, @@ -158,7 +201,6 @@ def atrous_conv2d(input, return output - def batch_norm(input, training=tf.constant(True), decay=0.95, @@ -190,18 +232,17 @@ def batch_norm(input, else: raise ValueError('Input tensor must have rank 2 or 4.') - output = tf.contrib.layers.batch_norm( - input, - decay=decay, - is_training=training, - scale=True, - epsilon=epsilon, - updates_collections=None, - scope=name, - reuse=reuse_vars) + output = tf.contrib.layers.batch_norm(input, + decay=decay, + is_training=training, + scale=True, + epsilon=epsilon, + updates_collections=None, + scope=name, + reuse=reuse_vars) if rank == 2: - return tf.squeeze(output, [1,2]) + return tf.squeeze(output, [1, 2]) return output @@ -236,8 +277,8 @@ def dropout(input, training=True, keep_prob=.8, noise_shape=None, seed=None): return tf.cond( training, lambda: tf.nn.dropout( - input, - keep_prob, - noise_shape=noise_shape, + input, + keep_prob, + noise_shape=noise_shape, seed=seed), lambda: input) diff --git a/tests/test_tftools.py b/tests/test_tftools.py index 427ec65..be89bdd 100644 --- a/tests/test_tftools.py +++ b/tests/test_tftools.py @@ -56,6 +56,53 @@ def test_conv2d(): tf.reset_default_graph() +def test_conv2d_transpose(): + batch = 1 + height = 16 + width = 16 + filter_size = 3 + in_dim = 4 + out_dim = 5 + + input_shape = [batch, height, width, in_dim] + output_shape = [batch, height / 2, width / 2, out_dim] + + with tf.Graph().as_default(): + x = tf.placeholder(tf.float32, input_shape) + y = layers.conv2d( + x, filter_size, + out_dim, 'conv2d', + strides=[1, 2, 2, 1]) + z = layers.conv2d_transpose( + y, filter_size, + in_dim, 'conv2d_tr', + strides=[1, 2, 2, 1]) + z_1 = layers.conv2d_transpose(y, + filter_size, + in_dim, + 'conv2d_tr_1', + padding='VALID', + strides=[1, 2, 2, 1]) + + y2 = layers.conv2d(x, + filter_size, + out_dim, + 'conv2d2', + padding='VALID', + strides=[1, 2, 2, 1]) + z2 = layers.conv2d_transpose(y2, + filter_size, + in_dim, + 'conv2d_tr2', + padding='VALID', + strides=[1, 2, 2, 1], + out_shape=input_shape) + + assert (np.all(z.get_shape().as_list() == input_shape)) + assert (z_1.get_shape()[1] == 17) + assert (np.all(z2.get_shape().as_list() == input_shape)) + + def test_atrous_conv2d(): input_shape = [10, 100, 100, 3] @@ -86,7 +133,7 @@ def test_batch_norm_2d(): y = layers.batch_norm(x, training) sess = tf.Session() x_ = np.float32(np.random.randn(*input_shape)) - + sess.run(tf.global_variables_initializer()) y_hat = sess.run(y, feed_dict={x: x_, training: True}) @@ -134,6 +181,7 @@ def test_batch_norm_3d(): tf.reset_default_graph() + def test_batchnorm_train_mode_2d(): batch = 5 width = 2 @@ -168,6 +216,7 @@ def test_batchnorm_train_mode_2d(): 'during training with batch norm' assert (np.all(y_eval2 == np.zeros(input_shape))), assert_str + def test_batchnorm_train_mode(): batch = 5 width = 2 @@ -200,7 +249,10 @@ def test_batchnorm_train_mode(): assert (np.all(y_eval1 == np.zeros(input_shape))), assert_str assert_str = 'batch mean and var are not used correctly' + \ 'during training with batch norm' - np.testing.assert_allclose(y_eval2, np.zeros(input_shape), atol=1e-4, err_msg=assert_str) + np.testing.assert_allclose(y_eval2, + np.zeros(input_shape), + atol=1e-4, + err_msg=assert_str) def test_batchnorm_test_mode(): @@ -213,7 +265,7 @@ def test_batchnorm_test_mode(): g = tf.Graph() with g.as_default(): - training = tf.placeholder(tf.bool, shape = (), name='is_train') + training = tf.placeholder(tf.bool, shape=(), name='is_train') x = tf.placeholder(tf.float32, input_shape) y = layers.batch_norm(x, training, name='bn') initializer = tf.global_variables_initializer() @@ -296,21 +348,18 @@ def test_placeholder_management(): plh_mgr.add_placeholder('sp_ids', tf.int64, [2, 4], sparse=True) plh_mgr.add_placeholder('weights', tf.float64, [2, 4], sparse=True) plh_mgr.add_placeholder('list_of_tensors', tf.int64, [4], list_len=3) - plh_mgr.add_placeholder( - 'list_of_sparse_tensors', - tf.int64, - [2,2], - list_len=2, - sparse=True) - + plh_mgr.add_placeholder('list_of_sparse_tensors', + tf.int64, [2, 2], + list_len=2, + sparse=True) + # Get a dictionary of placeholders plhs = plh_mgr # Define computation graph - y = tf.nn.embedding_lookup_sparse( - plhs['word_embed'], - plhs['sp_ids']['tensor'], - plhs['weights']['tensor']) + y = tf.nn.embedding_lookup_sparse(plhs['word_embed'], + plhs['sp_ids']['tensor'], + plhs['weights']['tensor']) z = dict() for t in range(len(plhs['list_of_tensors'])): @@ -318,8 +367,8 @@ def test_placeholder_management(): w = dict() for t in range(len(plhs['list_of_sparse_tensors'])): - w[t] = tf.sparse_tensor_to_dense( - plhs['list_of_sparse_tensors'][t]['tensor']) + w[t] = tf.sparse_tensor_to_dense(plhs['list_of_sparse_tensors'][t][ + 'tensor']) # Create data to be fed into the graph I = np.array([0, 0, 1, 1]) @@ -330,20 +379,19 @@ def test_placeholder_management(): weights = sps.coo_matrix((W, (I, J)), shape=(2, 4), dtype=np.float64) word_embed = np.eye(10, 10, dtype=np.float64) list_of_arrays = [ - np.array([1,2,3,-1]), - np.array([4,5,6,-2]), - np.array([7,8,9,-3]) + np.array([1, 2, 3, -1]), np.array([4, 5, 6, -2]), + np.array([7, 8, 9, -3]) ] list_of_sparse_matrices = [ - sps.eye(2,2,dtype=np.int64), - 2*sps.eye(2,2,dtype=np.int64), + sps.eye(2, 2, dtype=np.int64), + 2 * sps.eye(2, 2, dtype=np.int64), ] # Create input dict inputs = { - 'word_embed': word_embed, - 'sp_ids': sp_ids, - 'weights': weights, + 'word_embed': word_embed, + 'sp_ids': sp_ids, + 'weights': weights, 'list_of_tensors': list_of_arrays, 'list_of_sparse_tensors': list_of_sparse_matrices } @@ -363,7 +411,7 @@ def test_placeholder_management(): assert_str = 'passing list of lists failed' for t in range(len(list_of_arrays)): z_t_value = z[t].eval(feed_dict) - gt_z_t_value = list_of_arrays[t]+1 + gt_z_t_value = list_of_arrays[t] + 1 assert (np.array_equal(z_t_value, gt_z_t_value)), assert_str assert_str = 'passing list of sparse matrices failed' @@ -371,7 +419,7 @@ def test_placeholder_management(): w_t_value = w[t].eval(feed_dict) gt_w_t_value = list_of_sparse_matrices[t].todense() # convert to dense to compare - assert(np.array_equal(w_t_value,gt_w_t_value)), assert_str + assert (np.array_equal(w_t_value, gt_w_t_value)), assert_str tf.reset_default_graph()