def testHigherRank(self):
   np.random.seed(1)
   # We check that scalar and empty shapes work as well
   for shape in (7, 0), (4, 3, 2):
     for indices_shape in (), (0,), (3, 0), (3, 5):
       params = np.random.randn(*shape)
       indices = np.random.randint(shape[0], size=indices_shape)
       with self.test_session(use_gpu=self.use_gpu):
         tf_params = tf.constant(params)
         tf_indices = tf.constant(indices)
         gather = tf.gather(tf_params, tf_indices)
         self.assertAllEqual(params[indices], gather.eval())
         self.assertEqual(indices.shape + params.shape[1:], gather.get_shape())
         # Test gradients
         gather_grad = np.random.randn(*gather.get_shape().as_list())
         params_grad, indices_grad = tf.gradients(
             gather, [tf_params, tf_indices], gather_grad)
         self.assertEqual(indices_grad, None)
         self.assertEqual(type(params_grad), tf.IndexedSlices)
         params_grad = tf.convert_to_tensor(params_grad)
         correct_params_grad = np.zeros(shape)
         for i, g in zip(indices.flat,
                         gather_grad.reshape((indices.size,) + shape[1:])):
           correct_params_grad[i] += g
         self.assertAllClose(correct_params_grad, params_grad.eval())

  def testMutableHashTableExportInsert(self):
    with self.test_session():
      default_val = tf.constant([-1, -1], tf.int64)
      keys = tf.constant(["brain", "salad", "surgery"])
      values = tf.constant([[0, 1], [2, 3], [4, 5]], tf.int64)
      table1 = tf.contrib.lookup.MutableHashTable(tf.string, tf.int64,
                                                  default_val)
      self.assertAllEqual(0, table1.size().eval())
      table1.insert(keys, values).run()
      self.assertAllEqual(3, table1.size().eval())

      input_string = tf.constant(["brain", "salad", "tank"])
      expected_output = [[0, 1], [2, 3], [-1, -1]]
      output1 = table1.lookup(input_string)
      self.assertAllEqual(expected_output, output1.eval())

      exported_keys, exported_values = table1.export()
      self.assertAllEqual(3, exported_keys.eval().size)
      self.assertAllEqual(6, exported_values.eval().size)

      # Populate a second table from the exported data
      table2 = tf.contrib.lookup.MutableHashTable(tf.string, tf.int64,
                                                  default_val)
      self.assertAllEqual(0, table2.size().eval())
      table2.insert(exported_keys, exported_values).run()
      self.assertAllEqual(3, table2.size().eval())

      # Verify lookup result is still the same
      output2 = table2.lookup(input_string)
      self.assertAllEqual(expected_output, output2.eval())

  def testSignatureMismatch(self):
    with self.test_session():
      default_val = -1
      keys = tf.constant(["brain", "salad", "surgery"])
      values = tf.constant([0, 1, 2], tf.int64)
      table = tf.contrib.lookup.MutableHashTable(tf.string,
                                                 tf.int64,
                                                 default_val)

      # insert with keys of the wrong type
      with self.assertRaises(TypeError):
        table.insert(tf.constant([4, 5, 6]), values).run()

      # insert with values of the wrong type
      with self.assertRaises(TypeError):
        table.insert(keys, tf.constant(["a", "b", "c"])).run()

      self.assertAllEqual(0, table.size().eval())

      table.insert(keys, values).run()
      self.assertAllEqual(3, table.size().eval())

      # lookup with keys of the wrong type
      input_string = tf.constant([1, 2, 3], tf.int64)
      with self.assertRaises(TypeError):
        table.lookup(input_string).eval()

      # default value of the wrong type
      with self.assertRaises(TypeError):
        tf.contrib.lookup.MutableHashTable(tf.string, tf.int64, "UNK")

  def testTensorArrayGradientWritePackConcatAndRead(self):
    with self.test_session(use_gpu=self._use_gpu) as sess:
      ta = tensor_array_ops.TensorArray(
          dtype=tf.float32, tensor_array_name="foo", size=2,
          clear_after_read=False)

      value_0 = tf.constant([-1.0, 1.0])
      value_1 = tf.constant([-10.0, 10.0])

      w0 = ta.write(0, value_0)
      w1 = w0.write(1, value_1)
      p0 = w1.pack()
      r0 = w1.read(0)
      s0 = w1.concat()

      # Test gradient accumulation between read(0), pack(), and concat()
      with tf.control_dependencies([p0, r0, s0]):
        grad_r = tf.gradients(
            ys=[p0, r0, s0], xs=[value_0, value_1],
            grad_ys=[
                [[2.0, 3.0], [4.0, 5.0]],  # pack gradient
                [-0.5, 1.5],  # read(0) gradient
                [20.0, 30.0, 40.0, 50.0]])  # concat gradient
      grad_vals = sess.run(grad_r)  # 2 + 2 entries

      self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
      self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])

def resize_images(X, height_factor, width_factor, dim_ordering):
    '''Resizes the images contained in a 4D tensor of shape
    - [batch, channels, height, width] (for 'th' dim_ordering)
    - [batch, height, width, channels] (for 'tf' dim_ordering)
    by a factor of (height_factor, width_factor). Both factors should be
    positive integers.
    '''
    if dim_ordering == 'th':
        original_shape = int_shape(X)
        new_shape = tf.shape(X)[2:]
        new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
        X = permute_dimensions(X, [0, 2, 3, 1])
        X = tf.image.resize_nearest_neighbor(X, new_shape)
        X = permute_dimensions(X, [0, 3, 1, 2])
        X.set_shape((None, None, original_shape[2] * height_factor, original_shape[3] * width_factor))
        return X
    elif dim_ordering == 'tf':
        original_shape = int_shape(X)
        new_shape = tf.shape(X)[1:3]
        new_shape *= tf.constant(np.array([height_factor, width_factor]).astype('int32'))
        X = tf.image.resize_nearest_neighbor(X, new_shape)
        X.set_shape((None, original_shape[1] * height_factor, original_shape[2] * width_factor, None))
        return X
    else:
        raise Exception('Invalid dim_ordering: ' + dim_ordering)

    def prepareGraph(self):
        logging.debug("prepareGraph")
#         image_size = self.image_size
#         num_labels = self.num_labels
        
        graph = tf.Graph()
        self.graph = graph
        with graph.as_default():
            # Input data.
            # Load the training, validation and test data into constants that are
            # attached to the graph.
            self.getInputData()
#             tf_train_dataset, tf_train_labels = self.getInputData()
            tf_valid_dataset = tf.constant(self.valid_dataset)
            tf_test_dataset = tf.constant(self.test_dataset)
            
            self.setupVariables()
            
            self.setupLossFunction()
            # Optimizer.
            # We are going to find the minimum of this loss using gradient descent.
            self.setupOptimizer()
            
            # Predictions for the training, validation, and test data.
            # These are not part of training, but merely here so that we can report
            # accuracy figures as we train.
            train_prediction = tf.nn.softmax(self.getTempModleOutput_forTest(self.tf_train_dataset))
            valid_prediction = tf.nn.softmax(self.getTempModleOutput_forTest(tf_valid_dataset))
            test_prediction = tf.nn.softmax(self.getTempModleOutput_forTest(tf_test_dataset))
            
            self.train_prediction = train_prediction
            self.valid_prediction= valid_prediction
            self.test_prediction = test_prediction
               
        return

  def build_greedy_training(self, state, network_states):
    """Extracts features and advances a batch using the oracle path.

    Args:
      state: MasterState from the 'AdvanceMaster' op that advances the
          underlying master to this component.
      network_states: dictionary of component NetworkState objects

    Returns:
      state handle: final state after advancing
      cost: regularization cost, possibly associated with embedding matrices
      correct: since no gold path is available, 0.
      total: since no gold path is available, 0.
    """
    logging.info('Building component: %s', self.spec.name)
    stride = state.current_batch_size * self.training_beam_size
    with tf.variable_scope(self.name, reuse=True):
      state.handle, fixed_embeddings = fetch_differentiable_fixed_embeddings(
          self, state, stride)

    linked_embeddings = [
        fetch_linked_embedding(self, network_states, spec)
        for spec in self.spec.linked_feature
    ]

    with tf.variable_scope(self.name, reuse=True):
      tensors = self.network.create(
          fixed_embeddings, linked_embeddings, None, None, True, stride=stride)
    update_network_states(self, tensors, network_states, stride)
    cost = self.add_regularizer(tf.constant(0.))

    correct, total = tf.constant(0), tf.constant(0)
    return state.handle, cost, correct, total

def net(file_name, x, pooling_function='MAX'):
    mat_dict = scipy.io.loadmat(file_name)
    img_mean = mat_dict['meta'][0][0][1][0][0][0][0][0]
    layers = mat_dict['layers'][0]
    vgg = x
    content_activations = {}
    relu_num = 1
    pool_num = 1
    for layer_data in layers:
        layer = layer_data[0][0]
        layer_type = layer[1][0]
        if layer_type == 'conv':
            weights, biases, *rest = layer[2][0]
            # permute `weights` elements for input to TensorFlow
            weights = np.transpose(weights, (1, 0, 2, 3))
            W_conv = tf.constant(weights)
            # convert `biases` shape from [n,1] to [n]
            biases = biases.reshape(-1)
            b_conv = tf.constant(biases)
            vgg = conv2d(vgg, W_conv, 1) + b_conv
        elif layer_type == 'relu':
            vgg = tf.nn.relu(vgg)
            content_activations["relu"+str(pool_num)+"_"+str(relu_num)] = vgg
            relu_num += 1
        elif layer_type == 'pool':
            if pooling_function == 'AVG':
                vgg = avg_pool(vgg, 2)
            else:
                vgg = max_pool(vgg, 2)
            pool_num += 1
            relu_num = 1
    return vgg, content_activations, img_mean

  def encode_coordinates_alt(self, net):
    """An alternative implemenation for the encoding coordinates.

    Args:
      net: a tensor of shape=[batch_size, height, width, num_features]

    Returns:
      a list of tensors with encoded image coordinates in them.
    """
    batch_size, h, w, _ = net.shape.as_list()
    h_loc = [
      tf.tile(
          tf.reshape(
              tf.contrib.layers.one_hot_encoding(
                  tf.constant([i]), num_classes=h), [h, 1]), [1, w])
      for i in xrange(h)
    ]
    h_loc = tf.concat([tf.expand_dims(t, 2) for t in h_loc], 2)
    w_loc = [
      tf.tile(
          tf.contrib.layers.one_hot_encoding(tf.constant([i]), num_classes=w),
          [h, 1]) for i in xrange(w)
    ]
    w_loc = tf.concat([tf.expand_dims(t, 2) for t in w_loc], 2)
    loc = tf.concat([h_loc, w_loc], 2)
    loc = tf.tile(tf.expand_dims(loc, 0), [batch_size, 1, 1, 1])
    return tf.concat([net, loc], 3)

 def testShapeWrong(self):
   with tf.Graph().as_default():
     with self.assertRaisesWithPredicateMatch(
         ValueError,
         lambda e: ("Too many elements provided. Needed at most 5, "
                    "but received 7" == str(e))):
       tf.constant([1, 2, 3, 4, 5, 6, 7], shape=[5])

def boston_input_fn():
    boston = tf.contrib.learn.datasets.load_boston()
    features = tf.cast(
        tf.reshape(tf.constant(boston.data), [-1, 13]), tf.float32)
    labels = tf.cast(
        tf.reshape(tf.constant(boston.target), [-1, 1]), tf.float32)
    return features, labels

  def test_draw_bounding_boxes_on_image_tensors(self):
    """Tests that bounding box utility produces reasonable results."""
    category_index = {1: {'id': 1, 'name': 'dog'}, 2: {'id': 2, 'name': 'cat'}}

    fname = os.path.join(_TESTDATA_PATH, 'image1.jpg')
    image_np = np.array(Image.open(fname))
    images_np = np.stack((image_np, image_np), axis=0)

    with tf.Graph().as_default():
      images_tensor = tf.constant(value=images_np, dtype=tf.uint8)
      boxes = tf.constant([[[0.4, 0.25, 0.75, 0.75], [0.5, 0.3, 0.6, 0.9]],
                           [[0.25, 0.25, 0.75, 0.75], [0.1, 0.3, 0.6, 1.0]]])
      classes = tf.constant([[1, 1], [1, 2]], dtype=tf.int64)
      scores = tf.constant([[0.8, 0.1], [0.6, 0.5]])
      images_with_boxes = (
          visualization_utils.draw_bounding_boxes_on_image_tensors(
              images_tensor,
              boxes,
              classes,
              scores,
              category_index,
              min_score_thresh=0.2))

      with self.test_session() as sess:
        sess.run(tf.global_variables_initializer())

        # Write output images for visualization.
        images_with_boxes_np = sess.run(images_with_boxes)
        self.assertEqual(images_np.shape, images_with_boxes_np.shape)
        for i in range(images_with_boxes_np.shape[0]):
          img_name = 'image_' + str(i) + '.png'
          output_file = os.path.join(self.get_temp_dir(), img_name)
          logging.info('Writing output image %d to %s', i, output_file)
          image_pil = Image.fromarray(images_with_boxes_np[i, ...])
          image_pil.save(output_file)

def convert_data_to_tensors(x, y):
    inputs = tf.constant(x)
    inputs.set_shape([None, 1])
    
    outputs = tf.constant(y)
    outputs.set_shape([None, 1])
    return inputs, outputs

 def bn_variables(self, size, name):
     weights = OrderedDict()
     weights[name+'_mean'] = tf.Variable(tf.constant(0.0, shape=size))
     weights[name +'_variance'] = tf.Variable(tf.constant(1.0, shape=size))
     weights[name + '_offset'] = tf.Variable(tf.constant(0.0, shape=size))
     weights[name + '_scale'] = tf.Variable(tf.constant(1.0, shape=size))
     return weights

def custom_layer(input_matrix):
    input_matrix_sqeezed = tf.squeeze(input_matrix)
    A = tf.constant([[1., 2.], [-1., 3.]])
    b = tf.constant(1., shape=[2, 2])
    temp1 = tf.matmul(A, input_matrix_sqeezed)
    temp = tf.add(temp1, b) # Ax + b
    return(tf.sigmoid(temp))

def iris_input_fn(num_epochs=None):
  iris = tf.contrib.learn.datasets.load_iris()
  features = tf.reshape(tf.constant(iris.data), [-1, 4])
  if num_epochs:
    features = tf.train.limit_epochs(features, num_epochs=num_epochs)
  target = tf.reshape(tf.constant(iris.target), [-1])
  return features, target

  def test_quadratic(self):
    fdf = lambda x: ((x-1.3)**2, 2*(x-1.3))

    # Case 1: The starting value is close to 0 and doesn't bracket the min.
    close_start, far_start = tf.constant(0.1), tf.constant(7.0)
    results_close = self.evaluate(tfp.optimizer.linesearch.hager_zhang(
        fdf, initial_step_size=close_start))
    self.assertTrue(results_close.converged)
    self.assertAlmostEqual(results_close.left_pt, results_close.right_pt)
    f0, df0 = fdf(0.0)
    self.assertTrue(_is_exact_wolfe(results_close.left_pt,
                                    results_close.objective_at_left_pt,
                                    results_close.grad_objective_at_left_pt,
                                    f0,
                                    df0,
                                    0.1,
                                    0.9))

    results_far = self.evaluate(tfp.optimizer.linesearch.hager_zhang(
        fdf, initial_step_size=far_start))
    self.assertTrue(results_far.converged)
    self.assertAlmostEqual(results_far.left_pt, results_far.right_pt)
    self.assertTrue(_is_exact_wolfe(results_far.left_pt,
                                    results_far.objective_at_left_pt,
                                    results_far.grad_objective_at_left_pt,
                                    f0,
                                    df0,
                                    0.1,
                                    0.9))

  def testEmptyUpdateOps(self):
    with tf.Graph().as_default():
      tf.set_random_seed(0)
      tf_inputs = tf.constant(self._inputs, dtype=tf.float32)
      tf_labels = tf.constant(self._labels, dtype=tf.float32)

      tf_predictions = BatchNormClassifier(tf_inputs)
      slim.losses.log_loss(tf_predictions, tf_labels)
      total_loss = slim.losses.get_total_loss()
      optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)

      train_op = slim.learning.create_train_op(total_loss, optimizer,
                                               update_ops=[])

      moving_mean = tf.contrib.framework.get_variables_by_name('moving_mean')[0]
      moving_variance = tf.contrib.framework.get_variables_by_name(
          'moving_variance')[0]

      with tf.Session() as sess:
        # Initialize all variables
        sess.run(tf.global_variables_initializer())
        mean, variance = sess.run([moving_mean, moving_variance])
        # After initialization moving_mean == 0 and moving_variance == 1.
        self.assertAllClose(mean, [0] * 4)
        self.assertAllClose(variance, [1] * 4)

        for _ in range(10):
          sess.run([train_op])
        mean = moving_mean.eval()
        variance = moving_variance.eval()
        # Since we skip update_ops the moving_vars are not updated.
        self.assertAllClose(mean, [0] * 4)
        self.assertAllClose(variance, [1] * 4)

  def _test_normal_normal(self, default, dtype):
    with self.test_session() as sess:
      x_data = np.array([0.0] * 50, dtype=np.float32)

      mu = Normal(loc=tf.constant(0.0, dtype=dtype),
                  scale=tf.constant(1.0, dtype=dtype))
      x = Normal(loc=mu, scale=tf.constant(1.0, dtype=dtype),
                 sample_shape=50)

      n_samples = 2000
      # analytic solution: N(loc=0.0, scale=\sqrt{1/51}=0.140)
      if not default:
        qmu = Empirical(params=tf.Variable(tf.ones(n_samples, dtype=dtype)))
        inference = ed.HMC({mu: qmu}, data={x: x_data})
      else:
        inference = ed.HMC([mu], data={x: x_data})
        qmu = inference.latent_vars[mu]
      inference.run()

      self.assertAllClose(qmu.mean().eval(), 0, rtol=1e-1, atol=1e-1)
      self.assertAllClose(qmu.stddev().eval(), np.sqrt(1 / 51),
                          rtol=1e-1, atol=1e-1)

      old_t, old_n_accept = sess.run([inference.t, inference.n_accept])
      if not default:
        self.assertEqual(old_t, n_samples)
      else:
        self.assertEqual(old_t, 1e4)
      self.assertGreater(old_n_accept, 0.1)
      sess.run(inference.reset)
      new_t, new_n_accept = sess.run([inference.t, inference.n_accept])
      self.assertEqual(new_t, 0)
      self.assertEqual(new_n_accept, 0)

  def testNoneGlobalStep(self):
    with tf.Graph().as_default():
      tf.set_random_seed(0)
      tf_inputs = tf.constant(self._inputs, dtype=tf.float32)
      tf_labels = tf.constant(self._labels, dtype=tf.float32)

      tf_predictions = BatchNormClassifier(tf_inputs)
      slim.losses.log_loss(tf_predictions, tf_labels)
      total_loss = slim.losses.get_total_loss()
      optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)

      train_op = slim.learning.create_train_op(total_loss,
                                               optimizer,
                                               global_step=None)

      global_step = slim.get_or_create_global_step()

      with tf.Session() as sess:
        # Initialize all variables
        sess.run(tf.global_variables_initializer())

        for _ in range(10):
          sess.run([train_op])
        global_step = global_step.eval()
        # Since train_op don't use global_step it shouldn't change.
        self.assertAllClose(global_step, 0)