def _meshgrid(depth, height, width, z_near, z_far):
    with tf.variable_scope('_meshgrid'):
      x_t = tf.reshape(
          tf.tile(tf.linspace(-1.0, 1.0, width), [height * depth]),
          [depth, height, width])
      y_t = tf.reshape(
          tf.tile(tf.linspace(-1.0, 1.0, height), [width * depth]),
          [depth, width, height])
      y_t = tf.transpose(y_t, [0, 2, 1])
      sample_grid = tf.tile(
          tf.linspace(float(z_near), float(z_far), depth), [width * height])
      z_t = tf.reshape(sample_grid, [height, width, depth])
      z_t = tf.transpose(z_t, [2, 0, 1])

      z_t = 1 / z_t
      d_t = 1 / z_t
      x_t /= z_t
      y_t /= z_t

      x_t_flat = tf.reshape(x_t, (1, -1))
      y_t_flat = tf.reshape(y_t, (1, -1))
      d_t_flat = tf.reshape(d_t, (1, -1))

      ones = tf.ones_like(x_t_flat)
      grid = tf.concat([d_t_flat, y_t_flat, x_t_flat, ones], 0)
      return grid

    def _meshgrid(height, width, fp):
        x_t = tf.matmul(
            tf.ones(shape=tf.stack([height, 1])),
            tf.transpose(tf.expand_dims(tf.linspace(-1.0, 1.0, width), 1), [1, 0]))
        y_t = tf.matmul(
            tf.expand_dims(tf.linspace(-1.0, 1.0, height), 1),
            tf.ones(shape=tf.stack([1, width])))

        x_t_flat = tf.reshape(x_t, (1, -1))
        y_t_flat = tf.reshape(y_t, (1, -1))

        x_t_flat_b = tf.expand_dims(x_t_flat, 0) # [1, 1, h*w]
        y_t_flat_b = tf.expand_dims(y_t_flat, 0) # [1, 1, h*w]

        num_batch = tf.shape(fp)[0]
        px = tf.expand_dims(fp[:,:,0], 2) # [n, nx*ny, 1]
        py = tf.expand_dims(fp[:,:,1], 2) # [n, nx*ny, 1]
        d = tf.sqrt(tf.pow(x_t_flat_b - px, 2.) + tf.pow(y_t_flat_b - py, 2.))
        r = tf.pow(d, 2) * tf.log(d + 1e-6) # [n, nx*ny, h*w]
        x_t_flat_g = tf.tile(x_t_flat_b, tf.stack([num_batch, 1, 1])) # [n, 1, h*w]
        y_t_flat_g = tf.tile(y_t_flat_b, tf.stack([num_batch, 1, 1])) # [n, 1, h*w]
        ones = tf.ones_like(x_t_flat_g) # [n, 1, h*w]

        grid = tf.concat([ones, x_t_flat_g, y_t_flat_g, r], 1) # [n, nx*ny+3, h*w]
        return grid

def meshgrid(batch, height, width, is_homogeneous=True):
  """Construct a 2D meshgrid.

  Args:
    batch: batch size
    height: height of the grid
    width: width of the grid
    is_homogeneous: whether to return in homogeneous coordinates
  Returns:
    x,y grid coordinates [batch, 2 (3 if homogeneous), height, width]
  """
  x_t = tf.matmul(tf.ones(shape=tf.stack([height, 1])),
                  tf.transpose(tf.expand_dims(
                      tf.linspace(-1.0, 1.0, width), 1), [1, 0]))
  y_t = tf.matmul(tf.expand_dims(tf.linspace(-1.0, 1.0, height), 1),
                  tf.ones(shape=tf.stack([1, width])))
  x_t = (x_t + 1.0) * 0.5 * tf.cast(width - 1, tf.float32)
  y_t = (y_t + 1.0) * 0.5 * tf.cast(height - 1, tf.float32)
  if is_homogeneous:
    ones = tf.ones_like(x_t)
    coords = tf.stack([x_t, y_t, ones], axis=0)
  else:
    coords = tf.stack([x_t, y_t], axis=0)
  coords = tf.tile(tf.expand_dims(coords, 0), [batch, 1, 1, 1])
  return coords

def gabor(n_values=32, sigma=1.0, mean=0.0):
	x = tf.linspace(-3.0, 3.0, n_values)
	z = (tf.exp(tf.negative(tf.pow(x - mean, 2.0)/ (2.0 * tf.pow(sigma, 2.0)))) * (1.0 / (sigma * tf.sqrt(2.0 * 3.145))))
	gauss_kernel = tf.matmul(tf.reshape(z, [n_values, 1]), tf.reshape(z,[1, n_values]))
	x = tf.reshape(tf.sin(tf.linspace(-3.0, 3.0, n_values)), [n_values, 1])
	y = tf.reshape(tf.ones_like(x), [1, n_values])
	gabor_kernel = tf.multiply(tf.matmul(x ,y), gauss_kernel)
	return gabor_kernel

def get_input_vectors(shape, phases, scaling, offset):
    x = tf.reshape(tf_repeat(offset[0] + tf.linspace(0.0, tf.to_float(shape[0] - 1), shape[0]) / scaling,
                             shape[1] * phases),
                   [shape[0], shape[1], phases]) * tf.pow(2.0, tf.linspace(0.0, tf.to_float(phases - 1), phases))
    y = tf.reshape(tf_repeat(tf.tile(
        offset[1] + tf.linspace(0.0, tf.to_float(shape[1] - 1), shape[1]) / scaling,
        [shape[0]]
    ), phases), [shape[0], shape[1], phases]) * tf.pow(2.0, tf.linspace(0.0, tf.to_float(phases - 1), phases))
    z = tf.reshape(
        tf.tile(offset[2] + 10 * tf.linspace(0.0, tf.to_float(phases - 1), phases), [shape[0] * shape[1]]),
        [shape[0], shape[1], phases, 1])
    x = tf.reshape(x, [shape[0], shape[1], phases, 1])
    y = tf.reshape(y, [shape[0], shape[1], phases, 1])
    return tf.reshape(tf.concat(3, [x, y, z]), [shape[0] * shape[1] * phases, 3])

def _meshgrid_abs(height, width):
  """Meshgrid in the absolute coordinates."""
  x_t = tf.matmul(
      tf.ones(shape=tf.stack([height, 1])),
      tf.transpose(tf.expand_dims(tf.linspace(-1.0, 1.0, width), 1), [1, 0]))
  y_t = tf.matmul(
      tf.expand_dims(tf.linspace(-1.0, 1.0, height), 1),
      tf.ones(shape=tf.stack([1, width])))
  x_t = (x_t + 1.0) * 0.5 * tf.cast(width - 1, tf.float32)
  y_t = (y_t + 1.0) * 0.5 * tf.cast(height - 1, tf.float32)
  x_t_flat = tf.reshape(x_t, (1, -1))
  y_t_flat = tf.reshape(y_t, (1, -1))
  ones = tf.ones_like(x_t_flat)
  grid = tf.concat([x_t_flat, y_t_flat, ones], axis=0)
  return grid

    def compute_center_coords(self, y_true, y_pred):
        batch_size = tf.shape(y_pred)[0]
        h = tf.shape(y_pred)[1]
        w = tf.shape(y_pred)[2]
        n_chans = tf.shape(y_pred)[3]
        n_dims = 5

        # weighted center of mass
        x = tf.cast(tf.tile(tf.reshape(self.xs, [1, h, w]), [batch_size, 1, 1]), tf.float32)
        y = tf.cast(tf.tile(tf.reshape(self.ys, [1, h, w]), [batch_size, 1, 1]), tf.float32)

        eps = 1e-8
        # grayscale
        pred_gray = tf.reduce_mean(y_pred, axis=-1)  # should be batch_size x h x w
        # normalize
        pred_gray = pred_gray - tf.reduce_min(pred_gray, axis=[1, 2], keepdims=True)
        pred_gray = pred_gray / (eps + tf.reduce_max(pred_gray, axis=[1, 2], keepdims=True))
        pred_gray = tf.clip_by_value(pred_gray, 0., 1.)

        # make each of these (batch_size, 1)
        weighted_x = tf.round(tf.expand_dims(
            tf.reduce_sum(x * pred_gray, axis=[1, 2]) / (eps + tf.reduce_sum(pred_gray, axis=[1, 2])), axis=-1))
        weighted_y = tf.round(tf.expand_dims(
            tf.reduce_sum(y * pred_gray, axis=[1, 2]) / (eps + tf.reduce_sum(pred_gray, axis=[1, 2])), axis=-1))
        batch_indices = tf.reshape(tf.linspace(0., tf.cast(batch_size, tf.float32) - 1., batch_size), [batch_size, 1])
        indices = tf.cast(tf.concat([batch_indices, weighted_y, weighted_x], axis=-1), tf.int32)
        #center_rgb = transform_network_utils.interpolate([y_true,  weighted_x, weighted_y], constant_vals=1.)
        center_rgb = tf.gather_nd(y_true, indices)
        center_rgb = tf.reshape(center_rgb, [batch_size, n_chans])

        center_point_xyrgb = tf.concat([
                        weighted_x, weighted_y, center_rgb
                    ], axis=-1)

        return pred_gray, center_point_xyrgb

def mgrid(*args, **kwargs):
    """
    create orthogonal grid
    similar to np.mgrid

    Parameters
    ----------
    args : int
        number of points on each axis
    low : float
        minimum coordinate value
    high : float
        maximum coordinate value

    Returns
    -------
    grid : tf.Tensor [len(args), args[0], ...]
        orthogonal grid
    """
    low = kwargs.pop("low", -1)
    high = kwargs.pop("high", 1)
    low = tf.to_float(low)
    high = tf.to_float(high)
    coords = (tf.linspace(low, high, arg) for arg in args)
    grid = tf.pack(tf.meshgrid(*coords, indexing='ij'))
    return grid

  def testNanFromGradsDontPropagate(self):
    """Test that update with NaN gradients does not cause NaN in results."""
    def _nan_log_prob_with_nan_gradient(x):
      return np.nan * tf.reduce_sum(x)

    initial_x = tf.linspace(0.01, 5, 10)
    hmc = tfp.mcmc.HamiltonianMonteCarlo(
        target_log_prob_fn=_nan_log_prob_with_nan_gradient,
        step_size=2.,
        num_leapfrog_steps=5,
        seed=_set_seed(47))
    updated_x, kernel_results = hmc.one_step(
        current_state=initial_x,
        previous_kernel_results=hmc.bootstrap_results(initial_x))
    initial_x_, updated_x_, log_accept_ratio_ = self.evaluate(
        [initial_x, updated_x, kernel_results.log_accept_ratio])
    acceptance_probs = np.exp(np.minimum(log_accept_ratio_, 0.))

    tf.logging.vlog(1, 'initial_x = {}'.format(initial_x_))
    tf.logging.vlog(1, 'updated_x = {}'.format(updated_x_))
    tf.logging.vlog(1, 'log_accept_ratio = {}'.format(log_accept_ratio_))

    self.assertAllEqual(initial_x_, updated_x_)
    self.assertEqual(acceptance_probs, 0.)

    self.assertAllFinite(
        self.evaluate(tf.gradients(updated_x, initial_x)[0]))
    self.assertAllEqual(
        [True],
        [g is None for g in tf.gradients(
            kernel_results.proposed_results.grads_target_log_prob,
            initial_x)])

 def _LinSpace(self, start, stop, num):
   # NOTE(touts): Needs to pass a graph to get a new session each time.
   with tf.Graph().as_default() as graph:
     with self.test_session(graph=graph, force_gpu=self.force_gpu):
       tf_ans = tf.linspace(start, stop, num, name="linspace")
       self.assertEqual([num], tf_ans.get_shape())
       return tf_ans.eval()

    def __init__(self, config):
        self.dim_img = config.dim_img
        self.dim_wav = config.dim_wav
        self.dim_mem = config.dim_mem
        self.mem_size = self.steps = config.mem_size
        
        self.n_hop = config.n_hop
        self.batch_size = config.batch_size
        self.do_prob = config.do_prob # keep prob
        self.nl = config.nl # keep prob
        self.learning_rate = config.learning_rate

        self.global_step = tf.Variable(0, name='g_step')
        self.A = tf.Variable(tf.random_normal([self.dim_wav, self.dim_mem]), name='A')
        self.b_A = tf.Variable(tf.random_normal([self.dim_mem]), name='b_A')
        self.B = tf.Variable(tf.random_normal([self.dim_img, self.dim_mem]), name='B')
        self.b_B = tf.Variable(tf.random_normal([self.dim_mem]), name='b_B')
        self.C = tf.Variable(tf.random_normal([self.dim_wav, self.dim_mem]), name='C')
        self.b_C = tf.Variable(tf.random_normal([self.dim_mem]), name='b_C')
        
        self._temporal = tf.linspace(0.0, np.float32(self.mem_size-1), self.mem_size)
        self.T_A = self.T_C = tf.Variable(self._temporal/tf.reduce_sum(self._temporal))
        
        self.W_o = tf.Variable(tf.random_normal([self.dim_mem, 1]))
        self.b_o = tf.Variable(tf.random_normal([1]))

 def compute_auc(tp, fn, tn, fp, name):
   """Computes the roc-auc or pr-auc based on confusion counts."""
   rec = tf.div(tp + epsilon, tp + fn + epsilon)
   if curve == 'ROC':
     fp_rate = tf.div(fp, fp + tn + epsilon)
     x = fp_rate
     y = rec
   elif curve == 'R':  # recall auc
     x = tf.linspace(1., 0., num_thresholds)
     y = rec
   else:  # curve == 'PR'.
     prec = tf.div(tp + epsilon, tp + fp + epsilon)
     x = rec
     y = prec
   if summation_method == 'trapezoidal':
     return tf.reduce_sum(
       tf.multiply(x[:num_thresholds - 1] - x[1:],
                   (y[:num_thresholds - 1] + y[1:]) / 2.),
       name=name)
   elif summation_method == 'minoring':
     return tf.reduce_sum(
       tf.multiply(x[:num_thresholds - 1] - x[1:],
                   tf.minimum(y[:num_thresholds - 1], y[1:])),
       name=name)
   elif summation_method == 'majoring':
     return tf.reduce_sum(
       tf.multiply(x[:num_thresholds - 1] - x[1:],
                   tf.maximum(y[:num_thresholds - 1], y[1:])),
       name=name)
   else:
     raise ValueError('Invalid summation_method: %s' % summation_method)

  def testRWM1DNNormal(self):
    """Sampling from the Standard Normal Distribution."""
    dtype = np.float32

    with self.test_session(graph=tf.Graph()) as sess:
      target = tfd.Normal(loc=dtype(0), scale=dtype(1))

      def make_kernel_fn(target_log_prob_fn, seed):
        return tfp.mcmc.HamiltonianMonteCarlo(
            target_log_prob_fn=target_log_prob_fn,
            seed=seed, step_size=1.0, num_leapfrog_steps=3)

      remc = tfp.mcmc.ReplicaExchangeMC(
          target_log_prob_fn=target.log_prob,
          inverse_temperatures=10.**tf.linspace(0., -2., 5),
          make_kernel_fn=make_kernel_fn,
          seed=42)

      samples, _ = tfp.mcmc.sample_chain(
          num_results=1000,
          current_state=dtype(1),
          kernel=remc,
          num_burnin_steps=500,
          parallel_iterations=1)  # For determinism.

      sample_mean = tf.reduce_mean(samples, axis=0)
      sample_std = tf.sqrt(
          tf.reduce_mean(tf.squared_difference(samples, sample_mean),
                         axis=0))
      [sample_mean_, sample_std_] = sess.run([sample_mean, sample_std])

    self.assertAllClose(sample_mean_, 0., atol=0.1, rtol=0.1)
    self.assertAllClose(sample_std_, 1., atol=0.1, rtol=0.1)

def w(input_data, cu, kappas_t_1, config):
	
	batch_size = config.batch_size
	mixture_size = config.mixture_size
	vocab_length = config.vocab_length

	# split along dim of mixture size * 3
	hat_alphas_t, hat_betas_t, hat_kappas_t = tf.split(1, 3, input_data)

	alphas_t = tf.exp(hat_alphas_t)
	betas_t = tf.exp(hat_betas_t)
	kappas_t = tf.add(kappas_t_1, tf.exp(hat_kappas_t))

	speech_length = tf.shape(cu)[1]

	u = tf.linspace(1.0, tf.cast(speech_length,tf.float32) , speech_length)
	u = tf.expand_dims(u, 0)
	u = tf.expand_dims(u, 0)
	u = tf.tile(u, [batch_size, mixture_size, 1])

	alphas_t_expanded = tf.tile(tf.expand_dims(alphas_t, -1), [1, 1, speech_length])
	betas_t_expanded = tf.tile(tf.expand_dims(betas_t, -1), [1, 1, speech_length])
	kappas_t_expanded = tf.tile(tf.expand_dims(kappas_t, -1), [1, 1, speech_length])

	calc = tf.square(tf.sub(kappas_t_expanded, u))
	calc = tf.mul(calc, tf.neg(betas_t_expanded))
	calc = tf.exp(calc)
	calc = tf.mul(calc, alphas_t_expanded)

	phi_t = tf.expand_dims(tf.reduce_sum(calc, 1), 1)

	output = tf.squeeze(tf.batch_matmul(phi_t, cu), [1])

	return output, kappas_t, phi_t

    def _meshgrid(self, height, width, depth):
        x_t = tf.matmul(tf.ones(shape=tf.stack([height, 1])),
                        tf.transpose(tf.expand_dims(tf.linspace(0.0,
                                                                tf.cast(width, tf.float32)-1.0, width), 1), [1, 0]))
        y_t = tf.matmul(tf.expand_dims(tf.linspace(0.0,
                                                   tf.cast(height, tf.float32)-1.0, height), 1),
                        tf.ones(shape=tf.stack([1, width])))

        x_t = tf.tile(tf.expand_dims(x_t, 2), [1, 1, depth])
        y_t = tf.tile(tf.expand_dims(y_t, 2), [1, 1, depth])

        z_t = tf.linspace(0.0, tf.cast(depth, tf.float32)-1.0, depth)
        z_t = tf.expand_dims(tf.expand_dims(z_t, 0), 0)
        z_t = tf.tile(z_t, [height, width, 1])

        return x_t, y_t, z_t

 def testBijector(self, lower, upper):
   bijector = tfb.Reciprocal()
   self.assertEqual('reciprocal', bijector.name)
   x = tf.linspace(lower, upper, 100)
   y = 1. / x
   self.assertAllClose(self.evaluate(y), self.evaluate(bijector.forward(x)))
   self.assertAllClose(self.evaluate(x), self.evaluate(bijector.inverse(y)))

 def test_finds_max_of_long_array(self):
   # d - 1 == d in float32 and d = 3e7.
   # So this test only passes if we use double for the percentile indices.
   # If float is used, it fails with InvalidArgumentError about an index out of
   # bounds.
   x = tf.linspace(0., 3e7, num=int(3e7))
   minval = tfd.percentile(x, q=0, validate_args=True)
   self.assertAllEqual(0, self.evaluate(minval))

 def _lin_space_weights(num, img_size):
   if num > 1:
     start_weights = tf.linspace(img_size - 1.0, 0.0, num)
     stop_weights = img_size - 1 - start_weights
   else:
     start_weights = tf.constant(num * [.5 * (img_size - 1)], dtype=tf.float32)
     stop_weights = tf.constant(num * [.5 * (img_size - 1)], dtype=tf.float32)
   return (start_weights, stop_weights)

 def _multi_gamma_sequence(self, a, p, name="multi_gamma_sequence"):
   """Creates sequence used in multivariate (di)gamma; shape = shape(a)+[p]."""
   with self._name_scope(name, values=[a, p]):
     # Linspace only takes scalars, so we'll add in the offset afterwards.
     seq = tf.linspace(
         tf.constant(0., dtype=self.dtype), 0.5 - 0.5 * p, tf.cast(
             p, tf.int32))
     return seq + tf.expand_dims(a, [-1])

def calculate_image(noise_values, phases, shape):
    val = tf.floor((tf.add_n(tf.split(
        2,
        phases,
        tf.reshape(noise_values, [shape[0], shape[1], phases]) / tf.pow(
            2.0,
            tf.linspace(0.0, tf.to_float(phases - 1), phases))
    )) + 1.0) * 128)
    return tf.concat(2, [val, val, val])