def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None):
    '''
    LSTM计算的核心,首先得注意参数state_below，这是个3D矩阵，[n_Step，BatchSize，Emb_Dim] [句子数，[单词batch数，词向量维度] ]
    '''
    nsteps = state_below.shape[0] # 最高维
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]  # 取出单词batch数
    else:
        n_samples = 1

    assert mask is not None

    def _slice(_x, n, dim):

        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        return _x[:, n * dim:(n + 1) * dim]

    def _step(m_, x_, h_, c_):
        # x_是形参，下面的state_below是实参
        # _step中的四个形参： x_是state_below降为二维矩阵形成的序列，m_是mask降为1维“行”向量的序列
        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])  # 每次新的h_与lstm_U矩阵相乘，使得f、o、c均不再为零，其中f、o是中间变量
        preact += x_  # 2维矩阵序列(x_) + 2维矩阵 = 2维矩阵序列

        # 每一个preact矩阵序列 的4块并列子矩阵 进行切片分块运算
        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))  # 拿出了preact中的input GATE相关项做sigmoid激活
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))  # forget GATE
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))  # output GATE
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))  # cell

        c = f * c_ + i * c
        # [:,None] 表示“行数组”升一维变为“列向量”
        c = m_[:, None] * c + (1. - m_)[:, None] * c_ #c_代表初始时刻或上一时刻

        # 每个Step里，h结果是一个2D矩阵，[BatchSize，Emb_Dim]
        h = o * tensor.tanh(c)  # 相当于octave中的.*
        h = m_[:, None] * h + (1. - m_)[:, None] * h_  # h_是h的上一时刻或toutputs_info中的初始时刻的值

        return h, c  # 输出值对应着outputs_info中的元素
        # scan函数最终的return时，2维矩阵序列还原为三维矩阵，向量序列还原为2维矩阵，即scan函数的输出结果会增加1-D

    state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) +
                   tparams[_p(prefix, 'b')])  # 3维矩阵 乘 2维矩阵仍是3维矩阵，无须每一个Step做一次Wx+b，而是把所有Step的Wx一次性预计算好了

    dim_proj = options['dim_proj']
    # scan函数的一旦sequence不为空，就进入序列循环模式
    # theano.scan中的sequences中的张量会被降低一维成为在迭代函数中使用，原最高维的维数作为sequences的迭代的次数,out
    # 在scan函数的Sequence里，每步循环，都会降解n_Step维，得到一个Emb矩阵，作为输入X_
    rval, updates = theano.scan(_step,
                                sequences=[mask, state_below],  # mask对应m_，state_below对应x_，迭代次数由sequences的序列个数决定
                                outputs_info=[tensor.alloc(numpy_floatX(0.),
                                                           n_samples, # n_samples是句子中单词batch个数(即LSTM时序上输入的个数)
                                                           dim_proj), # 这个张量是outputs[-1]时刻所初始化的值，在第一次loop之后（outputs[0]之后）将会被覆盖，覆盖后此处对应着_step中的h_(h的前一时刻)
                                              tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj) # 第1次loop后此张量被覆盖，此处对应c_(c的前一时刻)
                                              ],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps)
    return rval[0]  # rval[0]是h  rval[1]是c  它们都是tensor.shared类型

def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None):
    nsteps = state_below.shape[0]
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = 1

    assert mask is not None

    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        return _x[:, n * dim:(n + 1) * dim]

    def _step(m_, x_, h_, c_):
        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])
        preact += x_

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

        if has_input_gate:
            if has_forget_gate:
                c = f * c_ + i * c
            else:
                c = c_ + i*c
        else:
            if has_forget_gate:
                c = f*c_ + c
            else:
                c = c_ + c

        c = m_[:, None] * c + (1. - m_)[:, None] * c_

        if has_output_gate:
            h = o * tensor.tanh(c)
        else:
            h = tensor.tanh(c)

        h = m_[:, None] * h + (1. - m_)[:, None] * h_

        return h, c

    state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) +
                   tparams[_p(prefix, 'b')])

    dim_proj = options['dim_proj']
    rval, updates = theano.scan(_step,
                                sequences=[mask, state_below],
                                outputs_info=[tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj),
                                              tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj)],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps)
    return rval[0]

    def apply(self, x):
        W, U, b = self.params
        ndim = self.ndim

        def _slice(x, n, dim):
            return x[:, n * dim:(n + 1) * dim]

        def _step(x_t, h_t, c_t):
            preact = T.dot(h_t, U) + x_t

            i = T.nnet.sigmoid(_slice(preact, 0, self.ndim))
            f = T.nnet.sigmoid(_slice(preact, 1, self.ndim))
            o = T.nnet.sigmoid(_slice(preact, 2, self.ndim))
            c = T.tanh(_slice(preact, 3, self.ndim))

            c = f * c_t + i * c
            h = o * T.tanh(c)

            return h, c

        state_below = T.dot(x, W) + b

        rval, _ = theano.scan(
                _step, [state_below],
                outputs_info = [T.alloc(numpy.float32(0.), x.shape[1], ndim),
                                T.alloc(numpy.float32(0.), x.shape[1], ndim)],
                profile = _doProfile)

        return rval[0]

def crop_images(data, image_shape, border_width=8, mode=0):
    """ Function used to crop the images by a certain border width.
        data         : input data, theano 4D tensor
        image_shape  : 4-tuple, (batch_size, num_channels, image_rows, image_cols)
        border_width : border width to be cropped, default value 8
        mode         : binary, 0 for random, 1 for centered crop.
    """
    if (mode == 0):
        row_step = image_shape[2] - border_width
        col_step = image_shape[3] - border_width
        output = T.alloc(0., image_shape[0], image_shape[1], row_step, col_step)
        for i in range(image_shape[0]):           
            begin_idx = numpy.random.randint(border_width)
            output = T.set_subtensor(output[i,:,:,:], 
                data[i,:,begin_idx:(begin_idx+row_step),begin_idx:(begin_idx+col_step)])
        return output
    else: 
        row_step = image_shape[2] - border_width
        col_step = image_shape[3] - border_width
        output = T.alloc(0., image_shape[0], image_shape[1], row_step, col_step)
        for i in range(image_shape[0]):           
            begin_idx = border_width / 2 
            output = T.set_subtensor(output[i,:,:,:], 
                data[i,:,begin_idx:(begin_idx+row_step),begin_idx:(begin_idx+col_step)])
        return output

    def output_probabilistic(self, m_w_previous, v_w_previous):
        if (self.non_linear):
            m_in = self.m_w - m_w_previous
            v_in = self.v_w
            # We compute the mean and variance after the ReLU activation
            lam = self.lam
            v_1 = 1 + 2*lam*v_in
            v_1_inv = v_1**-1

            s_1 = T.prod(v_1,axis=1)**-0.5
            v_2 = 1 + 4*lam*v_in
            v_2_inv = v_2**-1
            s_2 = T.prod(v_2,axis=1)**-0.5
            v_inv = v_in**-1
            exponent1 = m_in**2*(1 - v_1_inv)*v_inv
            exponent1 = T.sum(exponent1,axis=1)
            exponent2 = m_in**2*(1 - v_2_inv)*v_inv
            exponent2 = T.sum(exponent2,axis=1)
            m_a = s_1*T.exp(-0.5*exponent1)
            v_a = s_2*T.exp(-0.5*exponent2) - m_a**2

            return (m_a, v_a)

        else:
            m_w_previous_with_bias = \
            T.concatenate([ m_w_previous, T.alloc(1, 1) ], 0)
            v_w_previous_with_bias = \
            T.concatenate([ v_w_previous, T.alloc(0, 1) ], 0)

            m_linear = T.dot(self.m_w, m_w_previous_with_bias) / T.sqrt(self.n_inputs)
            v_linear = (T.dot(self.v_w, v_w_previous_with_bias) + \
                T.dot(self.m_w**2, v_w_previous_with_bias) + \
                T.dot(self.v_w, m_w_previous_with_bias**2)) / self.n_inputs
            return (m_linear, v_linear)

def gru_layer(tparams, state_below, init_state, options, prefix='gru', mask=None, **kwargs):
    """
    Feedforward pass through GRU
    """
    nsteps = state_below.shape[0]
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = 1

    dim = tparams[_p(prefix,'Ux')].shape[1]

    if init_state == None:
        init_state = tensor.alloc(0., n_samples, dim)

    if mask == None:
        mask = tensor.alloc(1., state_below.shape[0], 1)

    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n*dim:(n+1)*dim]
        return _x[:, n*dim:(n+1)*dim]

    state_below_ = tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]
    state_belowx = tensor.dot(state_below, tparams[_p(prefix, 'Wx')]) + tparams[_p(prefix, 'bx')]
    U = tparams[_p(prefix, 'U')]
    Ux = tparams[_p(prefix, 'Ux')]

    def _step_slice(m_, x_, xx_, h_, U, Ux):
        preact = tensor.dot(h_, U)
        preact += x_

        r = tensor.nnet.sigmoid(_slice(preact, 0, dim))
        u = tensor.nnet.sigmoid(_slice(preact, 1, dim))

        preactx = tensor.dot(h_, Ux)
        preactx = preactx * r
        preactx = preactx + xx_

        h = tensor.tanh(preactx)

        h = u * h_ + (1. - u) * h
        h = m_[:,None] * h + (1. - m_)[:,None] * h_

        return h

    seqs = [mask, state_below_, state_belowx]
    _step = _step_slice

    rval, updates = theano.scan(_step,
                                sequences=seqs,
                                outputs_info = [init_state],
                                non_sequences = [tparams[_p(prefix, 'U')],
                                                 tparams[_p(prefix, 'Ux')]],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps,
                                profile=False,
                                strict=True)
    rval = [rval]
    return rval

def RNN_layer(tparams,inputs,mask=None,init_h=None,prefix=None,name='rnn',std=True):
	"""
	inputs: n_steps*n_samples*x_size
	return h
	"""
	prefix=GetPrefix(prefix,name);
	# if length!=None: inputs=inputs[index:index+length,:,:];
	n_steps=inputs.shape[0];
	n_samples=inputs.shape[1];
	x_size=inputs.shape[2];

	hdim=tparams[_p(prefix,'wh')].shape[0];

	if mask == None:		
		mask = T.alloc(1., n_steps, n_samples);
	if init_h == None:       
		init_h = T.alloc(0., n_samples, hdim);


	def _step(m,x,h):
		inputs_h=( T.dot(x,tparams[_p(prefix,'wx')])+T.dot(h,tparams[_p(prefix,'wh')]) )/2+tparams[_p(prefix,'b')];
		_h=tanh(inputs_h);
		return _h;

	if std:	inputs=standardize(inputs);
	out,updates=theano.scan(lambda m,x,h:_step(m,x,h), 
							sequences=[mask,inputs], 
							outputs_info=[init_h],
							name=_p(prefix,'scan'),
							n_steps=n_steps,
							# truncate_gradient=10,
							profile=False);
	return out

    def encode(self, state_below):
        """
		:development:
			(1) may need to prepend encoding_length * padding array to the state_below to produce the same length sequence as state_below
			(2) can return an offset encoding by only returing certain indices of the encoding (though this is pretty wasteful)

		:type state_below: 2d tensor
		:param state_below: the enitre sequence of states from the layer below the current one

		:type rval: 2d tensor
		:param rval: an encoding of the state_below (the entire sequence of state) to be passed to the above layer
		"""
        # to make the encodings start with the first state in state_below, prepend encoding_length vectors of value zero
        zeros = T.alloc(np.cast[theano.config.floatX](0), self.encoding_length - 1, self.n_hid)
        state_below = T.concatenate((zeros, state_below))

        encoding_0 = T.alloc(np.cast[theano.config.floatX](0), self.n_hid)
        # negative, reverse indicies for the taps
        # e.g., [-4, -3, -2, -1, -0] would pass those indicies from state_below to the encode_step
        taps = [-1 * tap for tap in range(self.encoding_length)[::-1]]
        encodings, updates = scan(
            fn=self.encode_subsequence, sequences=dict(input=state_below, taps=taps), outputs_info=[encoding_0]
        )

        return encodings

    def generate_lstm(self, context):
        x0 = T.alloc(0., context.shape[0], self.embedding_dim)
        h0 = T.alloc(0., context.shape[0], self.hidden_dim)
        c0 = T.alloc(0., context.shape[0], self.hidden_dim)

        def _step(x_tm_1, h_tm_1, c_tm_1):
            lstm_preactive = T.dot(h_tm_1, self.decode_U)+ \
                             T.dot(context, self.decode_V)+ \
                             T.dot(x_tm_1, self.decode_W) + \
                             self.decode_b

            i = T.nnet.sigmoid(lstm_preactive[:,0:self.hidden_dim])
            f = T.nnet.sigmoid(lstm_preactive[:,self.hidden_dim:self.hidden_dim*2])
            o = T.nnet.sigmoid(lstm_preactive[:,self.hidden_dim*2:self.hidden_dim*3])
            c = T.tanh(lstm_preactive[:,self.hidden_dim*3:self.hidden_dim*4])

            c = f*c_tm_1 + i*c
            h = o*T.tanh(c)

            x_emb = T.dot(h, self.output_W) + self.output_b # (n_samples, embedding_dim)
            x_word = T.dot(x_emb, self.word_W) + self.word_b # (n_samples, n_words)

            x_index = T.argmax(x_word, axis=1)
            x = self.emb[x_index]

            return [x,h,c]

        rval, updates = theano.scan(
                fn=_step,
                outputs_info=[x0,h0,c0],
                n_steps=20)

        generated_sequence = rval[0]
        return generated_sequence

def arc_distance_theano_alloc_prepare(dtype='float64'):
    """
    Calculates the pairwise arc distance between all points in vector a and b.
    """
    a = tensor.matrix(dtype=str(dtype))
    b = tensor.matrix(dtype=str(dtype))
    # Theano don't implement all case of tile, so we do the equivalent with alloc.
    #theta_1 = tensor.tile(a[:, 0], (b.shape[0], 1)).T
    theta_1 = tensor.alloc(a[:, 0], b.shape[0], b.shape[0]).T
    phi_1 = tensor.alloc(a[:, 1], b.shape[0], b.shape[0]).T

    theta_2 = tensor.alloc(b[:, 0], a.shape[0], a.shape[0])
    phi_2 = tensor.alloc(b[:, 1], a.shape[0], a.shape[0])

    temp = (tensor.sin((theta_2 - theta_1) / 2)**2
            +
            tensor.cos(theta_1) * tensor.cos(theta_2)
            * tensor.sin((phi_2 - phi_1) / 2)**2)
    distance_matrix = 2 * (tensor.arctan2(tensor.sqrt(temp),
                                          tensor.sqrt(1 - temp)))
    name = "arc_distance_theano_alloc"
    rval = theano.function([a, b],
                           distance_matrix,
                           name=name)
    rval.__name__ = name

    return rval

    def sample(self, x0=None, h0=None, c0=None, n_samples=10, n_steps=10,
               condition_on=None, debug=False):
        if x0 is None:
            x0, _ = self.output_net.sample(
                p=T.constant(0.5).astype(floatX),
                size=(n_samples, self.output_net.dim_out)).astype(floatX)

        if h0 is None:
            h0 = T.alloc(0., x0.shape[0], self.dim_h).astype(floatX)
        if c0 is None:
            c0 = T.alloc(0., x0.shape[0], self.dim_h).astype(floatX)
        z0 = self.output_net.preact(h0)

        seqs = []
        outputs_info = [h0, c0, x0, None]
        non_seqs = []
        step = self.step_sample
        p0 = self.output_net.distribution(z0)

        non_seqs += self.get_sample_params()
        if debug:
            return self.step_sample(h0, x0, *self.get_sample_params())

        outs = scan(step, seqs, outputs_info, non_seqs, n_steps,
                    name=self.name+'_sampling', strict=False)
        (h, c, x, p), updates = outs

        x = concatenate([x0[None, :, :], x])
        h = concatenate([h0[None, :, :], h])
        p = concatenate([p0[None, :, :], p])

        return OrderedDict(x=x, p=p, h=h, x0=x0, p0=p0, h0=h0), updates

    def decode(self, hidden):
        hidden_ = T.alloc(0.,*self.hidden_shape)
        deconv_out = T.alloc(0.,*self.output_shape)
       
        # Zero padding How can I code easily?
        hidden_ = T.set_subtensor(hidden_[:,:,:,self.filter_shape[3]-1:],hidden)

        # Calculate output
        conv_odd = conv.conv2d(
            input = hidden_,
            filters = self.W_odd,
            filter_shape = self.filter_shape,
            image_shape = self.hidden_shape,)
        conv_even = conv.conv2d(
            input = hidden_,
            filters = self.W_even,
            filter_shape = self.filter_shape,
            image_shape = self.hidden_shape,)
        
        deconv_out = T.set_subtensor(deconv_out[:,:,:,::2], conv_odd)
        deconv_out = T.set_subtensor(deconv_out[:,:,:,1::2], conv_even)

        linout = deconv_out + self.b.dimshuffle('x',0,'x','x')
        
        if self.dec_hid == 'tanh':
            convout= T.tanh(linout)
        elif self.dec_hid == 'lin':
            convout=linout
        elif self.dec_hid == 'relu':
            convout=linout * (linout > 0.) + 0. * (linout < 0.)
        else:
            raise ValueError('Invalid dec_hid')
        #### Recurrent connection####
        return convout

  def __call__(self, input):
    nh = self.hidden_size
    # _in: input of t
    # _m : output of t - 1
    # _c : memory of t - 1 
    def _step(_in, _m, _c, nh):
      _x = tensor.concatenate([numpy.asarray([1.], dtype=numpy.float32), _in, _m])
      ifog = tensor.dot(_x, self.W)

      i = tensor.nnet.sigmoid(ifog[ : nh])
      f = tensor.nnet.sigmoid(ifog[nh : 2*nh])
      o = tensor.nnet.sigmoid(ifog[2*nh : 3*nh])
      g = tensor.tanh(ifog[3*nh : ])

      _c = f * _c + i * g
      _m = o * _c
      return _m, _c
    self._step = _step

    results, update = theano.scan(
        _step, 
        sequences=[input],
        outputs_info=[tensor.alloc(0.0, nh), tensor.alloc(0.0, nh)],
        non_sequences=[self.hidden_size]
      )
    return results[0] #(_m_list, _c_list)[0]

def ENCODER_R(X, tparams, options):

    # (tensor.alloc(numpy_floatX(1.), options['hidden_size'], 1)-tensor.nnet.sigmoid(tensor.dot(tparams['Wr_Z'], xr) + tensor.dot(tparams['Ur_Z'], hr_tm1))) * hr_tm1\
    #  + tensor.nnet.sigmoid(tensor.dot(tparams['Wr_Z'], xr) + tensor.dot(tparams['Ur_Z'], hr_tm1)) * tensor.tanh(\
    #     tensor.dot(tparams['Wr'], xr) + tensor.dot(tparams['Ur'],  \
    #         (tensor.nnet.sigmoid(tensor.dot(tparams['Wr_R'], xr) + \
    #             tensor.dot(tparams['Ur_R'], hr_tm1)) * hr_tm1)\
    #         )\
    #     )

    # (tensor.alloc(numpy_floatX(1.), options['hidden_size'])-tensor.nnet.sigmoid(tensor.dot\
    # (tparams["Emb"][xr], tparams['Wr_Z']) + tensor.dot(hr_tm1, tparams['Ur_Z']))) * hr_tm1\
    #  + tensor.nnet.sigmoid(tensor.dot(tparams["Emb"][xr], tparams['Wr_Z']) + tensor.dot(hr_tm1, \
    # tparams['Ur_Z'])) *  tensor.tanh(tensor.dot(tparams["Emb"][xr], tparams['Wr']) + \
    #     tensor.dot((tensor.nnet.sigmoid(tensor.dot(tparams["Emb"][xr], tparams['Wr_R']) + tensor\
    # .dot(hr_tm1, tparams['Ur_R'])) * hr_tm1) , tparams['Ur']))\

    # tparams["Emb"][xr]
    # X_Vec = word2VecLayer(X, tparams)

    results_r, updates = theano.scan(lambda xr, hr_tm1:    (tensor.alloc(numpy_floatX(1.), options['hidden_size'])\
        -tensor.nnet.sigmoid(tensor.dot(tparams["Emb"][xr], tparams['Wr_Z']) + tensor.dot(hr_tm1, tparams['Ur_Z']))) * hr_tm1\
     + tensor.nnet.sigmoid(tensor.dot(tparams["Emb"][xr], tparams['Wr_Z']) + tensor.dot(hr_tm1, \
        tparams['Ur_Z'])) *  tensor.tanh(tensor.dot(tparams["Emb"][xr], tparams['Wr']) + \
        tensor.dot((tensor.nnet.sigmoid(tensor.dot(tparams["Emb"][xr], tparams['Wr_R']) + tensor.\
            dot(hr_tm1, tparams['Ur_R'])) * hr_tm1) , tparams['Ur']))\
     ,  sequences=[X], outputs_info=tensor.alloc(numpy_floatX(0.), options['hidden_size']))
    #initial value of the scan can only be vec

    return results_r # [hi_right]  # return[ (n,) *l ] that is [(1*n) * l]

	def encode(self, state_below):
		"""
		:development:
			(1) may need to prepend encoding_length * padding array to the state_below to produce the same length sequence as state_below
			(2) can return an offset encoding by only returing certain indices of the encoding (though this is pretty wasteful)

		:type state_below: 2d tensor
		:param state_below: the enitre sequence of states from the layer below the current one

		:type rval: 2d tensor
		:param rval: an encoding of the state_below (the entire sequence of state) to be passed to the above layer
		"""

		total_sequence_length = T.cast(state_below.shape[0], theano.config.floatX)
		self.n_encodings = T.cast(T.ceil(total_sequence_length / self.encoding_length), 'int32')
		self.n_padding_timesteps = T.cast(self.n_encodings * self.encoding_length - total_sequence_length, 'int32')
		zeros = T.alloc(np.cast[theano.config.floatX](0), self.n_padding_timesteps, self.n_vis)
		state_below = T.concatenate((zeros, state_below))

		Wxh = self.Wxh
		bxh = self.bxh
		Whhe = self.Whhe

		state_below = state_below.reshape((self.encoding_length, self.n_encodings, self.n_vis))
		state_below = T.dot(state_below, Wxh) + bxh
		
		# a single output will be n_encoding rows with n_hid features each
		encoding_0 = T.alloc(np.cast[theano.config.floatX](0), self.n_encodings, self.n_hid)

		encodings, updates = scan(fn=self.encode_step, sequences=[state_below], outputs_info=[encoding_0], non_sequences=[Whhe])
		# encodings is a 3d vector (encoding_length, n_encodings, n_hid)
		# returns encodings[-1] in 2d vector shape = (n_encodings, n_hid)
		return encodings[-1]

    def encode_lstm(self, x, mask):
        def _step(m_tm_1, x_t, h_tm_1, c_tm_1):
            lstm_preactive = T.dot(h_tm_1, self.encode_U) + \
                             T.dot(x_t, self.encode_W) + \
                             self.encode_b

            i = T.nnet.sigmoid(lstm_preactive[:,0:self.hidden_dim])
            f = T.nnet.sigmoid(lstm_preactive[:,self.hidden_dim:self.hidden_dim*2])
            o = T.nnet.sigmoid(lstm_preactive[:,self.hidden_dim*2:self.hidden_dim*3])
            c = T.tanh(lstm_preactive[:,self.hidden_dim*3:self.hidden_dim*4])

            c = f*c_tm_1 + i*c
            c = m_tm_1[:,None]*c + (1.-m_tm_1)[:,None]*c_tm_1

            h = o*T.tanh(c)
            h = m_tm_1[:,None]*h + (1.-m_tm_1)[:,None]*h_tm_1

            return [h,c]

        h0 = T.alloc(0., x.shape[1], self.hidden_dim)
        c0 = T.alloc(0., x.shape[1], self.hidden_dim)

        rval, updates = theano.scan(
                fn=_step,
                sequences=[mask,x],
                outputs_info=[h0,c0]
                )

        h_list, c_list = rval
        return h_list

    def __init__(self, cell, rng, layer_id, shape, X, mask, is_train = 1, batch_size = 1, p = 0.5):
        prefix = "SentDecoderLayer_"
        layer_id = "_" + layer_id
        self.in_size, self.out_size = shape
        self.X = X
        self.summs = batch_size
        
        self.W_hy = init_weights((self.in_size, self.out_size), prefix + "W_hy" + layer_id)
        self.b_y = init_bias(self.out_size, prefix + "b_y" + layer_id)

        if cell == "gru":
            self.decoder = GRULayer(rng, prefix + layer_id, shape, self.X, mask, is_train, 1, p)
            def _active(pre_h, x):
                h = self.decoder._active(x, pre_h)
                y = T.tanh(T.dot(h, self.W_hy) + self.b_y)
                return h, y
            [h, y], updates = theano.scan(_active, n_steps = self.summs, sequences = [],
                                      outputs_info = [{'initial':self.X, 'taps':[-1]},
                                                      T.alloc(floatX(0.), 1, self.out_size)])
        elif cell == "lstm":
            self.decoder = LSTMLayer(rng, prefix + layer_id, shape, self.X, mask, is_train, 1, p)
            def _active(pre_h, pre_c, x):
                h, c = self.decoder._active(x, pre_h, pre_c)
                y = T.tanh(T.dot(h, self.W_hy) + self.b_y)
                return h, c, y
            [h, c, y], updates = theano.scan(_active, n_steps = self.summs, sequences = [],
                                             outputs_info = [{'initial':self.X, 'taps':[-1]},
                                                             {'initial':self.X, 'taps':[-1]},
                                                             T.alloc(floatX(0.), 1, self.out_size)])
       
        y = T.reshape(y, (self.summs, self.out_size))
        self.activation = y

        self.params = self.decoder.params + [self.W_hy, self.b_y]

    def __init__(self, input_var, 
                layerid,sequence,n_input_channels=1,
                height=3,width=3,n_filters=8):
        
        X = input_var
        imH, imW = X.shape[-2],X.shape[-1]

        H, W, F, C = height, width, n_filters, n_input_channels
        Tt, N = input_var.shape[0],input_var.shape[1]
        self.n_filters = n_filters


        self.Wx = shared(self.glorot_init(H*W*F,4*F,C,H,W), name='Wx'+layerid)
        self.Wh = shared(self.glorot_init(4*H*W*F,4*F,F,H,W),name='Wh'+layerid)
        self.b = shared(np.zeros(4*F,dtype=np.float32),name='b'+layerid)

        self.params = {
            self.Wx.name: self.Wx,
            self.Wh.name: self.Wh,
            self.b.name: self.b
            }

        [h,c], _ = scan(self.step,
            sequences=[X],
            outputs_info=[
                        T.alloc(np.cast['float32'](0), N,F,imH,imW),
                        T.alloc(np.cast['float32'](0), N,F,imH,imW)
                        ])

        self.output = h

    def build(self, antialias_samples=4):

        # returns top-level render function and associated variables
        image = T.alloc(0., self.camera.x_dims, self.camera.y_dims, 3)

        #Anti-Aliasing
        sampleDist_x = np.asarray(np.random.random((self.camera.x_dims, self.camera.y_dims,antialias_samples)),dtype=theano.config.floatX)
        sampleDist_y = np.asarray(np.random.random((self.camera.x_dims, self.camera.y_dims,antialias_samples)),dtype=theano.config.floatX)

        for sample in xrange(antialias_samples): #TODO USE SCAN

            #Make Rays
            self.camera.rays = self.camera.make_rays(self.camera.x_dims, self.camera.y_dims,\
                            sampleDist_x=(sampleDist_x[:,:,sample] + sample)/antialias_samples,
                            sampleDist_y=(sampleDist_y[:,:,sample] + sample)/antialias_samples)
            #self.camera.variables.add_child(self.camera.rays.variables)
            image_per_sample = T.alloc(0.0, self.camera.x_dims, self.camera.y_dims, 3)
            min_dists = T.alloc(float('inf'), self.camera.x_dims, self.camera.y_dims)

            # for each shape find its shadings and draw closer shapes on top
            for shape in self.shapes:
                dists = shape.distance(self.camera.rays)
                shadings = self.shader.shade(shape, self.lights, self.camera)
                #for each shape != obj, draw shadow of shape on obj
                #for obj2 in self.shapes:
                #    if obj == obj2: continue
                #    shadings = broadcasted_switch(obj2.shadow(
                #        obj.surface_pts(self.camera.rays), self.lights) < 0, shadings, [0., 0., 0.])
                image_per_sample = broadcasted_switch(dists < min_dists, shadings, image_per_sample)
                min_dists = T.switch(dists < min_dists, dists, min_dists)

            image = image + image_per_sample
        image = image / antialias_samples

        return image

    def fprop(self, data):
        if self.use_ground_truth:
            self.input_space.validate(data)
            features, phones = data

            init_h = T.alloc(numpy.cast[theano.config.floatX](0), self.nhid)
            init_out = T.alloc(numpy.cast[theano.config.floatX](0), 1)
            init_out = T.unbroadcast(init_out, 0)

            fn = lambda f, p, h, o: self.fprop_step(f, p, h, o)

            ((h, out), updates) = theano.scan(fn=fn,
                                              sequences=[features, phones],
                                              outputs_info=[dict(initial=init_h,
                                                                 taps=[-1]),
                                                            init_out])
            return out
        else:
            self.input_space.validate(data)
            features, phones = data

            init_in = features[0]
            init_h = T.alloc(numpy.cast[theano.config.floatX](0), self.nhid)
            init_out = T.alloc(numpy.cast[theano.config.floatX](0), 1)
            init_out = T.unbroadcast(init_out, 0)

            fn = lambda t, p, f, h, o: self.fprop_step_prime(t, p, f, h, o)

            ((f, h, out), updates) = theano.scan(fn=fn,
                                                 sequences=[features, phones],
                                                 outputs_info=[init_in,
                                                               dict(initial=init_h,
                                                                    taps=[-1]),
                                                               init_out])
            return out