def __init__(self, input_dim, proj_dim=128, neg_samples = 4,
                 init='uniform', activation='tanh', weights=None,W_regularizer = None, activity_regularizer=None,  **kwargs):

        super(WordTagContextProduct_tensor, self).__init__(**kwargs)
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.samples = neg_samples + 1
        #np.random.seed(0)
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.W_regularizer = regularizers.get(W_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)

        #self.input = T.imatrix()
        #self.input = T.itensor3()
        self.input = [T.itensor3(), T.itensor3()]
        # two different embeddings for pivot word and its context
        # because p(w|c) != p(c|w)
        self.W_w = self.init((input_dim, proj_dim))
        self.W_c = self.init((input_dim, proj_dim))
        

        self.params = [self.W_w, self.W_c]

        if weights is not None:
            self.set_weights(weights)

 def _get_input_tensor_variables(self):
     # x_w: 1D: batch, 2D: n_prds, 3D: n_words, 4D: 5 + window; elem=word id
     # x_p: 1D: batch, 2D: n_prds, 3D: n_words; elem=posit id
     # y: 1D: batch, 2D: n_prds, 3D: n_words; elem=label id
     if self.argv.mark_phi:
         return [T.itensor4('x_w'), T.itensor3('x_p'), T.itensor3('y')]
     return [T.itensor4('x_w'), T.itensor3('y')]

 def make_node(self, x, x2, x3, x4, x5):
     # check that the theano version has support for __props__.
     # This next line looks like it has a typo,
     # but it's actually a way to detect the theano version
     # is sufficiently recent to support the use of __props__.
     assert hasattr(self, '_props'), "Your version of theano is too old to support __props__."
     x = tensor.as_tensor_variable(x)
     x2 = tensor.as_tensor_variable(x2)
     x3 = tensor.as_tensor_variable(x3)
     x4 = tensor.as_tensor_variable(x4)
     x5 = tensor.as_tensor_variable(x5)
     
     if prm.att_doc:
         if prm.compute_emb:
             td = tensor.itensor4().type()
         else:
             td = tensor.ftensor4().type()
         tm = tensor.ftensor3().type()
     else:
         if prm.compute_emb:
             td = tensor.itensor3().type()
         else:
             td = tensor.ftensor3().type()
         tm = tensor.fmatrix().type()
     return theano.Apply(self, [x,x2,x3,x4,x5], [td, tm, \
                                        tensor.fmatrix().type(), tensor.ivector().type()])

def main():
    xs = itensor3('xs')
    ins = ((None, None, 93), xs)
    gru = GRU(
        inputs=ins,
        hiddens=128,
        direction='bidirectional'
    )
    print("GRU output (hiddens) shape: ", gru.output_size)
    print("GRU params: ", gru.get_params())

    lstm = LSTM(
        inputs=ins,
        hiddens=128,
        direction='bidirectional'
    )
    print("LSTM output (hiddens) shape: ", lstm.output_size)
    print("LSTM params: ", lstm.get_params())

    rnn = RNN(
        inputs=ins,
        hiddens=128,
        direction='bidirectional'
    )
    print("RNN output (hiddens) shape: ", rnn.output_size)
    print("RNN params: ", rnn.get_params())

 def __init__(self, batch_size, emb_X, num_words, lstm_params, conv_param, output_size, f1_classes):
     super().__init__(batch_size)
     self.num_words = num_words
     self.inputs = [T.itensor3('input'), T.tensor3('mask')]
     self.target = T.ivector('target')
     l = InputLayer((batch_size, num_words, None), self.inputs[0])
     l_mask = InputLayer((batch_size, num_words, None), self.inputs[1])
     l = ReshapeLayer(l, (-1, [2]))
     l_mask = ReshapeLayer(l_mask, (-1, [2]))
     l = EmbeddingLayer(l, emb_X.shape[0], emb_X.shape[1], W=emb_X)
     for lstm_param in lstm_params:
         l = LSTMLayer(
             l, lstm_param, grad_clipping=100, nonlinearity=tanh, mask_input=l_mask, only_return_final=True
         )
     l = ReshapeLayer(l, (batch_size, num_words, -1))
     l_convs = []
     for filter_size in conv_param[1]:
         l_cur = Conv1DLayer(l, conv_param[0], filter_size, pad='full', nonlinearity=rectify)
         l_cur = MaxPool1DLayer(l_cur, num_words + filter_size - 1, ignore_border=True)
         l_cur = FlattenLayer(l_cur)
         l_convs.append(l_cur)
     l = ConcatLayer(l_convs)
     l = DropoutLayer(l)
     l = DenseLayer(l, output_size, nonlinearity=log_softmax)
     self.constraints[l.W] = lambda u, v: norm_constraint(v, 3)
     self.pred = T.exp(get_output(l, deterministic=True))
     self.loss = T.mean(categorical_crossentropy_exp(self.target, get_output(l)))
     params = get_all_params(l, trainable=True)
     self.updates = adadelta(self.loss, params)
     self.metrics = {'train': [acc], 'val': [acc, f1(f1_classes)]}
     self.network = l
     self.compile()

def add_datasets_to_graph(list_of_datasets, list_of_names, graph, strict=True,
                          list_of_test_values=None):
    assert len(list_of_datasets) == len(list_of_names)
    datasets_added = []
    for n, (dataset, name) in enumerate(zip(list_of_datasets, list_of_names)):
        if dataset.dtype != "int32":
            if len(dataset.shape) == 1:
                sym = tensor.vector()
            elif len(dataset.shape) == 2:
                sym = tensor.matrix()
            elif len(dataset.shape) == 3:
                sym = tensor.tensor3()
            else:
                raise ValueError("dataset %s has unsupported shape" % name)
        elif dataset.dtype == "int32":
            if len(dataset.shape) == 1:
                sym = tensor.ivector()
            elif len(dataset.shape) == 2:
                sym = tensor.imatrix()
            elif len(dataset.shape) == 3:
                sym = tensor.itensor3()
            else:
                raise ValueError("dataset %s has unsupported shape" % name)
        else:
            raise ValueError("dataset %s has unsupported dtype %s" % (
                name, dataset.dtype))
        if list_of_test_values is not None:
            sym.tag.test_value = list_of_test_values[n]
        tag_expression(sym, name, dataset.shape)
        datasets_added.append(sym)
    graph["__datasets_added__"] = datasets_added
    return datasets_added

    def testSplitOutputByFilter(self):
        self.setSeeds()

        input_shape = (self.batch_size, self.max_seq_len,
                self.n_filters * self.filter_width)
        output_shape = (self.batch_size, self.n_filters,
                self.max_seq_len, self.filter_width)

        x = np.arange(np.prod(input_shape))
        x = x.reshape(input_shape).astype(np.int32)
        y = np.zeros_like(x)
        y = np.reshape(y, output_shape)

        for i in range(self.n_filters):
            s = x[:, :, i*self.filter_width:(i+1)*self.filter_width]
            y[:, i, :, :] = s

        xt = T.itensor3('xt')
        layer = SplitOutputByFilter(self.n_filters, self.filter_width)
        yt = layer._get_output(xt)

        f = theano.function(inputs=[xt], outputs=yt)
        y_theano = f(x)

        self.assertEquals(y.shape, y_theano.shape)
        self.assertTrue(np.all(y == y_theano))

    def __init__(self, input_dim, proj_dim=128, neg_samples = 4,
                 init='uniform', activation='sigmoid', weights=None, W_regularizer = None, activity_regularizer = None, **kwargs):

        super(WordTagContextProduct, self).__init__(**kwargs)
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.samples = neg_samples + 1
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.W_regularizer = regularizers.get(W_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.input = [T.itensor3(), T.itensor3()]
        self.W_w = self.init((input_dim, proj_dim))
        self.params = [self.W_w]
        if weights is not None:
            self.set_weights(weights)

    def __init__(self,batch_size=16, seed=1234,nhu=300,width=5,n_out=len(nerarray),activation_f="hardtanh",
                 embeddingfile=senna_embmtxfile,trainingfile=trainingfile,paramfile=None):
        modeldir=os.path.join(nerdir,"models",'model_%i'%(len(os.listdir(nerdir+"/models"))))
        os.mkdir(modeldir)   
        for handler in logging.root.handlers[:]:
            logging.root.removeHandler(handler)
        logging.basicConfig(filename=os.path.join(modeldir,'log.txt'), level=logging.INFO, 
                            format='%(asctime)s : %(levelname)s : %(message)s')    
        logger.info("\n"+"\n".join(["\t%s : "%key+str(val) for key,val in locals().iteritems() if key!="self"]))    
        self.modeldir=modeldir
        self.batch_size = batch_size
        activation=None        
        if activation_f=="hardtanh":
            activation=hardtanh
        elif activation_f=="tanh":
            activation=T.tanh
        self.load_data(embeddingfile,trainingfile,batch_size)
        #==============================================================================
        #         BUILD MODEL
        #==============================================================================
        logger.info('... building the model')
        # allocate symbolic variables for the data
        self.index = T.iscalar()  # index to a [mini]batch
        self.x = T.itensor3('x')  # the data is presented as matrix of integers
        self.y = T.ivector('y')  # the labels are presented as 1D vector of
                            # [int] labels
        self.permutation = T.ivector('permutation')
        if paramfile!=None:
            params=pickle.load(open(paramfile,"rb"))
        else:
            params=None
        self.model = SennaNER(input=self.x, embeddings=self.embeddings,features=capsfeatures,n_out=n_out, mini_batch_size=batch_size,
                                       nhu=nhu,width=width,activation=activation,seed=seed,params=params)

        self.test_model = theano.function(inputs=[self.index],
                outputs=self.model.errors(self.y),
                givens={
                    self.x: self.test_set_x[self.index * batch_size:(self.index + 1) * batch_size],
                    self.y: self.test_set_y[self.index * batch_size:(self.index + 1) * batch_size]},
                name="test_model")
    
        self.validation_cost = theano.function(inputs=[self.index],
                outputs=self.model.negative_log_likelihood(self.y),
                givens={
                    self.x: self.valid_set_x[self.index * batch_size:(self.index + 1) * batch_size],
                    self.y: self.valid_set_y[self.index * batch_size:(self.index + 1) * batch_size]},
                name="validation_cost")
    
        self.predictions = theano.function(inputs=[self.index],
                outputs=self.model.predictions,
                givens={
                    self.x: self.test_set_x[self.index * batch_size:(self.index + 1) * batch_size]},
                name="predictions")
    
        self.visualize_hidden = theano.function(inputs=[self.index],
                outputs=self.model.HiddenLayer.output,
                givens={
                    self.x: self.valid_set_x[self.index * batch_size:(self.index + 1) * batch_size]},
                name="visualize_hidden")

    def __init__(self,rng,model_params):
        self.input = T.itensor3('input') # the data is a minibatch
        self.label = T.imatrix('label') # label's shape (mini_batch size, max_term_per_sent)
        self.sent_length= T.ivector('sent_length') # sent_length is the number of terms in each sentence
        self.masks = T.imatrix('masks') # masks which used in error and likelihood calculation

        self.core = SentenceLevelNeuralModelCore(rng,self.input,self.label,self.sent_length,self.masks,model_params)

        self.params = self.core.wordvec.params() \
                + self.core.POSvec.params() \
                + self.core.wordpos_vec.params() \
                + self.core.verbpos_vec.params() \
                + self.core.conv_word.params() \
                + self.core.conv_POS.params() \
                + self.core.conv_wordpos.params() \
                + self.core.conv_verbpos.params() \
                + self.core.hidden_layer.params

        self.L2_sqr = (self.core.wordvec.embeddings ** 2).sum() \
                + (self.core.POSvec.embeddings ** 2).sum() \
                + (self.core.wordpos_vec.embeddings ** 2).sum() \
                + (self.core.verbpos_vec.embeddings ** 2).sum() \
                + (self.core.conv_word.W ** 2).sum() \
                + (self.core.conv_POS.W ** 2).sum() \
                + (self.core.conv_wordpos.W ** 2).sum() \
                + (self.core.conv_verbpos.W ** 2).sum() \
                + (self.core.hidden_layer.W ** 2).sum()

        self.negative_log_likelihood = self.core.likelihood()
        self.errors = self.core.errors()

        # we only use L2 regularization
        self.cost = self.negative_log_likelihood \
                + self.core.L2_reg * self.L2_sqr


        self.gparams = []
        for param in self.params:
            gparam = T.grad(self.cost, param)
            self.gparams.append(gparam)

        self.updates = []

        learning_rate = model_params['learning_rate']
        for param, gparam in zip(self.params, self.gparams):
            self.updates.append((param, param - learning_rate * gparam))


        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_word.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_POS.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_verbpos.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_wordpos.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_out,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.max_out,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.hidden_layer.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.negative_log_likelihood,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.cost,on_unused_input='ignore')
        self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.cost,updates=self.updates,on_unused_input='ignore')
        self.valid_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=[self.errors,self.core.sentce_loglikelihood.y_pred_pointwise],on_unused_input='ignore')

def tensor_max():
    e = np.asarray([[[2, 4], [5, 1]], [[3, 5], [4, 6]]], dtype='int32')
    w = T.itensor3('w')

    y = T.max(w, axis=1)
    f = theano.function(inputs=[w], outputs=y)

    print f(e)

def ndim_itensor(ndim, name=None):
    if ndim == 2:
        return T.imatrix(name)
    elif ndim == 3:
        return T.itensor3(name)
    elif ndim == 4:
        return T.itensor4(name)
    return T.imatrix(name=name)

    def __init__(self, nh, nc, ne, de, cs):
        '''
        nh ::隐藏层神经元个数
        nc ::输出层标签分类类别
        ne :: 单词的个数
        de :: 词向量的维度
        cs :: 上下文窗口
        '''
        #词向量实际为(ne, de)，外加1行，是为了边界标签-1而设定的
        self.emb = theano.shared(name='embeddings',value=0.2 * numpy.random.uniform(-1.0, 1.0,(ne+1, de)).astype(theano.config.floatX))#词向量空间
        self.wx = theano.shared(name='wx',value=0.2 * numpy.random.uniform(-1.0, 1.0,(de * cs, nh)).astype(theano.config.floatX))#输入数据到隐藏层的权重矩阵
        self.wh = theano.shared(name='wh', value=0.2 * numpy.random.uniform(-1.0, 1.0,(nh, nh)).astype(theano.config.floatX))#上一时刻隐藏到本时刻隐藏层循环递归的权值矩阵
        self.w = theano.shared(name='w',value=0.2 * numpy.random.uniform(-1.0, 1.0,(nh, nc)).astype(theano.config.floatX))#隐藏层到输出层的权值矩阵
        self.bh = theano.shared(name='bh', value=numpy.zeros(nh,dtype=theano.config.floatX))#隐藏层偏置参数
        self.b = theano.shared(name='b',value=numpy.zeros(nc,dtype=theano.config.floatX))#输出层偏置参数

        self.h0 = theano.shared(name='h0',value=numpy.zeros(nh,dtype=theano.config.floatX))

        self.lastlabel=theano.shared(name='lastlabel',value=0.2 * numpy.random.uniform(-1.0, 1.0,(nc, nc)).astype(theano.config.floatX))
        self.prelabel=theano.shared(name='prelabel',value=0.2 * numpy.random.uniform(-1.0, 1.0,(nc, nc)).astype(theano.config.floatX))
        self.bhmm=theano.shared(name='bhmm',value=numpy.zeros(nc,dtype=theano.config.floatX))

        self.params = [self.emb, self.wx, self.wh, self.w,self.bh, self.b, self.h0,self.lastlabel,self.prelabel,self.bhmm]#所有待学习的参数
        lr = T.scalar('lr')#学习率，一会儿作为输入参数



        idxs = T.itensor3()
        x = self.emb[idxs].reshape((idxs.shape[0],idxs.shape[1],de*idxs.shape[2]))
        y_sentence = T.imatrix('y_sentence')  # 训练样本标签,二维的(batch,sentence)
        def step(x_t, h_tm1):
            h_t = T.nnet.sigmoid(T.dot(x_t, self.wx) + T.dot(h_tm1, self.wh) + self.bh)#通过ht-1、x计算隐藏层
            s_temp=T.dot(h_t, self.w) + self.b#由于softmax不支持三维矩阵操作，所以这边需要对其进行reshape成2D，计算完毕后再reshape成3D
            return h_t, s_temp
        [h,s_temp], _ = theano.scan(step,sequences=x,outputs_info=[T.ones(shape=(x.shape[1],self.h0.shape[0])) * self.h0, None])
        p_y =T.nnet.softmax(T.reshape(s_temp,(s_temp.shape[0]*s_temp.shape[1],-1)))
        p_y=T.reshape(p_y,s_temp.shape)

        #加入前一时刻的标签约束项
        y_label3d = T.ftensor3('y_sentence3d')
        p_ytrain=self.add_layer(p_y,y_label3d)
        loss=self.nll_multiclass(p_ytrain,y_sentence)+0.0*((self.wx**2).sum()+(self.wh**2).sum()+(self.w**2).sum())
        #神经网络的输出
        sentence_gradients = T.grad(loss, self.params)
        sentence_updates = OrderedDict((p, p - lr*g) for p, g in zip(self.params, sentence_gradients))
        self.sentence_traintemp = theano.function(inputs=[idxs,y_sentence,y_label3d,lr],outputs=loss,updates=sentence_updates)





        '''self.sentence_train = theano.function(inputs=[idxs,y_sentence,lr],outputs=loss,updates=sentence_updates)'''
        #词向量归一化，因为我们希望训练出来的向量是一个归一化向量
        self.normalize = theano.function(inputs=[],updates={self.emb:self.emb /T.sqrt((self.emb**2).sum(axis=1)).dimshuffle(0, 'x')})

        #构造预测函数、训练函数，输入数据idxs每一行是一个样本(也就是一个窗口内的序列索引)
        #)
        self.classify = theano.function(inputs=[idxs], outputs=p_y)

def attention_q():
    query = T.itensor3('query')
    cands = T.itensor3('cands')

    d = 2
    W1_c = theano.shared(np.random.randint(-3, 3, (d, d)))
#    W1_c = theano.shared(np.ones((d, d), dtype='int32'))
    W1_h = theano.shared(np.random.randint(-3, 3, (d, d)))
#    W1_h = theano.shared(np.ones((d, d), dtype='int32'))
    w    = theano.shared(np.ones((d,), dtype='float32'))
    W2_r = theano.shared(np.random.randint(-1, 1, (d, d)))
    W2_h = theano.shared(np.random.randint(-1, 1, (d, d)))
#    W2_r = theano.shared(np.ones((d, d), dtype='float32'))
#    W2_h = theano.shared(np.ones((d, d), dtype='float32'))

#    q_in = np.asarray([[[1, 2], [3, 4], [5, 6]]], dtype='int32')
    q_in = np.ones((1, 3, 2), dtype='int32')
#    C_in = np.ones((1, 3, 2), dtype='int32')
#    C_in = np.ones((4, 3, 3, 2), dtype='int32')
    C_in = np.asarray(np.random.randint(-2, 2, (1, 3, 2)), dtype='int32')

    def forward(query, cands, eps=1e-8):
        # cands: 1D: n_queries, 2D: n_cands-1, 3D: dim_h
        # query: 1D: n_queries, 2D: n_words, 3D: dim_h
        # mask: 1D: n_queries, 2D: n_cands, 3D: n_words

        # 1D: n_queries, 2D: n_cands-1, 3D: n_words, 4D: dim_h
        M = T.dot(query, W1_c).dimshuffle(0, 'x', 1, 2) + T.dot(cands, W1_h).dimshuffle(0, 1, 'x', 2)

        # 1D: n_queries, 2D: n_cands-1, 3D: n_words
        alpha = T.nnet.softmax(T.dot(M, w).reshape((cands.shape[0] * cands.shape[1], query.shape[1])))
        alpha = alpha.reshape((cands.shape[0], cands.shape[1], query.shape[1], 1))

        # 1D: n_queries, 2D: n_cands-1, 3D: n_words
        r = T.sum(query.dimshuffle((0, 'x', 1, 2)) * alpha, axis=2)  # 4 * 3 * 2

        # 1D: n_queries, 2D: n_cands, 3D: dim_h
        h_after = T.dot(r, W2_r)  # 4 * 3 * 2
#        return h_after, h_after
        return h_after, r, alpha.reshape((alpha.shape[0], alpha.shape[1], alpha.shape[2])), M

    y, a, b, c = forward(query, cands)
    f = theano.function(inputs=[query, cands], outputs=[y, a, b, c], on_unused_input='ignore')
    print f(q_in, C_in)

def copy():
    e = np.asarray([[[2, 4], [5, 1]], [[3, 5], [4, 6]]], dtype='int32')
    w = T.itensor3('w')
    u = T.ones(shape=(2, w.shape[2]))

    y = T.repeat(T.max(w, axis=1, keepdims=True), 2, 1)
#    y = T.max(w, axis=1, keepdims=True) * u
    f = theano.function(inputs=[w], outputs=y)

    print f(e)

def test_lookup():
    a = T.itensor3()
    b = T.ivector()
    y = a[0][T.arange(b.shape[0]), b]
    f = theano.function(inputs=[a, b], outputs=[y])

    u = [[[1, 2], [2, 4]], [[3, 1], [2, 1]]]
    c = [0, 1]

    print f(u, c)

 def test_get_output_for(self):
     X = T.itensor3()
     X1 = np.empty((2, 2, 10), dtype='int32')
     for i, is_ in enumerate(itertools.product(*(range(n) for n in X1.shape[:-1]))):
         X1[is_] = np.arange(i, 10 + i)
     X2 = np.empty((2, 2, 3), dtype='int32')
     for i, is_ in enumerate(itertools.product(*(range(n) for n in X2.shape[:-1]))):
         X2[is_] = np.arange(7 + i, 10 + i)
     self.assertTrue(np.array_equal(
         theano.function([X], KMaxPool1DLayer(InputLayer((100, 100)), 3).get_output_for(X))(X1), X2
     ))

def dot():
    e1 = np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], dtype='int32')
    e2 = np.asarray([[1, 2], [3, 1]], dtype='int32')
    w = T.itensor3('w')
    v = T.imatrix('v')

    y = T.batched_dot(v, w.dimshuffle(0, 2, 1))
    u = w.T
    f = theano.function(inputs=[v, w], outputs=y)
    f2 = theano.function(inputs=[w], outputs=u)

    print f(e2, e1)

    def test_recurrent_lookup_table(self):
        E = np.random.uniform(size=(self.vocab_size, self.n_in)).astype('float32')
        W = np.random.uniform(size=(self.n_out, self.vocab_size)).astype('float32')
        b = np.random.uniform(size=self.vocab_size).astype('float32')
        embeddings = LookupTable(vocab_size=self.vocab_size, embedding_size=self.n_in, window_size=1, E_init=E, advanced_indexing=True)
        net = Recurrent([embeddings, self.layer, LinearLayer(n_in=self.n_out, n_out=self.vocab_size, W_init=W, b_init=b), Softmax()])

        x = np.array([
            [
                [0],
                [2]
            ],
            [
                [3],
                [1]
            ]
        ]).astype('int32')

        x_var = T.itensor3()
        p_var = net.forward(x_var)

        # the predictions for the two examples
        y = np.array([0, 2]).astype('int32')
        y_one_hot = np.array([[1, 0, 0, 0, 0],
                              [0, 0, 1, 0, 0]]).astype('int32')

        # time 1
        h1, mem1 = self.recurrent_step(E[[0,3]], *self.mem0)
        z1 = h1.dot(W) + b
        p1 = np.exp(z1) / np.exp(z1).sum(axis=1)[:, np.newaxis]

        # time 2
        h2, mem2 = self.recurrent_step(E[[2,1]], *mem1)
        z2 = h2.dot(W) + b
        p2 = np.exp(z2) / np.exp(z2).sum(axis=1)[:, np.newaxis]

        f = function([x_var], p_var, allow_input_downcast=True, on_unused_input='warn')
        got_p = f(x)

        self.assertTrue(np.allclose(got_p, p2))

        y_var = T.ivector()
        loss = cross_entropy_loss(p_var, y_var, one_hot_num_classes=self.vocab_size)
        f_loss = function([x_var, y_var], [p_var, loss], allow_input_downcast=True, on_unused_input='warn')

        got_p_out, got_loss = f_loss(x, y)

        expect_p_out = p2
        expect_loss = -1 * np.sum(y_one_hot * np.log(expect_p_out)) / 2.

        self.assertTrue(np.allclose(got_p_out, expect_p_out))
        self.assertTrue(np.allclose(got_loss, expect_loss))

     def __init__(self, nh, nc, ne, de, cs, bs): 
         ''' 
         nh :: dimension of the hidden layer 
         nc :: number of classes 
         ne :: number of word embeddings in the vocabulary 
         de :: dimension of the word embeddings 
         cs :: word window context size  
         bs :: batch size (number of samples)
         ''' 
         idxs = T.itensor3() # time->samples->features as many columns as context window size/lines as words in the sentence 
         # Data is given as a tensor (batch, sequence, context size)
         l_in = lasagne.layers.InputLayer((bs, None, cs), idxs)
         # We have a tensor (batch size, sequence length, concatenated context win. embeddings)
         l_emb = lasagne.layers.EmbeddingLayer(l_in, ne, de)
         l_flatt_emb = lasagne.layers.flatten(l_emb, outdim=3)
         
         print("Output of after embedding: {0}".format(lasagne.layers.get_output_shape(l_flatt_emb, (bs, 11, cs))))

         # Define recurent layer
         l_r = lasagne.layers.RecurrentLayer(l_flatt_emb, nh, nonlinearity=lasagne.nonlinearities.sigmoid)

         # Output shape should be (batch size, sequence, hidden)
         print("Output after recurrence: {0}".format(lasagne.layers.get_output_shape(l_r, (bs, 11, cs))))
         
         l_res = lasagne.layers.ReshapeLayer(l_r, (-1, l_r.output_shape[2]))
         print("Output after reshape: {0}".format(lasagne.layers.get_output_shape(l_res, (bs, 11, cs))))

         l_out = lasagne.layers.DenseLayer(l_res, nc, nonlinearity=lasagne.nonlinearities.softmax)
         print("Output shape: {0}".format(lasagne.layers.get_output_shape(l_out, (bs, 11, cs))))


         y_sentence = T.ivector('y_sentence')
         y_mask = T.vector('y_mask')
         pred = lasagne.layers.get_output(l_out)         
         c_pred = T.argmax(pred, axis = 1)
         sentence_nll = T.mean(lasagne.objectives.categorical_crossentropy(pred, y_sentence) * y_mask)
         sentence_error = T.sum(T.neq(c_pred, y_sentence)*y_mask)
         params = lasagne.layers.get_all_params(l_out)
         sentence_gradients = T.grad(sentence_nll, params)
         lr = 0.0627142536696559
         #sentence_updates = OrderedDict((p, p - lr*g) for p, g in zip(params, sentence_gradients))
         sentence_updates = lasagne.updates.momentum(sentence_nll, params, lr)

         self.train_sentence = theano.function(inputs  = [idxs, y_sentence, y_mask], 
                                               outputs = sentence_nll, 
                                               updates = sentence_updates) 

         self.normalize = theano.function( inputs = [], 
                                           updates = {l_emb.W: l_emb.W/T.sqrt((l_emb.W**2).sum(axis=1)).dimshuffle(0,'x')})  


         self.errors = theano.function(inputs=[idxs, y_sentence, y_mask], outputs=sentence_error)