def build_model(self):
   print '\n... building the model with unroll=%d, backroll=%d' \
     % (self.source.unroll, self.source.backroll)
   x = T.imatrix('x')
   y = T.imatrix('y')
   reset = T.scalar('reset')
   hiddens = [h['init'] for h in self.hiddens.values()]
   outputs_info = [None] * 3 + hiddens
   [losses, probs, errors, hids], updates = \
     theano.scan(self.step, sequences=[x, y], outputs_info=outputs_info)
   loss = losses.sum()
   error = errors.sum() / T.cast((T.neq(y, 255).sum()), floatX)
   hidden_updates_train = []
   hidden_updates_test = []
   for h in self.hiddens.values():
     h_train = ifelse(T.eq(reset, 0), \
       hids[-1-self.source.backroll, :], T.ones_like(h['init']))
     h_test = ifelse(T.eq(reset, 0), \
       hids[-1, :], T.ones_like(h['init']))
     hidden_updates_train.append((h['init'], h_train))
     hidden_updates_test.append((h['init'], h_test))
   updates = self.source.get_updates(loss, self.sgd_params)
   updates += hidden_updates_train
   rets = [loss, probs[-1, :], error]
   mode = theano.Mode(linker='cvm')
   train_model = theano.function([x, y, reset, self.lr], rets, \
     updates=updates, mode=mode)
   test_model = theano.function([x, y, reset], rets, \
     updates=hidden_updates_test, mode=mode)
   return train_model, test_model

	def build_and_train_model(self,n_hu,n_hl):
		print('Building Model')

		input_phrase = T.imatrix('train_inputmatrix')
		labels = T.imatrix('trainphrase_matrix')

		network = self.define_layers(input_phrase,labels,n_hu,n_hl)

		print("Defining loss")
		#Prediction or loss
		prediction = []
		prediction.append(T.clip(lasagne.layers.get_output(network[0]),1.0e-7,1.0-1.0e-7))
		prediction.append(T.clip(lasagne.layers.get_output(network[1]),1.0e-7,1.0-1.0e-7))

		loss = l.define_loss(prediction[0],prediction[1])
		self.model = network
		#define params
		params = lasagne.layers.get_all_params(network)
		updates = lasagne.updates.adadelta(loss,params)

		#run test

		train_fn = theano.function([input_phrase,labels],[loss, prediction[0], prediction[1]],updates=updates,allow_input_downcast=True)

		print("Model and params defined now training")
		epoch = 0
		for epoch in range(self.end_epoch):
			train_loss = 0
			train_pred = []
			start_time = time.time()
			loss, predicted, phrase = train_fn(self.train_inputmatrix,self.trainphrase_matrix)
			print('Training Loss: ' + str(loss) + ' Train Epoch ' + str(epoch))
			self.save_best(loss,predicted,network)

    def _make_stack(self, seq_length=4):
        self.embedding_dim = embedding_dim = 3
        self.vocab_size = vocab_size = 10
        self.seq_length = seq_length

        def compose_network(inp, inp_dim, outp_dim, vs, name="compose"):
            # Just add the two embeddings!
            W = T.concatenate([T.eye(outp_dim), T.eye(outp_dim)], axis=0)
            return inp.dot(W)

        X = T.imatrix("X")
        transitions = T.imatrix("transitions")
        apply_dropout = T.scalar("apply_dropout")
        vs = VariableStore()
        self.stack = HardStack(
            embedding_dim,
            embedding_dim,
            vocab_size,
            seq_length,
            compose_network,
            IdentityLayer,
            apply_dropout,
            vs,
            X=X,
            transitions=transitions,
            make_test_fn=True,
        )

        # Swap in our own dummy embeddings and weights.
        embeddings = np.arange(vocab_size).reshape((vocab_size, 1)).repeat(embedding_dim, axis=1)
        self.stack.embeddings.set_value(embeddings)

    def _classify(self,dataset_static,dataset_nonstatic):
        """
        Classify method for static or non-static models.
        :param classifier: model
        :param conv_layers: list of convPoolLayer objects
        :param Words: Dictionary of word index to word vectors
        :param dataset: Indices of words for the current sentence/dataset
        :param dim: dimension of word vector
        :param img_h: length of sentence vector after padding
        :return: [y_pred,prob_pred] The probability for each class
        """
        x_static = T.imatrix('x_static')
        x_nonstatic = T.imatrix('x_nonstatic')
        y = T.ivector('y')
        Words_static = theano.shared(value = self.Words_static, name = "Words_static")
        Words_nonstatic = theano.shared(value = self.Words_nonstatic, name = "Words_nonstatic")

        test_pred_layers = []
        test_size = np.shape(dataset_static)[0]
        test_layer0_input_static = Words_static[T.cast(x_static.flatten(),dtype="int32")].reshape((test_size,1,self.img_h,self.Words_static.shape[1]))
        test_layer0_input_nonstatic = Words_nonstatic[T.cast(x_nonstatic.flatten(),dtype="int32")].reshape((test_size,1,self.img_h,self.Words_nonstatic.shape[1]))
        for i in range(len(self.conv_layers)/2):
            test_layer0_output = self.conv_layers[i].predict(test_layer0_input_nonstatic, test_size)
            test_pred_layers.append(test_layer0_output.flatten(2))
        for i in range(len(self.conv_layers)/2,len(self.conv_layers)):
            test_layer0_output = self.conv_layers[i].predict(test_layer0_input_static, test_size)
            test_pred_layers.append(test_layer0_output.flatten(2))

        test_layer1_input = T.concatenate(test_pred_layers, 1)
        test_y_pred = self.classifier.predict(test_layer1_input)
        test_prob_pred = self.classifier.predict_p(test_layer1_input)
        test_model_all = theano.function([x_static,x_nonstatic], (test_y_pred,test_prob_pred))

        return test_model_all(dataset_static,dataset_nonstatic)

 def __init__(self, size_vocab, size_embed, size, size_out, depth, network, 
              alpha=0.5,
              gru_activation=clipped_rectify, 
              visual_activation=linear, 
              visual_encoder=StackedGRUH0, 
              cost_visual=CosineDistance,
              max_norm=None, 
              lr=0.0002, 
              dropout_prob=0.0):
     autoassign(locals())
     self.network = network(self.size_vocab, 
                            self.size_embed, 
                            self.size, 
                            self.size_out, 
                            self.depth,
                            gru_activation=self.gru_activation, 
                            visual_activation=self.visual_activation,
                            visual_encoder=self.visual_encoder,
                            dropout_prob=self.dropout_prob)
                            
     self.input         = T.imatrix()
     self.output_t_prev = T.imatrix()
     self.output_t      = T.imatrix()
     self.output_v      = T.fmatrix()
     self.OH       = OneHot(size_in=self.size_vocab)
     self.output_t_oh   = self.OH(self.output_t)
     self.updater = util.Adam(max_norm=self.max_norm, lr=self.lr)
     self.train = self._make_train()
     self.loss_test = self._make_loss_test()

	def train_ready(self):
		print "adopt softmax model plus contractive regularization ........ "
		print "weight 1   : "+str(self.lowreg_weight)
		print "weight 2   : "+str(self.highreg_weight)
		print "variance   : "+str(self.variance)
		print "nc  : "+str(self.nc)

		var_x = T.imatrix()
		var_y = T.imatrix()

		loss = self.reg_logp(var_x,var_y, self.lowreg_weight, self.highreg_weight, self.variance, self.nc)
	

		witems = self.w.values()
		#ave_w = sum(T.sum(item**2) for item in witems)/len(witems)
		wg = T.grad(loss, witems)
		#ave_g = sum(T.sum(item**2) for item in wg) /len(wg)

		weight_up = self.upda(wg, witems, self.lrate, self.mweight, self.opt, self.gradbound)

		if not self.fix_emb:
			dicitems = self.dic.values()
			dg = T.grad(loss, dicitems)

			dic_up = self.upda(dg, dicitems, self.lrate/10., self.mweight, self.opt)
			weight_up.update(dic_up)

		up  = weight_up

		self.updatefunc = theano.function([var_x, var_y], loss,updates = up)

def run():
    batch_size = 16
    prems = np.random.randint(low=0, high=99, size=(batch_size, 5), dtype='int32')
    hypoes = np.random.randint(low=0, high=99, size=(batch_size, 3), dtype='int32')
    labels = np.random.randint(low=0, high=3, size=(batch_size,), dtype='int32')
    print prems
    print hypoes
    print labels

    ematrix = np.random.uniform(low=-1, high=1, size=(100, 100)).astype(theano.config.floatX)

    t_prems = T.imatrix('p')
    t_hypoes = T.imatrix('h')
    t_ematrix = theano.shared(ematrix, 't_ematrix')

    r_prems = T.repeat(t_prems, 3, axis= 1)
    r_hypoes = T.concatenate([t_hypoes]* 5, axis=1)

    batch_prems = t_ematrix[r_prems]
    batch_hypoes = t_ematrix[r_hypoes]

    batch_prem_hypo = T.concatenate((batch_prems, batch_hypoes), axis=2)

    get_b_prems = theano.function(inputs=[t_prems], outputs=batch_prems)
    get_r_prems = theano.function(inputs=[t_prems], outputs=r_prems)
    get_b_hypoes = theano.function(inputs=[t_hypoes], outputs=batch_hypoes)
    get_r_hypoes = theano.function(inputs=[t_hypoes], outputs=r_hypoes)
    get_b_ph = theano.function(inputs=[t_prems, t_hypoes], outputs=batch_prem_hypo)

    # print get_b_prems(prems)
    print get_r_prems(prems)
    print get_r_hypoes(hypoes)

    print get_b_prems(prems).shape
    print get_b_hypoes(hypoes).shape

    print get_b_ph(prems, hypoes).shape

    W = theano.shared(
        value=np.random.uniform(
            low=-np.sqrt(1. / 6),
            high=np.sqrt(1. / 6),
            size=(200, 400)
        ).astype(theano.config.floatX),
        name='W'
    )

    U = theano.shared(
        value=np.random.uniform(
            low=-np.sqrt(1. / 6),
            high=np.sqrt(1. / 6),
            size=(400,)
        ).astype(theano.config.floatX),
        name='U'
    )

    result = T.dot(T.dot(batch_prem_hypo, W), U)

    get_result = theano.function(inputs=[t_prems, t_hypoes], outputs=result)
    print get_result(prems, hypoes).shape

def create_model(num_timesteps, num_blocks, hidden_size, learning_rate, \
    grad_clip=10, dropout_p=0.5, num_lstm_layers=1, use_forward_and_backward_lstm=False):
    '''
     returns train function which reports both loss and accuracy
     and test function, which also reports both loss and accuracy
    '''
    
    l_in, l_mask, l_out, l_out_slice, l_lstm, l_lstm_slice = \
    _build_net_layers(num_timesteps, num_blocks, hidden_size, learning_rate, \
        grad_clip, dropout_p, num_lstm_layers, use_forward_and_backward_lstm)

    inp = T.tensor3('input')
    truth = T.imatrix("truth")
    mask = T.imatrix("mask")

    # pred should be of shape (batchsize, num_timesteps, num_asts)
    pred = lasagne.layers.get_output(l_out)
    # pred_slice should be of shape (batchsize, num_asts), only contains
    # predictions for the last timestep
    pred_slice = lasagne.layers.get_output(l_out_slice)
    # the hidden representations for the last timestep (batchsize, hidden_size)
    hidden_slice = lasagne.layers.get_output(l_lstm_slice)
    # truth should also be of shape (batchsize, num_timesteps, num_asts)

    pred_2d = pred.reshape((-1, num_blocks))
    truth_1d = truth.reshape((-1,))

    # pred_2d_shape = T.shape(pred_2d)
    # truth_1d_shape = T.shape(truth_1d)

    # categorical_crossentropy
    loss = T.nnet.categorical_crossentropy(pred_2d, truth_1d).mean()
    # categorical accuracy
    # acc = T.nnet.categorical_crossentropy(pred_2d, truth_1d).mean()
    acc = lasagne.objectives.categorical_accuracy(pred_2d, truth_1d).mean()
    # update function
    print("Computing updates ...")
    all_params = lasagne.layers.get_all_params(l_out)
    updates = lasagne.updates.adam(loss, all_params, learning_rate)

    # training function
    print("Compiling functions ...")
    train_loss = theano.function([l_in.input_var, l_mask.input_var, truth], loss, updates=updates, allow_input_downcast=True)
    compute_loss = theano.function([l_in.input_var, l_mask.input_var, truth], loss, allow_input_downcast=True)
    # training function, returns loss and acc
    compute_pred = theano.function([l_in.input_var, l_mask.input_var, truth],  [pred_2d, truth_1d], updates=updates, allow_input_downcast=True)
    train_loss_acc = theano.function([l_in.input_var, l_mask.input_var, truth], [loss, acc, pred], updates=updates, allow_input_downcast=True)
    # computes loss and accuracy, without training
    compute_loss_acc = theano.function([l_in.input_var, l_mask.input_var, truth], [loss, acc, pred], allow_input_downcast=True)

    # In order to generate text from the network, we need the probability distribution of the next character given
    # the state of the network and the input (a seed).
    # In order to produce the probability distribution of the prediction, we compile a function called probs. 
    probs = theano.function([l_in.input_var, l_mask.input_var], pred_slice, allow_input_downcast=True)

    generate_hidden_representations = theano.function([l_in.input_var, l_mask.input_var], hidden_slice, allow_input_downcast=True)

    print("Compiling done!")
    
    return train_loss_acc, compute_loss_acc, probs, generate_hidden_representations, compute_pred, l_out

def set_model(argv, vocab_word, init_emb):
    x_span = T.imatrix("x_span")
    x_word = T.imatrix("x_word")
    x_ctx = T.imatrix("x_ctx")
    x_dist = T.imatrix("x_dist")
    x_slen = T.imatrix("x_slen")
    y = T.ivector("y")

    """ Set params for the model """
    n_vocab = vocab_word.size()
    dim_x_word = argv.emb
    dim_x_dist = 10  # (0, ..., 10-)
    dim_h = argv.hidden
    L2_reg = argv.reg

    """ Instantiate the model """
    return Model(
        x_span=x_span,
        x_word=x_word,
        x_ctx=x_ctx,
        x_dist=x_dist,
        x_slen=x_slen,
        y=y,
        init_emb=init_emb,
        n_vocab=n_vocab,
        dim_w_p=dim_x_word,
        dim_d=dim_x_dist,
        dim_h=dim_h,
        L2_reg=L2_reg,
    )

    def build_model_1(self):

        x = T.imatrix('x').astype(theano.config.floatX)
        drop_masks = T.imatrix('drop_masks').astype(theano.config.floatX)
        y = T.ivector('y')

        self.layers[0] = LSTMLayer(random_state=self.random_state,input=x,drop_masks=drop_masks,input_dim=self.input_dim,output_dim=self.hidden_dims[0])
        params = self.layers[0].params

        self.layers[1] = OutputLayer(input=self.layers[0].output,
                                                         input_dim=self.layers[0].output_dim, output_dim=self.output_dim,random_state=self.random_state)

        params += self.layers[1].params
        _EPSILON = 10e-8

        L1 = 0.001 * T.sum([T.sum(param) for param in params])
        L2 = 0.001 * T.sum([T.sum(param ** param) for param in params])
        cost = T.sum(T.nnet.categorical_crossentropy(T.clip(self.layers[self.number_of_layers].probabilities[-1], _EPSILON, 1.0 - _EPSILON),y)) + L1 + L2

        #grads = T.grad(cost, params)

        #updates = [(param_i, param_i - self.learning_rate * grad_i) for param_i,grad_i in zip(params,grads)]
        updates =  LearningAlgorithms.adam(cost,params,learning_rate=0.001)

        self.sgd_step = theano.function([x,drop_masks, y], L1, updates=updates)
        self.predict = theano.function([x,drop_masks],self.layers[self.number_of_layers].probabilities[-1])

        self.test_model = theano.function([x,drop_masks, y], cost)

    def __theano_build__(self):
        params = self.params
        param_names = self.param_names
        hidden_dim = self.hidden_dim

        x1  = T.imatrix('x1')    # first sentence
        x2  = T.imatrix('x2')    # second sentence
        x1_mask = T.fmatrix('x1_mask')    #mask
        x2_mask = T.fmatrix('x2_mask')
        y   = T.ivector('y')     # label
        y_c = T.ivector('y_c')   # class weights 
        
        # Embdding words
        _E1 = params["E"].dot(params["W"][0]) + params["B"][0]
        _E2 = params["E"].dot(params["W"][1]) + params["B"][1]
        statex1 = _E1[x1.flatten(), :].reshape([x1.shape[0], x1.shape[1], hidden_dim])
        statex2 = _E2[x2.flatten(), :].reshape([x2.shape[0], x2.shape[1], hidden_dim])
        
        def rnn_cell(x, mx, ph, Wh):
            h = T.tanh(ph.dot(Wh) + x)
            h = mx[:, None] * h + (1-mx[:, None]) * ph
            return [h] 
            
        [h1], updates = theano.scan(
            fn=rnn_cell,
            sequences=[statex1, x1_mask],
            truncate_gradient=self.truncate,
            outputs_info=[dict(initial=T.zeros([self.batch_size, self.hidden_dim]))],
            non_sequences=params["W"][2])
        
        [h2], updates = theano.scan(
            fn=rnn_cell,
            sequences=[statex2, x2_mask],
            truncate_gradient=self.truncate,
            outputs_info=[dict(initial=h1[-1])],
            non_sequences=params["W"][3])
       
        #predict
        _s = T.nnet.softmax(h1[-1].dot(params["lrW"][0]) + h2[-1].dot(params["lrW"][1]) + params["lrb"])
        _p = T.argmax(_s, axis=1)
        _c = T.nnet.categorical_crossentropy(_s, y)
        _c = T.sum(_c * y_c)
        _l = T.sum(params["lrW"]**2)
        _cost = _c + 0.01 * _l
        
        # SGD parameters
        learning_rate = T.scalar('learning_rate')
        decay = T.scalar('decay')
        
        # Gradients and updates
        _grads, _updates = rms_prop(_cost, param_names, params, learning_rate, decay)
        
        # Assign functions
        self.bptt = theano.function([x1, x2, x1_mask, x2_mask, y, y_c], _grads)
        self.loss = theano.function([x1, x2, x1_mask, x2_mask, y, y_c], _c)
        self.weights = theano.function([x1, x2, x1_mask, x2_mask], _s)
        self.predictions = theano.function([x1, x2, x1_mask, x2_mask], _p)
        self.sgd_step = theano.function(
            [x1, x2, x1_mask, x2_mask, y, y_c, learning_rate, decay],
            updates=_updates)

    def build(self):
        """build the model. This method should be called after self.add_data.
        """
        x_sym = sparse.csr_matrix('x', dtype = 'float32')
        y_sym = T.imatrix('y')
        g_sym = T.imatrix('g')
        gy_sym = T.vector('gy')
        ind_sym = T.ivector('ind')

        l_x_in = lasagne.layers.InputLayer(shape = (None, self.x.shape[1]), input_var = x_sym)
        l_g_in = lasagne.layers.InputLayer(shape = (None, 2), input_var = g_sym)
        l_ind_in = lasagne.layers.InputLayer(shape = (None, ), input_var = ind_sym)
        l_gy_in = lasagne.layers.InputLayer(shape = (None, ), input_var = gy_sym)

        num_ver = max(self.graph.keys()) + 1
        l_emb_in = lasagne.layers.SliceLayer(l_g_in, indices = 0, axis = 1)
        l_emb_in = lasagne.layers.EmbeddingLayer(l_emb_in, input_size = num_ver, output_size = self.embedding_size)
        l_emb_out = lasagne.layers.SliceLayer(l_g_in, indices = 1, axis = 1)
        if self.neg_samp > 0:
            l_emb_out = lasagne.layers.EmbeddingLayer(l_emb_out, input_size = num_ver, output_size = self.embedding_size)

        l_emd_f = lasagne.layers.EmbeddingLayer(l_ind_in, input_size = num_ver, output_size = self.embedding_size, W = l_emb_in.W)
        l_x_hid = layers.SparseLayer(l_x_in, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
        
        if self.use_feature:
            l_emd_f = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
            l_y = lasagne.layers.ConcatLayer([l_x_hid, l_emd_f], axis = 1)
            l_y = layers.DenseLayer(l_y, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
        else:
            l_y = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)

        py_sym = lasagne.layers.get_output(l_y)
        loss = lasagne.objectives.categorical_crossentropy(py_sym, y_sym).mean()
        if self.layer_loss and self.use_feature:
            hid_sym = lasagne.layers.get_output(l_x_hid)
            loss += lasagne.objectives.categorical_crossentropy(hid_sym, y_sym).mean()
            emd_sym = lasagne.layers.get_output(l_emd_f)
            loss += lasagne.objectives.categorical_crossentropy(emd_sym, y_sym).mean()

        if self.neg_samp == 0:
            l_gy = layers.DenseLayer(l_emb_in, num_ver, nonlinearity = lasagne.nonlinearities.softmax)
            pgy_sym = lasagne.layers.get_output(l_gy)
            g_loss = lasagne.objectives.categorical_crossentropy(pgy_sym, lasagne.layers.get_output(l_emb_out)).sum()
        else:
            l_gy = lasagne.layers.ElemwiseMergeLayer([l_emb_in, l_emb_out], T.mul)
            pgy_sym = lasagne.layers.get_output(l_gy)
            g_loss = - T.log(T.nnet.sigmoid(T.sum(pgy_sym, axis = 1) * gy_sym)).sum()

        params = [l_emd_f.W, l_emd_f.b, l_x_hid.W, l_x_hid.b, l_y.W, l_y.b] if self.use_feature else [l_y.W, l_y.b]
        if self.update_emb:
            params = lasagne.layers.get_all_params(l_y)
        updates = lasagne.updates.sgd(loss, params, learning_rate = self.learning_rate)

        self.train_fn = theano.function([x_sym, y_sym, ind_sym], loss, updates = updates, on_unused_input = 'ignore')
        self.test_fn = theano.function([x_sym, ind_sym], py_sym, on_unused_input = 'ignore')
        self.l = [l_gy, l_y]

        g_params = lasagne.layers.get_all_params(l_gy, trainable = True)
        g_updates = lasagne.updates.sgd(g_loss, g_params, learning_rate = self.g_learning_rate)
        self.g_fn = theano.function([g_sym, gy_sym], g_loss, updates = g_updates, on_unused_input = 'ignore')

    def setup_encode(self):

        # dimensions: (batch, time, 12)
        chord_types = T.btensor3()
        # dimensions: (batch, time)
        chord_roots = T.imatrix()
        # dimensions: (batch, time)
        relative_posns = [T.imatrix() for _ in self.encodings]
        # dimesions: (batch, time, output_data)
        encoded_melodies = [T.btensor3() for _ in self.encodings]
        n_batch, n_time = chord_roots.shape

        all_activations = []
        for encoding, enc_lstmstack, encoded_melody, relative_pos in zip(self.encodings, self.enc_lstmstacks, encoded_melodies, relative_posns):
            activations = enc_lstmstack.do_preprocess_scan( timestep=T.tile(T.arange(n_time), (n_batch,1)) ,
                                                        relative_position=relative_pos,
                                                        cur_chord_type=chord_types,
                                                        cur_chord_root=chord_roots,
                                                        cur_input=encoded_melody,
                                                        deterministic_dropout=True )
            all_activations.append(activations)
        reduced_activations = functools.reduce((lambda x,y: x+y), all_activations)
        strengths, vects = self.qman.get_strengths_and_vects(reduced_activations)

        self.encode_fun = theano.function(
            inputs=[chord_types, chord_roots] + relative_posns + encoded_melodies,
            outputs=[strengths, vects],
            allow_input_downcast=True,
            mode=(NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True) if self.nanguard else None))

 def _get_input_tensor_variables(self):
     # x_w: 1D: batch, 2D: n_words, 3D: 5 + window; word id
     # x_p: 1D: batch, 2D: n_words; posit id
     # y: 1D: batch, 2D: n_words; label id
     if self.argv.mark_phi:
         return [T.itensor3('x_w'), T.imatrix('x_p'), T.imatrix('y')]
     return [T.itensor3('x_w'), T.imatrix('y')]

    def test_sparseblockgemvF(self):
        """
            Test the fortan order for W (which can happen in the grad for some
            graphs).
        """
        b = tensor.fmatrix()
        W = tensor.ftensor4()
        h = tensor.ftensor3()
        iIdx = tensor.imatrix()
        oIdx = tensor.imatrix()

        o = self.gemv_op(b.take(oIdx, axis=0),
                         tensor.DimShuffle((False, False, False, False),
                                           (0, 1, 3, 2))
                         (tensor.as_tensor_variable(W)),
                         h, iIdx, oIdx)

        f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode)

        W_val, h_val, iIdx_val, b_val, oIdx_val = \
            BlockSparse_Gemv_and_Outer.gemv_data()

        th_out = f(numpy.swapaxes(W_val, 2, 3), h_val, iIdx_val, b_val,
                   oIdx_val)
        ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy(
            b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val)

        utt.assert_allclose(ref_out, th_out)

 def __init__( self,
               config,
               qvocab_len,
               max_qlen,
               num_ans,
               num_qtypes,  
               l_saver):
     self.config                     = config
     self.qn                         = T.imatrix()
     self.lstm_mask                  = T.imatrix()
     self.iX                         = T.fmatrix()
     self.Y                          = T.ivector()
     self.qtype                      = T.ivector()
     self.sparse_indices             = T.ivector()
     self.qembd                      = T.fmatrix()
     self.ql_out                     = T.fmatrix()
     self.timer                      = l.timer_type()
     self.saver, self.exp_saver      = l_saver
     self.qlstm_hidden_dim           = 300 
     self.qn_classifier_emb_size     = 75
     self.max_ql                     = max_qlen
     self.qvocab_len                 = qvocab_len 
     self.bptt_trunk_steps           = -1 
     self.mlp_input_dim              = 1024
     self.num_qtypes                 = num_qtypes
     self.num_ans                    = num_ans
     self.grad_clip                  = config['grad_clip']
     self.params                     = {}
     print "Models Initialization done ..."

    def train_model_func(self, batch_size, num_batches, summary_sz, input_sz):
        summaries = T.imatrix('summaries')
        docs = T.imatrix('docs')

        s = np.zeros((batch_size * num_batches, summary_sz))
        d = np.zeros((batch_size * num_batches, input_sz))

        summary_superbatch = theano.shared( s.astype(theano.config.floatX),
                                           name = 's_summs', borrow = True )
        doc_superbatch = theano.shared( d.astype(theano.config.floatX),
                                        name = 's_docs', borrow = True )

        self.ssb = summary_superbatch
        self.dsb = doc_superbatch

        cost = self.negative_log_likelihood_batch(docs, summaries, batch_size)
        regularization_cost = self.l2_coefficient * sum([(p ** 2).sum() for p in self.params])

        self.get_batch_cost_unregularized = theano.function([docs, summaries], cost, allow_input_downcast=True)
        #theano.printing.debugprint(cost)
        cost = cost + regularization_cost

        params = {p.name: p for p in self.params} 
        grads = T.grad(cost, self.params)
        #grads = theano.printing.Print("grads")(grads)

        # learning rate
        lr = T.scalar(name='lr')
        gradient_update = optimisers.sgd_(lr, self.params, grads, docs, summaries,
                                                                cost, self.dsb, self.ssb, batch_size)
        return gradient_update

def run():
    # params
    dims = 10
    negrate = 1
    batsize = 300
    epochs = 300

    #paths
    datafileprefix = "../../data/nycfilms/"
    dirfwdsuffix = "direct_forward.plustypes.ssd"

    # get the data and split
    dirfwdf = open(datafileprefix+dirfwdsuffix)
    datadf = readdata(dirfwdf)
    traind, validd, testd = datadf.split((70, 15, 15), random=True)

    numents = int(datadf.ix[:, 0].max())+1
    print numents
    numrels = int(datadf.ix[:, 1].max())+1
    print numrels

    # define model
    inp = Input(T.imatrix())

    eemb = VectorEmbed.indim(numents).outdim(dims).Wreg(l2reg(0.00001))()
    remb = VectorEmbed.indim(numrels).outdim(dims).Wreg(l2reg(0.00001))()

    # for debugging
    eembd = SymTensor(T.fmatrix())
    rembd = SymTensor(T.fmatrix())
    dotp = SymTensor(T.fmatrix())

    out = ((inp[:, 0] >> eemb >> eembd) & (inp[:, 1] >> remb >> rembd)) >> DotProduct() >> dotp >> Tanh()

    # for plotting purposes: relation to relation dot product (or relation-type)
    r2rinp = Input(T.imatrix())
    rel2rel = ((r2rinp[:, 0] >> remb) & (r2rinp[:, 1] >> remb)) >> DotProduct()

    outtest = Output(T.fvector())

    loss = (out & outtest) >> HingeLoss()
    trainer = Trainer\
        .batsize(batsize)\
        .epochs(epochs)\
        .onrun(getonrun())\
        .offrun(offrun)\
        .offepoch(getoffepoch(out, rel2rel))\
        .onbatch(getonbatch(negrate, numents, numrels))\
        .optimizer(sgd(lr=1.))\
        .batchtransformer(transbat)
    trainer\
        .loss(loss)\

    trainer.train(traind.values, validd.values)\
           .test(testd.values)

    explore(eemb, remb)
    # functions for interactive exploration

    embed()

    def __init__(self,rng,model_params):
        self.input = T.itensor3('input') # the data is a minibatch
        self.label = T.imatrix('label') # label's shape (mini_batch size, max_term_per_sent)
        self.sent_length= T.ivector('sent_length') # sent_length is the number of terms in each sentence
        self.masks = T.imatrix('masks') # masks which used in error and likelihood calculation

        self.core = SentenceLevelNeuralModelCore(rng,self.input,self.label,self.sent_length,self.masks,model_params)

        self.params = self.core.wordvec.params() \
                + self.core.POSvec.params() \
                + self.core.wordpos_vec.params() \
                + self.core.verbpos_vec.params() \
                + self.core.conv_word.params() \
                + self.core.conv_POS.params() \
                + self.core.conv_wordpos.params() \
                + self.core.conv_verbpos.params() \
                + self.core.hidden_layer.params

        self.L2_sqr = (self.core.wordvec.embeddings ** 2).sum() \
                + (self.core.POSvec.embeddings ** 2).sum() \
                + (self.core.wordpos_vec.embeddings ** 2).sum() \
                + (self.core.verbpos_vec.embeddings ** 2).sum() \
                + (self.core.conv_word.W ** 2).sum() \
                + (self.core.conv_POS.W ** 2).sum() \
                + (self.core.conv_wordpos.W ** 2).sum() \
                + (self.core.conv_verbpos.W ** 2).sum() \
                + (self.core.hidden_layer.W ** 2).sum()

        self.negative_log_likelihood = self.core.likelihood()
        self.errors = self.core.errors()

        # we only use L2 regularization
        self.cost = self.negative_log_likelihood \
                + self.core.L2_reg * self.L2_sqr


        self.gparams = []
        for param in self.params:
            gparam = T.grad(self.cost, param)
            self.gparams.append(gparam)

        self.updates = []

        learning_rate = model_params['learning_rate']
        for param, gparam in zip(self.params, self.gparams):
            self.updates.append((param, param - learning_rate * gparam))


        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_word.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_POS.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_verbpos.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_wordpos.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.conv_out,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.max_out,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.hidden_layer.output,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.core.negative_log_likelihood,on_unused_input='ignore')
        #self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.cost,on_unused_input='ignore')
        self.train_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=self.cost,updates=self.updates,on_unused_input='ignore')
        self.valid_model = theano.function(inputs=[self.input,self.label,self.masks], outputs=[self.errors,self.core.sentce_loglikelihood.y_pred_pointwise],on_unused_input='ignore')

def ndim_itensor(ndim, name=None):
    if ndim == 2:
        return T.imatrix(name)
    elif ndim == 3:
        return T.itensor3(name)
    elif ndim == 4:
        return T.itensor4(name)
    return T.imatrix(name=name)