def make_node(self, x):
        ###
        # At least for small matrices (5x5), the .sum() method of a csc matrix returns a dense matrix
        # as the result whether axis is 0 or 1... weird!
        ###
        if self.axis is None:
            z = tensor.tensor(broadcastable=(), dtype=x.dtype)
        elif self.axis == 0:
            if x.format == 'csc':
                z = tensor.tensor(broadcastable=(False,), dtype=x.dtype)
            elif x.format == 'csr':
                #return SparseVector() #WRITEME!
                raise NotImplementedError()
            else:
                raise NotImplementedError()
        elif self.axis == 1:
            if x.format == 'csc':
                #return SparseVector() #WRITEME!
                raise NotImplementedError()
            elif x.format == 'csr':
                z = tensor.tensor(broadcastable=(False,), dtype=x.dtype)
            else:
                raise NotImplementedError()
        else:
            assert False #axis should have been verified by self.__init__

        return gof.Apply(self, [x], [z])

 def make_node(self, pvals):
     pvals = T.as_tensor_variable(pvals)
     if self.odtype == 'auto':
         odtype = pvals.dtype
     vals = T.tensor(dtype=odtype, broadcastable=pvals.type.broadcastable)
     indx = T.tensor(dtype='int32', broadcastable=pvals.type.broadcastable)
     return Apply(self, [pvals,], [vals, indx])

    def __init__(self, activation, dim=2, issequence=False, inpshape=None):
        """
        :type activation: callable
        :param activation: Activation function (any element-wise symbolic function)

        :type dim: int
        :param dim: Dimensionality of the input data

        :type issequence: bool
        :param issequence: Whether the input is a sequence

        :type inpshape: list
        :param inpshape: Input shape
        """

        super(activationlayer, self).__init__()

        # Parse data dimensionality
        assert not (dim is None and inpshape is None), "Data dimension can not be parsed. Provide dim or inpshape."

        # Meta
        self.activation = activation
        self.dim = dim if dim is not None else {4: 2, 5: 3}[len(inpshape)]
        self.allowsequences = True
        self.issequence = self.dim == 2 and len(inpshape) == 5 if issequence is None else issequence
        self.inpdim = len(inpshape) if inpshape is not None else 5 if self.issequence else {2: 4, 3: 5}[dim]

        # Shape inference
        self.inpshape = [None, ] * self.inpdim if inpshape is None else list(inpshape)

        # Containers for input and output
        self.x = T.tensor('floatX', [False, ] * self.inpdim, name='x:' + str(id(self)))
        self.y = T.tensor('floatX', [False, ] * self.inpdim, name='y:' + str(id(self)))

def test_elemwise_grad_broadcast():
    # This crashed in the past.

    x = tensor.tensor(dtype="float32", broadcastable=(True, False, False, False))
    y = tensor.tensor(dtype="float32", broadcastable=(True, True, False, False))

    theano.grad(theano.tensor.tanh(x).sum(), x)
    theano.grad(theano.tensor.tanh(x + y).sum(), y)
    theano.grad(theano.tensor.tanh(x + y).sum(), [x, y])

 def setUp(self, dtype='float64'):
     self.dtype = dtype
     self.mode = theano.compile.get_default_mode().including('fast_run')
     self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
     self.a = tensor.tensor(dtype=dtype, broadcastable=())
     self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.Aval = numpy.ones((2,3), dtype=dtype)
     self.xval = numpy.asarray([1,2], dtype=dtype)
     self.yval = numpy.asarray([1.5,2.7,3.9], dtype=dtype)

 def setUp(self, dtype='float64'):
     # This tests can run even when theano.config.blas.ldflags is empty.
     self.dtype = dtype
     self.mode = theano.compile.get_default_mode().including('fast_run')
     self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
     self.a = tensor.tensor(dtype=dtype, broadcastable=())
     self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.Aval = np.ones((2, 3), dtype=dtype)
     self.xval = np.asarray([1, 2], dtype=dtype)
     self.yval = np.asarray([1.5, 2.7, 3.9], dtype=dtype)

 def setUp(self):
     self.mode = mode_with_gpu
     dtype = self.dtype = 'float32'  # optimization isn't dtype-dependent
     self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
     self.a = tensor.tensor(dtype=dtype, broadcastable=())
     self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
     # data on the gpu make the op always inplace
     self.ger = gpu_ger_inplace
     self.ger_destructive = gpu_ger_inplace
     self.gemm = tcn.blas.gpu_gemm_inplace

 def setUp(self):
     self.mode = theano.compile.get_default_mode().including('fast_run')
     dtype = self.dtype = 'float64'  # optimization isn't dtype-dependent
     self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
     self.a = tensor.tensor(dtype=dtype, broadcastable=())
     self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.Aval = numpy.ones((2,3), dtype=dtype)
     self.xval = numpy.asarray([1,2], dtype=dtype)
     self.yval = numpy.asarray([1.5,2.7,3.9], dtype=dtype)
     if not theano.tensor.blas_scipy.optimizations_enabled:
         self.SkipTest()

 def test_numpy_2d(self):
     for shp0 in [(2, 3)]:
         x = tensor.tensor(dtype="floatX", broadcastable=(False,) * len(shp0))
         a = numpy.asarray(self.rng.rand(*shp0)).astype(config.floatX)
         for shp1 in [(6, 7)]:
             if len(shp0) + len(shp1) == 2:
                 continue
             y = tensor.tensor(dtype="floatX", broadcastable=(False,) * len(shp1))
             f = function([x, y], kron(x, y))
             b = self.rng.rand(*shp1).astype(config.floatX)
             out = f(a, b)
             assert numpy.allclose(out, numpy.kron(a, b))

    def setUp(self, dtype="float64"):
        if theano.config.blas.ldflags == "":
            raise SkipTest("This test is useful only when Theano" " is directly linked to blas.")

        self.dtype = dtype
        self.mode = theano.compile.get_default_mode().including("fast_run")
        self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
        self.a = tensor.tensor(dtype=dtype, broadcastable=())
        self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
        self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
        self.Aval = numpy.ones((2, 3), dtype=dtype)
        self.xval = numpy.asarray([1, 2], dtype=dtype)
        self.yval = numpy.asarray([1.5, 2.7, 3.9], dtype=dtype)

 def setUp(self):
     self.mode = theano.compile.get_default_mode()
     self.mode = self.mode.including("fast_run")
     self.mode = self.mode.excluding("c_blas")  # c_blas trumps scipy Ops
     dtype = self.dtype = "float64"  # optimization isn't dtype-dependent
     self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False))
     self.a = tensor.tensor(dtype=dtype, broadcastable=())
     self.x = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.y = tensor.tensor(dtype=dtype, broadcastable=(False,))
     self.Aval = numpy.ones((2, 3), dtype=dtype)
     self.xval = numpy.asarray([1, 2], dtype=dtype)
     self.yval = numpy.asarray([1.5, 2.7, 3.9], dtype=dtype)
     if not theano.tensor.blas_scipy.have_fblas:
         self.SkipTest()

 def make_node(self, x, gz):
     assert isinstance(x, Variable)
     assert isinstance(gz, Variable)
     gx = tensor(dtype=scal.upcast(gz.dtype, x.dtype),
                 broadcastable=x.broadcastable)
     op = self
     return Apply(op, [x, gz], [gx])

def test_incsub_f16():
    shp = (3, 3)
    shared = gpuarray_shared_constructor
    xval = np.arange(np.prod(shp), dtype='float16').reshape(shp) + 1
    yval = np.empty((2,) + shp[1:], dtype='float16')
    yval[:] = 2
    x = shared(xval, name='x')
    y = tensor.tensor(dtype='float16',
                      broadcastable=(False,) * len(shp),
                      name='y')
    expr = tensor.advanced_inc_subtensor1(x, y, [0, 2])
    f = theano.function([y], expr, mode=mode_with_gpu)
    assert sum([isinstance(node.op, GpuAdvancedIncSubtensor1)
                for node in f.maker.fgraph.toposort()]) == 1
    rval = f(yval)
    rep = xval.copy()
    np.add.at(rep, [[0, 2]], yval)
    assert np.allclose(rval, rep)

    expr = tensor.inc_subtensor(x[1:], y)
    f = theano.function([y], expr, mode=mode_with_gpu)
    assert sum([isinstance(node.op, GpuIncSubtensor)
                for node in f.maker.fgraph.toposort()]) == 1
    rval = f(yval)
    rep = xval.copy()
    rep[1:] += yval
    assert np.allclose(rval, rep)

 def make_node(self, A, b):
     A = as_tensor_variable(A)
     b = as_tensor_variable(b)
     otype = tensor.tensor(
             broadcastable=b.broadcastable,
             dtype = (A*b).dtype)
     return Apply(self, [A,b], [otype])

 def make_node(self, A, b):
     assert imported_scipy, "Scipy not available. Scipy is needed for the Solve op"
     A = as_tensor_variable(A)
     b = as_tensor_variable(b)
     assert A.ndim == 2
     assert b.ndim in [1, 2]
     otype = tensor.tensor(broadcastable=b.broadcastable, dtype=(A * b).dtype)
     return Apply(self, [A, b], [otype])

    def test_perform(self):
        if not imported_scipy:
            raise SkipTest('kron tests need the scipy package to be installed')

        for shp0 in [(2,), (2, 3), (2, 3, 4), (2, 3, 4, 5)]:
            for shp1 in [(6,), (6, 7), (6, 7, 8), (6, 7, 8, 9)]:
                if len(shp0) + len(shp1) == 2:
                    continue
                x = tensor.tensor(dtype='floatX',
                                  broadcastable=(False,) * len(shp0))
                y = tensor.tensor(dtype='floatX',
                                  broadcastable=(False,) * len(shp1))
                f = function([x, y], kron(x, y))
                a = numpy.asarray(self.rng.rand(*shp0)).astype(config.floatX)
                b = self.rng.rand(*shp1).astype(config.floatX)
                out = f(a, b)
                assert numpy.allclose(out, scipy.linalg.kron(a, b))

    def test_infer_shape(self):
        image = tensor.dtensor4()
        maxout = tensor.dtensor4()
        gz = tensor.dtensor4()
        rng = numpy.random.RandomState(utt.fetch_seed())
        maxpoolshps = ((1, 1), (2, 2), (3, 3), (2, 3), (3, 2))

        image_val = rng.rand(4, 6, 7, 9)
        out_shapes = [[[[4, 6, 7, 9], [4, 6, 7, 9]],
                       [[4, 6, 3, 4], [4, 6, 4, 5]],
                       [[4, 6, 2, 3], [4, 6, 3, 3]],
                       [[4, 6, 3, 3], [4, 6, 4, 3]],
                       [[4, 6, 2, 4], [4, 6, 3, 5]]],
                      [[None, None],
                       [[4, 6, 4, 5], None],
                       [[4, 6, 3, 3], None],
                       [[4, 6, 4, 3], None],
                       [[4, 6, 3, 5], None]],
                      [[None, None],
                       [None, None],
                       [[4, 6, 3, 4], None],
                       [[4, 6, 4, 4], None],
                       [None, None]]]

        for i, maxpoolshp in enumerate(maxpoolshps):
            for j, ignore_border in enumerate([True, False]):
                for k, padding in enumerate([(0, 0), (1, 1), (1, 2)]):
                    if out_shapes[k][i][j] is None:
                        continue
                    # checking shapes generated by DownsampleFactorMax
                    self._compile_and_check([image],
                                            [DownsampleFactorMax(maxpoolshp,
                                                                 ignore_border=ignore_border,
                                                                 padding=padding)(image)],
                                            [image_val], DownsampleFactorMax)

                    # checking shapes generated by MaxPoolGrad
                    maxout_val = rng.rand(*out_shapes[k][i][j])
                    gz_val = rng.rand(*out_shapes[k][i][j])
                    self._compile_and_check([image, maxout, gz],
                                            [MaxPoolGrad(maxpoolshp,
                                                         ignore_border=ignore_border,
                                                         padding=padding)
                                            (image, maxout, gz)],
                                            [image_val, maxout_val, gz_val],
                                            MaxPoolGrad,
                                            warn=False)
        # checking with broadcastable input
        image = tensor.tensor(dtype='float64',
                              broadcastable=(False, False, True, True))
        image_val = rng.rand(4, 6, 1, 1)
        self._compile_and_check(
            [image],
            [DownsampleFactorMax((2, 2),
                                 ignore_border=True,
                                 padding=(0, 0))(image)],
            [image_val], DownsampleFactorMax)

 def make_node(self, input, axis=-1):
     input = theano.tensor.as_tensor_variable(input)
     if axis is None:
         axis = theano.Constant(theano.gof.generic, None)
         # axis=None flattens the array before sorting
         out_type = tensor(dtype=input.dtype, broadcastable=[False])
     else:
         axis = theano.tensor.as_tensor_variable(axis)
         out_type = input.type()
     return theano.Apply(self, [input, axis], [out_type])

def test_local_dimshuffle_subtensor():

    dimshuffle_subtensor = out2in(local_dimshuffle_subtensor)

    x = tensor.dtensor4('x')
    x = tensor.patternbroadcast(x, (False, True, False, False))
    i = tensor.iscalar('i')

    out = x[:, :, 10:30, ::i].dimshuffle(0, 2, 3)

    g = FunctionGraph([x, i], [out])
    dimshuffle_subtensor(g)

    topo = g.toposort()
    assert any([not isinstance(x, DimShuffle) for x in topo])

    # Test dimshuffle remove dimensions the subtensor don't "see".
    x = tensor.tensor(broadcastable=(False, True, False), dtype='float64')
    out = x[i].dimshuffle(1)

    g = FunctionGraph([x, i], [out])
    dimshuffle_subtensor(g)

    topo = g.toposort()
    assert any([not isinstance(x, DimShuffle) for x in topo])

    # Test dimshuffle remove dimensions the subtensor don't "see" but
    # have in between dimensions.
    x = tensor.tensor(broadcastable=(False, True, False, True),
                      dtype='float64')
    out = x[i].dimshuffle(1)

    f = theano.function([x, i], out)

    topo = f.maker.fgraph.toposort()
    assert any([not isinstance(x, DimShuffle) for x in topo])
    assert f(np.random.rand(5, 1, 4, 1), 2).shape == (4,)

    # Test a corner case that had Theano return a bug.
    x = tensor.dtensor4('x')
    x = tensor.patternbroadcast(x, (False, True, False, False))

    assert x[:,:, 0:3, ::-1].dimshuffle(0,2,3).eval({x: np.ones((5, 1, 6, 7))}).shape == (5, 3, 7)

    def check_format_ndim(format, ndim):
        x = tensor.tensor(dtype=config.floatX, broadcastable=([False] * ndim), name="x")

        s = SparseFromDense(format)(x)
        s_m = -s
        d = dense_from_sparse(s_m)
        c = d.sum()
        g = tensor.grad(c, x)
        f = theano.function([x], [s, g])
        f(numpy.array(0, dtype=config.floatX, ndmin=ndim))
        f(numpy.array(7, dtype=config.floatX, ndmin=ndim))