def test_DownsampleFactorMaxPaddingStride_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        # maxpool, stride, padding, input sizes
        examples = (
            ((10,), (5,), (3,), (2,)),
            ((10,), (5,), (3,), (2, 2)),
            ((10,), (5,), (3,), (1, 1, 2)),
            ((10, 10), (5, 3), (3, 2), (1, 1, 2, 2)),
            ((10, 5), (3, 5), (2, 3), (1, 1, 2, 1)),
            ((5, 5), (3, 3), (3, 3), (1, 1, 2, 2)),
            ((5, 5, 5), (3, 3, 3), (3, 3, 3), (1, 1, 2, 2, 2)),
        )
        # average_inc_pad and average_exc_pad do not
        # support grad with padding
        for mode in ['max', 'sum']:
            for example in examples:
                (maxpoolshp, stridesize, paddingsize, inputsize) = example
                imval = rng.rand(*inputsize) * 10.0

                def mp(input):
                    return Pool(
                        ndim=len(maxpoolshp),
                        ignore_border=True,
                        mode=mode,
                        )(input, maxpoolshp, stridesize, paddingsize)
                utt.verify_grad(mp, [imval], rng=rng)

def test_pseudoinverse_grad():
    rng = np.random.RandomState(utt.fetch_seed())
    d1 = rng.randint(4) + 2
    d2 = rng.randint(4) + 2
    r = rng.randn(d1, d2).astype(theano.config.floatX)

    utt.verify_grad(pinv, [r])

    def test_AveragePoolGrad_grad_st_extra(self):
        """checks the gradient of the gradient for the case that
        stride is used for extra examples"""
        rng = numpy.random.RandomState(utt.fetch_seed())
        avgpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9))
        stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1),
                       (2, 3), (10, 10), (1, 1))
        imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5),
                   (8, 5), (8, 5), (8, 5))

        for indx in numpy.arange(len(avgpoolshps)):
            imvsize = imvsizs[indx]
            imval = rng.rand(1, 2, imvsize[0], imvsize[1])
            stride = stridesizes[indx]
            avgpoolshp = avgpoolshps[indx]
            for ignore_border in [True, False]:
                for mode in ['sum', 'average_inc_pad', 'average_exc_pad']:
                    grad_shape = Pool.out_shape(
                        imval.shape, avgpoolshp,
                        ignore_border=ignore_border, st=stride)
                    grad_val = rng.rand(*grad_shape)

                    def mp(input, grad):
                        grad_op = AveragePoolGrad(
                            avgpoolshp, ignore_border=ignore_border,
                            st=stride, mode=mode)
                        return grad_op(input, grad)

                    # skip the grad verification when the output is empty
                    if numpy.prod(grad_shape) == 0:
                        continue
                    utt.verify_grad(mp, [imval, grad_val], rng=rng)

    def test_verify_grad_with_zeros(self):
        # including zeros, as the case with zeros is important
        # (and special cases: 1 zero in the row, more than 1 zero in the row)
        x_val = numpy.asarray([[1., 2., 3.], [0., 5., 6.], [0., 0., 9.]],
             dtype='float32')
        x = theano.tensor.dmatrix()

        # sanity check
        x2 = theano.tensor.dmatrix()
        p = Prod(axis=1)(x)
        p2 = Prod(axis=1)(x2)
        fn = theano.function([x, x2], [p - p2], mode=self.mode)
        #print "hand computed diff for each row"
        x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [
            0., 0., 9.01]])
        #print fn(x_val, x2_val)
        fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)],
             mode=self.mode)
        #print "real grad"
        #print fn2(x_val)
        fn3 = theano.function([x], [p], mode=self.mode)
        assert numpy.allclose(fn3(x_val), [6., 0., 0.])

        # now with verify_grad
        unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)

    def test_maxpool(self):
        # generate flatted images
        maxpoolshps = ((2, 2), (3, 3), (4, 4), (5, 5), (6, 6))
        imval = numpy.random.rand(4, 5, 10, 10)

        images = tensor.dmatrix()
        for maxpoolshp in maxpoolshps:

            # symbolic stuff
            output, outshp = sp.max_pool(images, imval.shape[1:], maxpoolshp)
            f = function([images], [output])
            output_val = f(imval.reshape(imval.shape[0], -1))

            # numeric verification
            my_output_val = numpy.zeros(
                (imval.shape[0], imval.shape[1], imval.shape[2] / maxpoolshp[0], imval.shape[3] / maxpoolshp[1])
            )
            assert numpy.prod(my_output_val.shape[1:]) == numpy.prod(numpy.r_[imval.shape[1], outshp])

            for n in range(imval.shape[0]):
                for k in range(imval.shape[1]):
                    for i in range(imval.shape[2] / maxpoolshp[0]):
                        for j in range(imval.shape[3] / maxpoolshp[1]):
                            ii, jj = i * maxpoolshp[0], j * maxpoolshp[1]
                            patch = imval[n, k, ii : ii + maxpoolshp[0], jj : jj + maxpoolshp[1]]
                            my_output_val[n, k, i, j] = numpy.max(patch)
            my_output_val = my_output_val.reshape(imval.shape[0], -1)
            assert numpy.all(output_val == my_output_val)

            def mp(input):
                output, outshp = sp.max_pool(input, imval.shape[1:], maxpoolshp)
                return output

            utt.verify_grad(mp, [imval.reshape(imval.shape[0], -1)])

    def test_AveragePoolPaddingStride_grad_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        imgsizes = ((10, 10), (10, 5), (5, 5))
        avgpoolsizes = ((5, 3), (3, 5), (3, 3))
        stridesizes = ((3, 2), (2, 3), (3, 3))
        paddingsizes = ((2, 2), (2, 1), (2, 2))

        for i in range(len(imgsizes)):
            imgsize = imgsizes[i]
            imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
            avgpoolsize = avgpoolsizes[i]
            stridesize = stridesizes[i]
            paddingsize = paddingsizes[i]

            # 'average_exc_pad' with non-zero padding is not implemented
            for mode in ['sum', 'average_inc_pad']:
                grad_shape = DownsampleFactorMax.out_shape(imval.shape,
                                                           avgpoolsize, st=stridesize,
                                                           ignore_border=True, padding=paddingsize)
                grad_val = rng.rand(*grad_shape) * 10.0

                def mp(input, grad):
                    grad_op = AveragePoolGrad(avgpoolsize, ignore_border=True,
                                              st=stridesize, padding=paddingsize,
                                              mode=mode)
                    return grad_op(input, grad)
                utt.verify_grad(mp, [imval, grad_val], rng=rng)

    def test_DownsampleFactorMaxGrad_grad_st_extra(self):
        """checks the gradient of the gradient for the case that
        stride is used for extra examples"""
        rng = numpy.random.RandomState(utt.fetch_seed())
        maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9))
        stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1))
        imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5))

        for indx in numpy.arange(len(maxpoolshps)):
            imvsize = imvsizs[indx]
            imval = rng.rand(1, 2, imvsize[0], imvsize[1])
            stride = stridesizes[indx]
            maxpoolshp = maxpoolshps[indx]
            for ignore_border in [True, False]:
                grad_shape = DownsampleFactorMax.out_shape(
                    imval.shape, maxpoolshp, ignore_border=ignore_border, st=stride
                )
                grad_val = rng.rand(*grad_shape)

                def mp(input, grad):
                    out = DownsampleFactorMax(maxpoolshp, ignore_border=ignore_border, st=stride)(input)
                    grad_op = DownsampleFactorMaxGrad(maxpoolshp, ignore_border=ignore_border, st=stride)
                    return grad_op(input, out, grad)

                # skip the grad verification when the output is empty
                if numpy.prod(grad_shape) == 0:
                    continue
                utt.verify_grad(mp, [imval, grad_val], rng=rng)

 def test_batched_dot_gradient(self):
     for threshold in [0, 100]:
         unittest_tools.verify_grad(
             GpuBatchedDot(stream_threshold=threshold),
             [numpy.random.randn(5,7,2).astype(numpy.float32),
              numpy.random.randn(5,2,6).astype(numpy.float32)],
             mode=mode_with_gpu)

    def test_verify_grad_gauntlet(self):
        maxpoolshps = ((1, 1), (3, 3), (5, 3),)
        stridesizes = ((1, 1), (3, 3), (5, 7),)
        # generate random images
        imval = self.rng.rand(4, 10, 16, 16)

        for index_type, index_scope, maxpoolshp, stride, ignore_border in product(['flattened',
                                                        'array'],
                                                        ['local'],
                                                       maxpoolshps,
                                                       stridesizes,
                                                       [True, False]):
                                                 
                unpoolswitch_op = UnpoolSwitch(ds = maxpoolshp,
                                               st=stride, index_type=index_type, 
                                               index_scope=index_scope)
                                               
                if(index_type == 'flattened'):                                               
                    def op_with_fixed_switchs(x):
                        x_with_zero_switchs = T.concatenate((x, T.zeros_like(x)), 1)
                        return unpoolswitch_op(x_with_zero_switchs)
                else:
                    def op_with_fixed_switchs(x):
                        x_with_zero_switchs = T.concatenate((x, T.zeros_like(x), T.zeros_like(x)), 1)
                        return unpoolswitch_op(x_with_zero_switchs)

                utt.verify_grad(op_with_fixed_switchs, [imval], rng=self.rng)

    def test_max_pool_3d_3D(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        maxpoolshps = ((1, 1, 1), (3, 2, 1))
        imval = rng.rand(4, 5, 6)
        images = tensor.dtensor3()

        for maxpoolshp, ignore_border, mode in product(maxpoolshps,
                                                       [True, False],
                                                       ['max', 'sum',
                                                        'average_inc_pad',
                                                        'average_exc_pad']):
                # print 'maxpoolshp =', maxpoolshp
                # print 'ignore_border =', ignore_border
                numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp,
                                                          ignore_border,
                                                          mode=mode)
                output = pool_3d(images, maxpoolshp, ignore_border,
                                 mode=mode)
                output_val = function([images], output)(imval)
                utt.assert_allclose(output_val, numpy_output_val)

                def mp(input):
                    return pool_3d(input, maxpoolshp, ignore_border,
                                   mode=mode)
                utt.verify_grad(mp, [imval], rng=rng)

    def test_DownsampleFactorMaxGradGrad_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        imgsizes = ((10, 10), (10, 5), (5, 5))
        maxpoolsizes = ((5, 3), (3, 5), (3, 3))
        stridesizes = ((3, 2), (2, 3), (3, 3))
        paddingsizes = ((2, 2), (2, 1), (2, 2))

        for i in range(len(imgsizes)):
            imgsize = imgsizes[i]
            imval1 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
            imval2 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
            maxpoolsize = maxpoolsizes[i]
            stridesize = stridesizes[i]
            paddingsize = paddingsizes[i]

            def mp(input1, input2):
                pooled_out = Pool(
                    maxpoolsize, ignore_border=True,
                    st=stridesize,
                    padding=paddingsize,
                    )(input1)
                out = DownsampleFactorMaxGradGrad(
                    ds=maxpoolsize,
                    ignore_border=True,
                    st=stridesize,
                    padding=paddingsize)(input1, pooled_out, input2)
                return out
            utt.verify_grad(mp, [imval1, imval2], rng=rng)

    def test_DownsampleFactorMaxGradGrad_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        # maxpool, stride, padding, input sizes
        examples = (
            ((3,), (2,), (2,), (10,)),
            ((3,), (2,), (2,), (2, 10,)),
            ((3,), (2,), (2,), (2, 1, 10,)),
            ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
            ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
            ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)),
            ((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)),
            ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)),
            ((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)),
            ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)),
        )

        for (maxpoolshp, stridesize, paddingsize, inputsize) in examples:
            imval1 = rng.rand(*inputsize) * 10.0
            imval2 = rng.rand(*inputsize) * 10.0

            def mp(input1, input2):
                op1 = Pool(ndim=len(maxpoolshp), ignore_border=True)
                pooled_out = op1(input1, maxpoolshp, stridesize, paddingsize)
                op2 = DownsampleFactorMaxGradGrad(
                    ndim=len(maxpoolshp),
                    ignore_border=True)
                out = op2(input1, pooled_out, input2, maxpoolshp, stridesize, paddingsize)
                return out
            utt.verify_grad(mp, [imval1, imval2], rng=rng)

    def test_AveragePoolPaddingStride_grad_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        # avgpool, stride, padding, input sizes
        examples = (
            ((3,), (2,), (2,), (10,)),
            ((3,), (2,), (2,), (2, 10,)),
            ((3,), (2,), (2,), (2, 1, 10,)),
            ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
            ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
            ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)),
            ((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)),
            ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)),
            ((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)),
            ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)),
        )

        for (avgpoolshp, stridesize, paddingsize, inputsize) in examples:
            imval = rng.rand(*inputsize) * 10.0

            # 'average_exc_pad' with non-zero padding is not implemented
            for mode in ['sum', 'average_inc_pad']:
                grad_shape = Pool.out_shape(imval.shape,
                                            avgpoolshp,
                                            ndim=len(avgpoolshp),
                                            st=stridesize,
                                            ignore_border=True,
                                            padding=paddingsize)
                grad_val = rng.rand(*grad_shape) * 10.0

                def mp(input, grad):
                    grad_op = AveragePoolGrad(ndim=len(avgpoolshp),
                                              ignore_border=True,
                                              mode=mode)
                    return grad_op(input, grad, avgpoolshp, stridesize, paddingsize)
                utt.verify_grad(mp, [imval, grad_val], rng=rng)

    def test_DownsampleFactorMax_grad(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        # maxpool, input sizes
        examples = (
            ((2,), (3,)),
            ((2,), (2, 3)),
            ((2,), (2, 3, 3)),
            ((1, 1), (2, 3, 3, 4)),
            ((3, 2), (2, 3, 3, 4)),
            ((2, 3), (2, 3, 3, 4)),
            ((1, 1, 1), (2, 3, 3)),
            ((3, 2, 2), (2, 3, 3, 4)),
            ((2, 3, 2), (2, 3, 3, 4, 4)),
            ((2, 2, 3), (2, 3, 3, 4, 4)),
        )

        for example, ignore_border, mode in product(examples,
                                                    [True, False],
                                                    ['max',
                                                     'sum',
                                                     'average_inc_pad',
                                                     'average_exc_pad']):
            (maxpoolshp, inputsize) = example
            imval = rng.rand(*inputsize) * 10.0

            # more variance means numeric gradient will be more accurate
            def mp(input):
                return Pool(ndim=len(maxpoolshp),
                            ignore_border=ignore_border,
                            mode=mode)(input, maxpoolshp)
            utt.verify_grad(mp, [imval], rng=rng)

    def test_1Drfft(self):
        inputs_val = np.random.random((1, N)).astype(theano.config.floatX)

        x = T.matrix('x')
        rfft = fft.rfft(x)
        f_rfft = theano.function([x], rfft)
        res_rfft = f_rfft(inputs_val)
        res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) +
                         1j * np.asarray(res_rfft[:, :, 1]))

        rfft_ref = np.fft.rfft(inputs_val, axis=1)

        utt.assert_allclose(rfft_ref, res_rfft_comp)

        m = rfft.type()
        print(m.ndim)
        irfft = fft.irfft(m)
        f_irfft = theano.function([m], irfft)
        res_irfft = f_irfft(res_rfft)

        utt.assert_allclose(inputs_val, np.asarray(res_irfft))

        # The numerical gradient of the FFT is sensitive, must set large
        # enough epsilon to get good accuracy.
        eps = 1e-1

        def f_rfft(inp):
            return fft.rfft(inp)
        inputs_val = np.random.random((1, N)).astype(theano.config.floatX)
        utt.verify_grad(f_rfft, [inputs_val], eps=eps)

        def f_irfft(inp):
            return fft.irfft(inp)
        inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX)
        utt.verify_grad(f_irfft, [inputs_val], eps=eps)

def test_light_curve_grad():
    u_val = np.array([0.2, 0.3, 0.1, 0.5])
    b_val = np.linspace(-1.5, 1.5, 20)
    r_val = 0.1 + np.zeros_like(b_val)

    lc = lambda u, b, r: StarryLightCurve(u)._compute_light_curve(b, r)  # NOQA
    utt.verify_grad(lc, [u_val, b_val, r_val])

    def test_DownsampleFactorMaxGrad_grad_st(self):
        """checks the gradient of the gradient for
        the case that stride is used"""
        rng = numpy.random.RandomState(utt.fetch_seed())
        maxpoolshps = ((1, 1), (3, 3), (5, 3))
        stridesizes = ((1, 1), (3, 3), (5, 7))
        imval = rng.rand(1, 2, 16, 16)

        for maxpoolshp in maxpoolshps:
            for ignore_border in [True, False]:
                for stride in stridesizes:
                    grad_shape = DownsampleFactorMax.out_shape(
                        imval.shape, maxpoolshp,
                        ignore_border=ignore_border, st=stride)
                    grad_val = rng.rand(*grad_shape)

                    def mp(input, grad):
                        out = DownsampleFactorMax(
                            maxpoolshp, ignore_border=ignore_border,
                            st=stride)(input)
                        grad_op = MaxPoolGrad(
                            maxpoolshp, ignore_border=ignore_border,
                            st=stride)
                        return grad_op(input, out, grad)

                    utt.verify_grad(mp, [imval, grad_val], rng=rng)

    def test_verify_grad_with_zeros(self):
        # including zeros, as the case with zeros is important
        # (and special cases: 1 zero in the row, more than 1 zero in the row)
        x_val = numpy.asarray([[1.0, 2.0, 3.0], [0.0, 5.0, 6.0], [0.0, 0.0, 9.0]], dtype="float32")
        x = theano.tensor.dmatrix()

        # sanity check
        p = Prod(axis=1)(x)

        # Uncomment this for debugging if needed
        # x2 = theano.tensor.dmatrix()
        # p2 = Prod(axis=1)(x2)
        # fn = theano.function([x, x2], [p - p2], mode=self.mode)
        # print("hand computed diff for each row")
        # x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [
        #     0., 0., 9.01]])
        # print(fn(x_val, x2_val))
        # fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)],
        #                       mode=self.mode)
        # print("real grad")
        # print(fn2(x_val))
        fn3 = theano.function([x], [p], mode=self.mode)
        assert numpy.allclose(fn3(x_val), [6.0, 0.0, 0.0])

        # now with verify_grad
        unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)

    def test_max_pool_2d_2D(self):
        rng = numpy.random.RandomState(utt.fetch_seed())
        maxpoolshps = ((1, 1), (3, 2))
        imval = rng.rand(4, 5)
        images = tensor.dmatrix()

        for maxpoolshp, ignore_border, mode in product(maxpoolshps,
                                                       [True, False],
                                                       ['max', 'sum',
                                                        'average_inc_pad',
                                                        'average_exc_pad']):
                # print 'maxpoolshp =', maxpoolshp
                # print 'ignore_border =', ignore_border
                numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
                                                          ignore_border,
                                                          mode=mode)
                output = max_pool_2d(images, maxpoolshp, ignore_border,
                                     mode=mode)
                output_val = function([images], output)(imval)
                assert numpy.all(output_val == numpy_output_val), (
                    "output_val is %s, numpy_output_val is %s"
                    % (output_val, numpy_output_val))

                def mp(input):
                    return max_pool_2d(input, maxpoolshp, ignore_border,
                                       mode=mode)
                utt.verify_grad(mp, [imval], rng=rng)

    def run_fwd(
        self,
        inputs_shape,
        filters_shape,
        ref=dnn_conv,
        subsample=(1, 1),
        verify_grad=True,
        mode=mode_without_gpu,
        border_mode="valid",
        filter_flip=True,
        device="cpu",
        provide_shape=False,
        target_op=None,
    ):

        inputs_val = numpy.random.random(inputs_shape).astype("float32")
        filters_val = numpy.random.random(filters_shape).astype("float32")
        if device == "gpu":
            inputs = gpu_shared(inputs_val)
            filters = gpu_shared(filters_val)
        else:
            inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val))
            filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val))
        if provide_shape:
            imshp = inputs_shape
            kshp = filters_shape
        else:
            imshp = None
            kshp = None
        if filter_flip:
            conv_mode = "conv"
        else:
            conv_mode = "cross"

        c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode)
        c = conv.conv2d(
            inputs,
            filters,
            border_mode=border_mode,
            subsample=subsample,
            filter_flip=filter_flip,
            input_shape=imshp,
            filter_shape=kshp,
        )
        f_ref = theano.function([], c_ref, mode=mode)
        f = theano.function([], c, mode)

        if target_op is not None:
            assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()])

        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, "trace"))
        res_ref = numpy.array(f_ref())
        res = numpy.array(f())
        utt.assert_allclose(res_ref, res)
        if verify_grad:
            utt.verify_grad(
                conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample),
                [inputs_val, filters_val],
                mode=mode,
            )