def gpu_to_cpu_model(model):
    for layer in model.layers:
        for member, value in layer.__dict__.items():
            if is_shared_var(value):
                layer.__dict__[member] = T._shared(np.array(value.get_value(), floatX),
                                          name=value.name, borrow=False)
        for i in xrange(len(layer.params)):
            if is_shared_var(layer.params[i]):
                layer.params[i] = T._shared(np.array(layer.params[i].get_value(), floatX),
                                          name=layer.params[i].name, borrow=False)
    return model

    def test_gemv2(self):
        ''' test vector1+dot(vector2,matrix) '''
        v1 = theano.shared(numpy.array(numpy.random.rand(5), dtype='float32'))
        v2 = tensor._shared(numpy.array(numpy.random.rand(2), dtype='float32'))
        m = theano.shared(numpy.array(numpy.random.rand(5, 2),
                                      dtype='float32'))

        no_gpu_f = theano.function([], v2 + theano.dot(v1, m),
                                   mode=mode_without_gpu)
        gpu_f = theano.function([], v2 + theano.dot(v1, m),
                                mode=mode_with_gpu)
        # gpu_f2 is needed to test the case when the input is not on the gpu
        # but the output is moved to the gpu.
        gpu_f2 = theano.function(
            [], tcn.gpu_from_host(v2 + theano.dot(v1, m)),
            mode=mode_with_gpu)

        # Assert they produce the same output
        assert numpy.allclose(no_gpu_f(), gpu_f(), atol=self.atol)
        assert numpy.allclose(no_gpu_f(), gpu_f2(), atol=self.atol)
        # Assert that the gpu version actually uses gpu
        assert sum([node.op is gpu_gemv_inplace for node in
                    gpu_f2.maker.fgraph.toposort()]) == 1
        assert sum([node.op is gpu_gemv_inplace for node in
                    gpu_f.maker.fgraph.toposort()]) == 1

def sharedX(value, name=None, borrow=True, keep_on_cpu=False):
    """ Transform value into a shared variable of type floatX """
    if keep_on_cpu:
        return T._shared(theano._asarray(value, dtype=theano.config.floatX),
                         name=name,
                         borrow=borrow)

    return theano.shared(theano._asarray(value, dtype=theano.config.floatX),
                         name=name,
                         borrow=borrow)

def test_local_gpu_subtensor():
    # Test shared forced on CPU.
    t = tensor._shared(np.zeros(20, "float32"))
    f = theano.function([], t[3:4], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test graph input.
    t = tensor.fmatrix()
    f = theano.function([t], t[3:4], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test multiple use of the input
    # We want the subtensor to be on the GPU to prevent multiple transfer.
    t = tensor.fmatrix()
    f = theano.function([t], [t[3:4], t + 1], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert not any([type(node.op) is tensor.Subtensor for node in topo])
    assert any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test multiple use of the input + input as output
    # We want the subtensor to be on the GPU to prevent multiple transfer.
    t = tensor.fmatrix()
    f = theano.function([t], [t[3:4], t + 1, t], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert not any([type(node.op) is tensor.Subtensor for node in topo])
    assert any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test shared forced on CPU end we do computation on the output of
    # the subtensor.
    t = tensor._shared(np.zeros(20, "float32"))
    f = theano.function([], t[3:4] + 1, mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])
    # Our optimizer isn't smart enough to move to the GPU Elemwise.
    # If it where just a little bit smarter, it could wrongly move it to the GPU.
    # If it where super smart, it would know it should not move it to the GPU.
    assert any([isinstance(node.op, tensor.Elemwise) for node in topo])

def test_local_gpu_subtensor():
    # Test shared forced on CPU.
    t = tensor._shared(numpy.zeros(20, "float32"))
    f = theano.function([], t[3:4], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test graph input.
    t = tensor.fmatrix()
    f = theano.function([t], t[3:4], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test multiple use of the input
    # We want the subtensor to be on the GPU to prevent multiple transfer.
    t = tensor.fmatrix()
    f = theano.function([t], [t[3:4], t + 1], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert not any([type(node.op) is tensor.Subtensor for node in topo])
    assert any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test multiple use of the input + input as output
    # We want the subtensor to be on the GPU to prevent multiple transfer.
    t = tensor.fmatrix()
    f = theano.function([t], [t[3:4], t + 1, t], mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert not any([type(node.op) is tensor.Subtensor for node in topo])
    assert any([isinstance(node.op, GpuSubtensor) for node in topo])

    # Test shared forced on CPU end we do computation on the output of
    # the subtensor.
    t = tensor._shared(numpy.zeros(20, "float32"))
    f = theano.function([], t[3:4] + 1, mode=mode_with_gpu)
    topo = f.maker.fgraph.toposort()
    assert any([type(node.op) is tensor.Subtensor for node in topo])
    assert not any([isinstance(node.op, GpuSubtensor) for node in topo])
    assert any([isinstance(node.op, GpuElemwise) for node in topo])

def categorical_sampler(rstream, p, draw_shape, dtype="int32"):
    if not isinstance(p, theano.Variable):
        p = tensor._shared(numpy.asarray(p, dtype=theano.config.floatX))
    if p.ndim != 1:
        raise NotImplementedError()
    if draw_shape.ndim != 1:
        raise TypeError()
    op = Categorical(
        False, tensor.TensorType(broadcastable=(False,) * tensor.get_vector_length(draw_shape), dtype=dtype)
    )
    rstate = rstream.new_shared_rstate()
    new_rstate, out = op(rstate, p, draw_shape)
    rstream.add_default_update(out, rstate, new_rstate)
    return out

 def shared(self, x):
     return tensor._shared(x)

    def shared_dataset(data_xy):
        data_x, data_y = data_xy
        shared_x = T._shared(numpy.asarray(data_x, dtype=theano.config.floatX),borrow=True)
        shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX),borrow=True)

        return shared_x, T.cast(shared_y, 'int32')

import pynet.layer as layers

floatX = theano.config.floatX

parser = argparse.ArgumentParser(description='''Convert gpu pickle pynet model to cpu pickle pynet model''')
parser.add_argument('--gpu_model', metavar='Path', required=True, help='the path to the gpu model pickle file')
parser.add_argument('--cpu_model', metavar='Path', required=True, help='''path to save the cpu model pickle file''')
args = parser.parse_args()

print ('loading gpu autoencoder..')
fin = open(args.gpu_model)
gpu_model = cPickle.load(fin)

ae = AutoEncoder(input_dim=gpu_model.input_dim)
for layer in gpu_model.encode_layers:
    layerW = T._shared(np.array(layer.W.get_value(), floatX),
                        name=layer.W.name, borrow=False)
    layerb = T._shared(np.array(layer.b.get_value(), floatX),
                        name=layer.b.name, borrow=False)
    encode_layer = getattr(layers, layer.__class__.__name__)(dim=layer.dim, name=layer.name,
                                                            W=layerW, b=layerb)
    ae.add_encode_layer(encode_layer)
    print 'encode layer', encode_layer.name, encode_layer.dim
print 'encode layers', ae.encode_layers

for ae_layer, gpu_layer in zip(reversed(ae.encode_layers), gpu_model.decode_layers):
    gpu_decode_layer_b = T._shared(np.array(gpu_layer.b.get_value(), floatX),
                        name=gpu_layer.b.name, borrow=False)
    decode_layer = getattr(layers, gpu_layer.__class__.__name__)(name=gpu_layer.name, dim=gpu_layer.dim,
                                                                W=ae_layer.W.T, b=gpu_decode_layer_b)
    ae.add_decode_layer(decode_layer)
    print 'decode layer', decode_layer.name, decode_layer.dim

parser = argparse.ArgumentParser()
parser.add_argument("source", type=str, help="Pickled network to steal params from.")
parser.add_argument("dest", type=str, help="File to place new network in.")
parser.add_argument(
    "--cpu", "-c", dest="cpu", action="store_const", const=True, default=False, help="Convert network to run on a CPU."
)
args = parser.parse_args()

print "loading model..."
f = file(args.source, "rb")
old_network = cPickle.load(f)
f.close()

params = old_network.params
if args.cpu:
    print "converting gpu parameters..."
    new_params = []
    for param in params:
        param = T._shared(param.get_value())
        new_params.append(param)
    params = new_params

new_network = network(batch_size=None, params=params)

print "saving model..."
f = file(args.dest, "wb")
cPickle.dump(new_network, f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()

    def __init__(self, options, channel, data, model):
        """
        Parameters:
            options: Dictionary
            `options` is expected to contain the following keys:
                `cbs` -> int
                    Number of samples to consider at a time when computing
                    some property of the model
                `gbs` -> int
                    Number of samples over which to compute the gradients
                `mbs` -> int
                    Number of samples over which to compute the metric
                `ebs` -> int
                    Number of samples over which to evaluate the training
                    error
                `mreg` -> float
                    Regularization added to the metric
                `mrtol` -> float
                    Relative tolerance for inverting the metric
                `miters` -> int
                    Number of iterations
                `seed` -> int
                    Random number generator seed
                `profile` -> bool
                    Flag, if profiling should be on or not
                `verbose` -> int
                    Verbosity level
                `lr` -> float
                    Learning rate
            channel: jobman channel or None
            data: dictionary-like object return by numpy.load containing the
                data
            model : model
        """
        n_params = len(model.params)
        self.data = data

        if options['device'] != 'gpu':
            xdata = theano.shared(data['train_x'][:options['gbs']],
                                  name='xdata')
            ydata = TT._shared(data['train_y'][:options['gbs']],
                               name='ydata')
            self.xdata = xdata
            self.ydata = ydata
            shared_data = [xdata, ydata]
        else:
            self.cpu_shared_data = []
            xdata = theano.shared(data['train_x'], name='xdata')
            ydata = TT._shared(data['train_y'], name='ydata')
            self.xdata = xdata
            self.ydata = ydata
            shared_data = [xdata, ydata]

        self.rng = numpy.random.RandomState(options['seed'])
        n_samples = data['train_x'].shape[0]
        self.grad_batches = n_samples // options['gbs']
        self.metric_batches = n_samples // options['mbs']
        self.eval_batches = n_samples // options['ebs']

        self.verbose = options['verbose']
        if options['device'] != 'gpu':
            # Store eucledian gradients
            self.gs = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX))
                       for shp in model.params_shape]
            # Store riemannian gradients
            self.rs = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX))
                       for shp in model.params_shape]
        else:
            # Store eucledian gradients
            self.gs = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX))
                       for shp in model.params_shape]
            # Store riemannian gradients
            self.rs = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX))
                       for shp in model.params_shape]

        self.permg = self.rng.permutation(self.grad_batches)
        self.permr = self.rng.permutation(self.metric_batches)
        self.perme = self.rng.permutation(self.eval_batches)
        self.k = 0
        self.posg = 0
        self.posr = 0
        self.pose = 0

        # Step 1. Compile function for computing eucledian gradients

        # inputs
        gbdx = TT.iscalar('grad_batch_idx')
        print 'Constructing grad function'
        srng = RandomStreams(numpy.random.randint(1e5))
        loc_inputs = [x.type() for x in model.inputs]
        def grad_step(*args):
            idx = TT.cast(args[0], 'int32')
            nw_inps = [x[idx * options['cbs']: \
                         (idx + 1) * options['cbs']]
                       for x in loc_inputs]
            replace = dict(zip(model.inputs, nw_inps))
            nw_cost = safe_clone(model.train_cost, replace=replace)
            gs = TT.grad(nw_cost, model.params)
            nw_gs = [op + np for op, np in zip(args[1: 1 + n_params], gs)]
            return [args[0] + const(1)] + \
#.........这里部分代码省略.........

import pynet.layer as layers

floatX = theano.config.floatX

parser = argparse.ArgumentParser(description='''Convert gpu pickle pynet model to cpu pickle pynet model''')
parser.add_argument('--gpu_model', metavar='Path', required=True, help='the path to the gpu model pickle file')
parser.add_argument('--cpu_model', metavar='Path', required=True, help='''path to save the cpu model pickle file''')
args = parser.parse_args()

print ('loading gpu mlp..')
fin = open(args.gpu_model)
gpu_model = cPickle.load(fin)

mlp = MLP(input_dim=gpu_model.input_dim)
for layer in gpu_model.layers:
    layerW = T._shared(np.array(layer.W.get_value(), floatX),
                        name=layer.W.name, borrow=False)
    layerb = T._shared(np.array(layer.b.get_value(), floatX),
                        name=layer.b.name, borrow=False)
    mlp_layer = getattr(layers, layer.__class__.__name__)(dim=layer.dim, name=layer.name,
                                                            W=layerW, b=layerb)
    mlp.add_layer(mlp_layer)
    print 'mlp layer', mlp_layer.name, mlp_layer.dim
print 'layers', mlp.layers

fout = open(args.cpu_model, 'wb')
cPickle.dump(mlp, fout)
print ('Done!')
fin.close()
fout.close()

    def __init__(self,
                 options,
                 channel,
                 data,
                 model):
        """
        Parameters:
            options: Dictionary
            `options` is expected to contain the following keys:
                `cbs` -> int
                    Number of samples to consider at a time when computing
                    some property of the model
                `gbs` -> int
                    Number of samples over which to compute the gradients
                `mbs` -> int
                    Number of samples over which to compute the krylov
                    subspace
                `ebs` -> int
                    Number of samples over which to evaluate the training
                    error
                `seed` -> int
                    Random number generator seed
                `profile` -> bool
                    Flag, if profiling should be on or not
                `verbose` -> int
                    Verbosity level
                `lbfgsIters' -> int
                `krylovDim` -> int
            channel: jobman channel or None
            data: dictionary-like object return by numpy.load containing the
                data
            model : model
        """
        n_params = len(model.params)
        self.data = data
        xdata = theano.shared(data['train_x'],
                              name='xdata')
        ydata = theano.shared(data['train_y'],
                          name='ydata')
        self.xdata = xdata
        self.ydata = ydata
        shared_data = [xdata, ydata]
        self.rng = numpy.random.RandomState(options['seed'])
        n_samples = data['train_x'].shape[0]
        self.grad_batches = n_samples // options['gbs']
        self.metric_batches = n_samples // options['mbs']
        self.eval_batches = n_samples // options['ebs']

        self.verbose = options['verbose']
        rng = numpy.random.RandomState(options['seed'])
        self.rng = rng
        self.options = options
        self.channel = channel
        self.model = model
        n_dimensions = options['krylovDim']
        self.n_dimensions = n_dimensions
        if options['device']=='gpu':
            cfn_subspaces = \
                [theano.shared(numpy.zeros(
                                (n_dimensions,) + shp, dtype='float32'),
                               name='cfn{%s|%d}' % (str(param.name), i))
                 for i, (shp, param) in enumerate(zip(model.params_shape,
                                                      model.params))]
            old_deltas = \
                [theano.shared(numpy.zeros(shp, dtype='float32'),
                               name='delta{%s|%d}' % (str(param.name), i))
                 for i, (shp, param) in
                            enumerate(zip(model.params_shape, model.params))]
            self.gs = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX))
                   for shp in model.params_shape]
        else:
            cfn_subspaces = \
                [TT._shared(numpy.zeros(
                                (n_dimensions,) + shp, dtype='float32'),
                               name='cfn{%s|%d}' % (str(param.name), i))
                 for i, (shp, param) in enumerate(zip(model.params_shape,
                                                      model.params))]
            old_deltas = \
                [TT._shared(numpy.zeros(shp, dtype='float32'),
                               name='delta{%s|%d}' % (str(param.name), i))
                 for i, (shp, param) in
                            enumerate(zip(model.params_shape, model.params))]
            self.gs = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX))
                   for shp in model.params_shape]
        self.cfn_subspaces = cfn_subspaces
        self.old_deltas = old_deltas

        self.permg = self.rng.permutation(self.grad_batches)
        self.permr = self.rng.permutation(self.metric_batches)
        self.perme = self.rng.permutation(self.eval_batches)
        self.k = 0
        self.posg = 0
        self.posr = 0
        self.pose = 0

        # Step 1. Compile function for computing eucledian gradients
        print 'Constructing grad function'
        loc_inputs = [x.type(name='locx') for x in model.inputs]
        def grad_step(*args):
            idx = TT.cast(args[0], 'int32')
#.........这里部分代码省略.........

 def shared(self, x, name=None):
     return tensor._shared(x, name)

    def __init__(self, options, channel, data, model):
        """
        Parameters:
            options: Dictionary
            `options` is expected to contain the following keys:
                `cbs` -> int
                    Number of samples to consider at a time when computing
                    some property of the model
                `gbs` -> int
                    Number of samples over which to compute the gradients
                `mbs` -> int
                    Number of samples over which to compute the metric
                `ebs` -> int
                    Number of samples over which to evaluate the training
                    error
                `mreg` -> float
                    Regularization added to the metric
                `mrtol` -> float
                    Relative tolerance for inverting the metric
                `miters` -> int
                    Number of iterations
                `seed` -> int
                    Random number generator seed
                `profile` -> bool
                    Flag, if profiling should be on or not
                `verbose` -> int
                    Verbosity level
                `lr` -> float
                    Learning rate
            channel: jobman channel or None
            data: dictionary-like object return by numpy.load containing the
                data
            model : model
        """
        n_params = len(model.params)
        self.data = data
        self.model = model
        if options['device'] == 'gpu':
            xdata = theano.shared(data['train_x'], name='xdata')
            print_mem('xdata')
            self.ydata = TT._shared(data['train_y'], name='ydata')
            self.xdata = xdata
            self.shared_data = [xdata, self.ydata]
            self.cpu_shared_data = []
        else:
            xdata = theano.shared(data['train_x'][:options['gbs']],
                                  name='xdata')
            print_mem('xdata')
            self.ydata = TT._shared(data['train_y'][:options['gbs']],
                                    name='ydata')
            self.xdata = xdata
            self.shared_data = [xdata, self.ydata]
            cxdata = TT._shared(data['train_x'], name='cpu_xdata',
                                borrow=True)
            self.cydata = TT._shared(data['train_y'], name='cpu_ydata',
                                     borrow=True)
            cydata = TT.cast(self.cydata, 'int32')
            self.cxdata = cxdata
            self.cpu_shared_data = [cxdata, cydata]

        self.options = options

        self.rng = numpy.random.RandomState(options['seed'])
        n_samples = data['train_x'].shape[0]
        self.n_samples = n_samples
        self.grad_batches = n_samples // options['gbs']
        self.metric_batches = n_samples // options['mbs']
        self.eval_batches = n_samples // options['ebs']

        self.verbose = options['verbose']
        # Store eucledian gradients
        cst = time.time()
        if options['device'] == 'gpu':
            self.gs = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name ='g%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]
            # Store riemannian gradients
            self.rs = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name='r%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]
            # Store jacobi diagonal
            self.js = [theano.shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name='j%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]
        else:
            self.gs = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name ='g%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]
            # Store riemannian gradients
            self.rs = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name='r%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]
            # Store jacobi diagonal
            self.js = [TT._shared(numpy.zeros(shp, dtype=theano.config.floatX),
                                 name='j%d'%idx)
                       for idx, shp in enumerate(model.params_shape)]


        self.permg = self.rng.permutation(self.grad_batches)
        self.permr = self.rng.permutation(self.metric_batches)
#.........这里部分代码省略.........