parallel processing - Julia: How to copy data to another processor in Julia

Question

How do you move data from one processor to another in julia?

Say I have an array

a = [1:10]

Or some other data structure. What is the proper way to put it on all other available processors so that it will be available on those processors as the same variable name?

Answer 1

I didn't know how to do this at first, so I spent some time figuring it out.

Here are some functions I wrote to pass objects:

sendto

Send an arbitrary number of variables to specified processes.

New variables are created in the Main module on specified processes. The name will be the key of the keyword argument and the value will be the associated value.

function sendto(p::Int; args...)
    for (nm, val) in args
        @spawnat(p, eval(Main, Expr(:(=), nm, val)))
    end
end


function sendto(ps::Vector{Int}; args...)
    for p in ps
        sendto(p; args...)
    end
end

Examples

# creates an integer x and Matrix y on processes 1 and 2
sendto([1, 2], x=100, y=rand(2, 3))

# create a variable here, then send it everywhere else
z = randn(10, 10); sendto(workers(), z=z)

getfrom

Retrieve an object defined in an arbitrary module on an arbitrary process. Defaults to the Main module.

The name of the object to be retrieved should be a symbol.

getfrom(p::Int, nm::Symbol; mod=Main) = fetch(@spawnat(p, getfield(mod, nm)))

Examples

# get an object from named x from Main module on process 2. Name it x
x = getfrom(2, :x)

passobj

Pass an arbitrary number of objects from one process to arbitrary processes. The variable must be defined in the from_mod module of the src process and will be copied under the same name to the to_mod module on each target process.

function passobj(src::Int, target::Vector{Int}, nm::Symbol;
                 from_mod=Main, to_mod=Main)
    r = RemoteRef(src)
    @spawnat(src, put!(r, getfield(from_mod, nm)))
    for to in target
        @spawnat(to, eval(to_mod, Expr(:(=), nm, fetch(r))))
    end
    nothing
end


function passobj(src::Int, target::Int, nm::Symbol; from_mod=Main, to_mod=Main)
    passobj(src, [target], nm; from_mod=from_mod, to_mod=to_mod)
end


function passobj(src::Int, target, nms::Vector{Symbol};
                 from_mod=Main, to_mod=Main)
    for nm in nms
        passobj(src, target, nm; from_mod=from_mod, to_mod=to_mod)
    end
end

Examples

# pass variable named x from process 2 to all other processes
passobj(2, filter(x->x!=2, procs()), :x)

# pass variables t, u, v from process 3 to process 1
passobj(3, 1, [:t, :u, :v])

# Pass a variable from the `Foo` module on process 1 to Main on workers
passobj(1, workers(), [:foo]; from_mod=Foo)