func (l *LocalCluster) runDockerSpy() {
	l.panicOnStop()

	create := func() (*Container, error) {
		return createContainer(l,
			container.Config{
				Image: dockerspyImage + ":" + dockerspyTag,
				Cmd:   strslice.New("--dns-domain=" + domain),
			}, container.HostConfig{
				Binds:           []string{"/var/run/docker.sock:/var/run/docker.sock"},
				PublishAllPorts: true,
			},
			"docker-spy",
		)
	}
	c, err := create()
	if dockerclient.IsErrImageNotFound(err) {
		if err := pullImage(l, types.ImagePullOptions{ImageID: dockerspyImage, Tag: dockerspyTag}); err != nil {
			log.Fatal(err)
		}

		c, err = create()
	}
	maybePanic(err)
	maybePanic(c.Start())
	l.dns = c
	if ci, err := c.Inspect(); err != nil {
		log.Error(err)
	} else {
		log.Infof("started %s: %s", c.Name(), ci.NetworkSettings.IPAddress)
	}
}

// CreateContainer aka schedule a brand new container into the cluster.
func (c *Cluster) CreateContainer(config *cluster.ContainerConfig, name string, authConfig *types.AuthConfig) (*cluster.Container, error) {
	poolUUID := os.Getenv("CX_POOL")
	log.Debug("CX: Adding label pool=" + poolUUID)
	config.Labels["pool"] = poolUUID

	container, err := c.createContainer(config, name, false, authConfig)

	if err != nil {
		var retries int64
		//  fails with image not found, then try to reschedule with image affinity
		// ENGINEAPIFIXME: The first error can be removed once dockerclient is removed
		bImageNotFoundError, _ := regexp.MatchString(`image \S* not found`, err.Error())
		if (bImageNotFoundError || client.IsErrImageNotFound(err)) && !config.HaveNodeConstraint() {
			// Check if the image exists in the cluster
			// If exists, retry with an image affinity
			if c.Image(config.Image) != nil {
				container, err = c.createContainer(config, name, true, authConfig)
				retries++
			}
		}

		for ; retries < c.createRetry && err != nil; retries++ {
			log.WithFields(log.Fields{"Name": "Swarm"}).Warnf("Failed to create container: %s, retrying", err)
			container, err = c.createContainer(config, name, false, authConfig)
		}
	}
	return container, err
}

// OutOfSync checks if the container is out of sync with the service definition.
// It looks if the the service hash container label is the same as the computed one.
func (c *Container) OutOfSync(imageName string) (bool, error) {
	info, err := c.findInfo()
	if err != nil {
		return false, err
	}

	if info.Config.Image != imageName {
		logrus.Debugf("Images for %s do not match %s!=%s", c.name, info.Config.Image, imageName)
		return true, nil
	}

	if info.Config.Labels[HASH.Str()] != c.getHash() {
		logrus.Debugf("Hashes for %s do not match %s!=%s", c.name, info.Config.Labels[HASH.Str()], c.getHash())
		return true, nil
	}

	image, _, err := c.client.ImageInspectWithRaw(context.Background(), info.Config.Image, false)
	if err != nil {
		if client.IsErrImageNotFound(err) {
			logrus.Debugf("Image %s do not exist, do not know if it's out of sync", info.Config.Image)
			return false, nil
		}
		return false, err
	}

	logrus.Debugf("Checking existing image name vs id: %s == %s", image.ID, info.Image)
	return image.ID != info.Image, err
}

func (cli *DockerCli) inspectAll(ctx context.Context, getSize bool) inspect.GetRefFunc {
	return func(ref string) (interface{}, []byte, error) {
		c, rawContainer, err := cli.client.ContainerInspectWithRaw(ctx, ref, getSize)
		if err != nil {
			// Search for image with that id if a container doesn't exist.
			if client.IsErrContainerNotFound(err) {
				i, rawImage, err := cli.client.ImageInspectWithRaw(ctx, ref, getSize)
				if err != nil {
					if client.IsErrImageNotFound(err) {
						// Search for task with that id if an image doesn't exists.
						t, rawTask, err := cli.client.TaskInspectWithRaw(ctx, ref)
						if err != nil {
							return nil, nil, fmt.Errorf("Error: No such image, container or task: %s", ref)
						}
						if getSize {
							fmt.Fprintln(cli.err, "WARNING: --size ignored for tasks")
						}
						return t, rawTask, nil
					}
					return nil, nil, err
				}
				return i, rawImage, err
			}
			return nil, nil, err
		}
		return c, rawContainer, err
	}
}

func (c *Container) createContainer(imageName, oldContainer string, configOverride *config.ServiceConfig) (*types.Container, error) {
	serviceConfig := c.service.serviceConfig
	if configOverride != nil {
		serviceConfig.Command = configOverride.Command
		serviceConfig.Tty = configOverride.Tty
		serviceConfig.StdinOpen = configOverride.StdinOpen
	}
	configWrapper, err := ConvertToAPI(c.service)
	if err != nil {
		return nil, err
	}

	configWrapper.Config.Image = imageName

	if configWrapper.Config.Labels == nil {
		configWrapper.Config.Labels = map[string]string{}
	}

	configWrapper.Config.Labels[SERVICE.Str()] = c.service.name
	configWrapper.Config.Labels[PROJECT.Str()] = c.service.context.Project.Name
	configWrapper.Config.Labels[HASH.Str()] = c.getHash()
	// libcompose run command not yet supported, so always "False"
	configWrapper.Config.Labels[ONEOFF.Str()] = "False"
	configWrapper.Config.Labels[NUMBER.Str()] = getContainerNumber(c)
	configWrapper.Config.Labels[VERSION.Str()] = ComposeVersion

	err = c.populateAdditionalHostConfig(configWrapper.HostConfig)
	if err != nil {
		return nil, err
	}

	if oldContainer != "" {
		info, err := c.client.ContainerInspect(context.Background(), oldContainer)
		if err != nil {
			return nil, err
		}
		configWrapper.HostConfig.Binds = util.Merge(configWrapper.HostConfig.Binds, volumeBinds(configWrapper.Config.Volumes, &info))
	}

	logrus.Debugf("Creating container %s %#v", c.name, configWrapper)

	container, err := c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name)
	if err != nil {
		if client.IsErrImageNotFound(err) {
			logrus.Debugf("Not Found, pulling image %s", configWrapper.Config.Image)
			if err = c.pull(configWrapper.Config.Image); err != nil {
				return nil, err
			}
			if container, err = c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name); err != nil {
				return nil, err
			}
		} else {
			logrus.Debugf("Failed to create container %s: %v", c.name, err)
			return nil, err
		}
	}

	return GetContainerByID(c.client, container.ID)
}

func ensureImage(cli DockerClient, image string) (string, error) {
	ctx := context.Background()
	info, _, err := cli.ImageInspectWithRaw(ctx, image, false)
	if err == nil {
		logrus.Debugf("Image found locally %s", image)
		return info.ID, nil
	}

	if !client.IsErrImageNotFound(err) {
		logrus.Errorf("Error inspecting image %q: %v", image, err)
		return "", err
	}

	// Image must be tagged reference if it does not exist
	ref, err := reference.Parse(image)
	if err != nil {
		logrus.Errorf("Image is not valid reference %q: %v", image, err)
		return "", err
	}
	tagged, ok := ref.(reference.NamedTagged)
	if !ok {
		logrus.Errorf("Tagged reference required %q", image)
		return "", errors.New("invalid reference, tag needed")
	}

	pullStart := time.Now()
	pullOptions := types.ImagePullOptions{
		PrivilegeFunc: registryAuthNotSupported,
	}
	resp, err := cli.ImagePull(ctx, tagged.String(), pullOptions)
	if err != nil {
		logrus.Errorf("Error pulling image %q: %v", tagged.String(), err)
		return "", err
	}
	defer resp.Close()

	outFd, isTerminalOut := term.GetFdInfo(os.Stdout)

	if err = jsonmessage.DisplayJSONMessagesStream(resp, os.Stdout, outFd, isTerminalOut, nil); err != nil {
		logrus.Errorf("Error copying pull output: %v", err)
		return "", err
	}
	// TODO: Get pulled digest

	logFields := logrus.Fields{
		timerKey: time.Since(pullStart),
		"image":  tagged.String(),
	}
	logrus.WithFields(logFields).Info("image pulled")

	info, _, err = cli.ImageInspectWithRaw(ctx, tagged.String(), false)
	if err != nil {
		return "", nil
	}

	return info.ID, nil
}

func (d *kubeDockerClient) InspectImage(image string) (*dockertypes.ImageInspect, error) {
	resp, _, err := d.client.ImageInspectWithRaw(getDefaultContext(), image, true)
	if err != nil {
		if dockerapi.IsErrImageNotFound(err) {
			err = imageNotFoundError{ID: image}
		}
		return nil, err
	}
	return &resp, nil
}

// IsImageInLocalRegistry determines whether the supplied image is in the local registry.
func (d *stiDocker) IsImageInLocalRegistry(name string) (bool, error) {
	name = getImageName(name)
	resp, err := d.InspectImage(name)
	if resp != nil {
		return true, nil
	}
	if err != nil && !dockerapi.IsErrImageNotFound(err) {
		return false, errors.NewInspectImageError(name, err)
	}
	return false, nil
}

func (s stiDocker) InspectImage(name string) (*dockertypes.ImageInspect, error) {
	ctx, _ := context.WithTimeout(context.Background(), 2*time.Minute)
	resp, _, err := s.client.ImageInspectWithRaw(ctx, name, true)
	if err != nil {
		if dockerapi.IsErrImageNotFound(err) {
			return nil, fmt.Errorf("no such image :%q", name)
		}
		return nil, err
	}
	return &resp, nil
}

func (i integrationTest) InspectImage(name string) (*dockertypes.ImageInspect, error) {
	ctx, cancel := getDefaultContext()
	defer cancel()
	resp, _, err := i.engineClient.ImageInspectWithRaw(ctx, name)
	if err != nil {
		if dockerapi.IsErrImageNotFound(err) {
			return nil, fmt.Errorf("no such image :%q", name)
		}
		return nil, err
	}
	return &resp, nil
}

func (cli *DockerCli) createContainer(config *container.Config, hostConfig *container.HostConfig, networkingConfig *networktypes.NetworkingConfig, cidfile, name string) (*types.ContainerCreateResponse, error) {
	var containerIDFile *cidFile
	if cidfile != "" {
		var err error
		if containerIDFile, err = newCIDFile(cidfile); err != nil {
			return nil, err
		}
		defer containerIDFile.Close()
	}

	_, ref, err := reference.ParseIDOrReference(config.Image)
	if err != nil {
		return nil, err
	}
	if ref != nil {
		ref = reference.WithDefaultTag(ref)
	}

	//create the container
	response, err := cli.client.ContainerCreate(context.Background(), config, hostConfig, networkingConfig, name)

	//if image not found try to pull it
	if err != nil {
		if client.IsErrImageNotFound(err) && ref != nil {
			fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.String())

			// we don't want to write to stdout anything apart from container.ID
			if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil {
				return nil, err
			}
			// Retry
			var retryErr error
			response, retryErr = cli.client.ContainerCreate(context.Background(), config, hostConfig, networkingConfig, name)
			if retryErr != nil {
				return nil, retryErr
			}
		} else {
			return nil, err
		}
	}

	for _, warning := range response.Warnings {
		fmt.Fprintf(cli.err, "WARNING: %s\n", warning)
	}
	if containerIDFile != nil {
		if err = containerIDFile.Write(response.ID); err != nil {
			return nil, err
		}
	}
	return &response, nil
}

func (c *Container) createContainer(imageName, oldContainer string) (*types.Container, error) {
	configWrapper, err := ConvertToAPI(c.service)
	if err != nil {
		return nil, err
	}

	configWrapper.Config.Image = imageName

	if configWrapper.Config.Labels == nil {
		configWrapper.Config.Labels = map[string]string{}
	}

	configWrapper.Config.Labels[NAME.Str()] = c.name
	configWrapper.Config.Labels[SERVICE.Str()] = c.service.name
	configWrapper.Config.Labels[PROJECT.Str()] = c.service.context.Project.Name
	configWrapper.Config.Labels[HASH.Str()] = c.getHash()

	err = c.populateAdditionalHostConfig(configWrapper.HostConfig)
	if err != nil {
		return nil, err
	}

	if oldContainer != "" {
		info, err := c.client.ContainerInspect(context.Background(), oldContainer)
		if err != nil {
			return nil, err
		}
		configWrapper.HostConfig.Binds = util.Merge(configWrapper.HostConfig.Binds, volumeBinds(configWrapper.Config.Volumes, &info))
	}

	logrus.Debugf("Creating container %s %#v", c.name, configWrapper)

	container, err := c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name)
	if err != nil {
		if client.IsErrImageNotFound(err) {
			logrus.Debugf("Not Found, pulling image %s", configWrapper.Config.Image)
			if err = c.pull(configWrapper.Config.Image); err != nil {
				return nil, err
			}
			if container, err = c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name); err != nil {
				return nil, err
			}
		} else {
			logrus.Debugf("Failed to create container %s: %v", c.name, err)
			return nil, err
		}
	}

	return GetContainerByID(c.client, container.ID)
}

// Create a new container
func (e *Engine) Create(config *ContainerConfig, name string, pullImage bool, authConfig *types.AuthConfig) (*Container, error) {
	var (
		err        error
		createResp types.ContainerCreateResponse
	)

	// Convert our internal ContainerConfig into something Docker will
	// understand.  Start by making a copy of the internal ContainerConfig as
	// we don't want to mess with the original.
	dockerConfig := *config

	// nb of CPUs -> real CpuShares

	// FIXME remove "duplicate" lines and move this to cluster/config.go
	dockerConfig.HostConfig.CPUShares = int64(math.Ceil(float64(config.HostConfig.CPUShares*1024) / float64(e.Cpus)))

	createResp, err = e.apiClient.ContainerCreate(context.Background(), &dockerConfig.Config, &dockerConfig.HostConfig, &dockerConfig.NetworkingConfig, name)
	e.CheckConnectionErr(err)
	if err != nil {
		// If the error is other than not found, abort immediately.
		if (err != dockerclient.ErrImageNotFound && !engineapi.IsErrImageNotFound(err)) || !pullImage {
			return nil, err
		}
		// Otherwise, try to pull the image...
		if err = e.Pull(config.Image, authConfig); err != nil {
			return nil, err
		}
		// ...And try again.
		createResp, err = e.apiClient.ContainerCreate(context.Background(), &dockerConfig.Config, &dockerConfig.HostConfig, &dockerConfig.NetworkingConfig, name)
		e.CheckConnectionErr(err)
		if err != nil {
			return nil, err
		}
	}

	// Register the container immediately while waiting for a state refresh.
	// Force a state refresh to pick up the newly created container.
	e.refreshContainer(createResp.ID, true)
	e.RefreshVolumes()
	e.RefreshNetworks()

	e.Lock()
	container := e.containers[createResp.ID]
	e.Unlock()

	if container == nil {
		err = errors.New("Container created but refresh didn't report it back")
	}
	return container, err
}

func (d *kubeDockerClient) InspectImage(image string) (*dockertypes.ImageInspect, error) {
	ctx, cancel := d.getTimeoutContext()
	defer cancel()
	resp, _, err := d.client.ImageInspectWithRaw(ctx, image, true)
	if ctxErr := contextError(ctx); ctxErr != nil {
		return nil, ctxErr
	}
	if err != nil {
		if dockerapi.IsErrImageNotFound(err) {
			err = imageNotFoundError{ID: image}
		}
		return nil, err
	}
	return &resp, nil
}

func (d *kubeDockerClient) InspectImage(image string) (*docker.Image, error) {
	resp, _, err := d.client.ImageInspectWithRaw(getDefaultContext(), image, true)
	if err != nil {
		// TODO(random-liu): Use IsErrImageNotFound instead of ErrNoSuchImage
		if dockerapi.IsErrImageNotFound(err) {
			err = docker.ErrNoSuchImage
		}
		return nil, err
	}
	imageInfo := &docker.Image{}
	if err := convertType(&resp, imageInfo); err != nil {
		return nil, err
	}
	return imageInfo, nil
}

func (s *Service) ensureImageExists() (string, error) {
	err := s.imageExists()

	if err == nil {
		return s.imageName(), nil
	}

	if err != nil && !client.IsErrImageNotFound(err) {
		return "", err
	}

	if s.Config().Build != "" {
		if s.context.NoBuild {
			return "", fmt.Errorf("Service %q needs to be built, but no-build was specified", s.name)
		}
		return s.imageName(), s.build()
	}

	return s.imageName(), s.Pull()
}

func (cli *DockerCli) inspectAll(getSize bool) inspectSearcher {
	return func(ref string) (interface{}, []byte, error) {
		c, rawContainer, err := cli.client.ContainerInspectWithRaw(ref, getSize)
		if err != nil {
			// Search for image with that id if a container doesn't exist.
			if client.IsErrContainerNotFound(err) {
				i, rawImage, err := cli.client.ImageInspectWithRaw(ref, getSize)
				if err != nil {
					if client.IsErrImageNotFound(err) {
						return nil, nil, fmt.Errorf("Error: No such image or container: %s", ref)
					}
					return nil, nil, err
				}
				return i, rawImage, err
			}
			return nil, nil, err
		}
		return c, rawContainer, err
	}
}

func (p *Project) ensureImageExists(ref string, force bool) error {
	if !force {
		// Check if ref is already there
		_, _, err := p.Client.ImageInspectWithRaw(context.Background(), ref, false)
		if err != nil && !client.IsErrImageNotFound(err) {
			return err
		}
		if err == nil {
			return nil
		}
	}

	// And pull it
	responseBody, err := p.Client.ImagePull(context.Background(), ref, types.ImagePullOptions{})
	if err != nil {
		fmt.Printf("%v", err)
		return err
	}
	defer responseBody.Close()

	_, err = ioutil.ReadAll(responseBody)
	return err
}

func createContainer(ctx context.Context, dockerCli *client.DockerCli, config *container.Config, hostConfig *container.HostConfig, networkingConfig *networktypes.NetworkingConfig, cidfile, name string) (*types.ContainerCreateResponse, error) {
	stderr := dockerCli.Err()

	var containerIDFile *cidFile
	if cidfile != "" {
		var err error
		if containerIDFile, err = newCIDFile(cidfile); err != nil {
			return nil, err
		}
		defer containerIDFile.Close()
	}

	var trustedRef reference.Canonical
	_, ref, err := reference.ParseIDOrReference(config.Image)
	if err != nil {
		return nil, err
	}
	if ref != nil {
		ref = reference.WithDefaultTag(ref)

		if ref, ok := ref.(reference.NamedTagged); ok && client.IsTrusted() {
			var err error
			trustedRef, err = dockerCli.TrustedReference(ctx, ref)
			if err != nil {
				return nil, err
			}
			config.Image = trustedRef.String()
		}
	}

	//create the container
	response, err := dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, name)

	//if image not found try to pull it
	if err != nil {
		if apiclient.IsErrImageNotFound(err) && ref != nil {
			fmt.Fprintf(stderr, "Unable to find image '%s' locally\n", ref.String())

			// we don't want to write to stdout anything apart from container.ID
			if err = pullImage(ctx, dockerCli, config.Image, stderr); err != nil {
				return nil, err
			}
			if ref, ok := ref.(reference.NamedTagged); ok && trustedRef != nil {
				if err := dockerCli.TagTrusted(ctx, trustedRef, ref); err != nil {
					return nil, err
				}
			}
			// Retry
			var retryErr error
			response, retryErr = dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, name)
			if retryErr != nil {
				return nil, retryErr
			}
		} else {
			return nil, err
		}
	}

	for _, warning := range response.Warnings {
		fmt.Fprintf(stderr, "WARNING: %s\n", warning)
	}
	if containerIDFile != nil {
		if err = containerIDFile.Write(response.ID); err != nil {
			return nil, err
		}
	}
	return &response, nil
}

// create the volumes container that keeps all of the volumes used by
// the cluster.
func (l *LocalCluster) initCluster() {
	log.Infof("initializing cluster")
	l.panicOnStop()

	// Create the temporary certs directory in the current working
	// directory. Boot2docker's handling of binding local directories
	// into the container is very confusing. If the directory being
	// bound has a parent directory that exists in the boot2docker VM
	// then that directory is bound into the container. In particular,
	// that means that binds of /tmp and /var will be problematic.
	var err error
	l.CertsDir, err = ioutil.TempDir(pwd, ".localcluster.certs.")
	maybePanic(err)

	binds := []string{
		l.CertsDir + ":/certs",
		filepath.Join(pwd, "..") + ":/go/src/github.com/cockroachdb/cockroach",
	}

	if l.logDir != "" {
		l.keepLogs = true
		if !filepath.IsAbs(l.logDir) {
			l.logDir = filepath.Join(pwd, l.logDir)
		}
		binds = append(binds, l.logDir+":/logs")
		// If we don't make sure the directory exists, Docker will and then we
		// may run into ownership issues (think Docker running as root, but us
		// running as a regular Joe as it happens on CircleCI).
		maybePanic(os.MkdirAll(l.logDir, 0777))
	}
	if *cockroachImage == builderImage {
		path, err := filepath.Abs(*cockroachBinary)
		maybePanic(err)
		binds = append(binds, path+":/"+filepath.Base(*cockroachBinary))
	}

	vols := map[string]struct{}{}
	for i := 0; i < l.numLocal; i++ {
		for j := 0; j < l.numStores; j++ {
			vols[dataStr(i, j)] = struct{}{}
		}
	}
	create := func() (*Container, error) {
		return createContainer(
			l,
			container.Config{
				Image:      *cockroachImage,
				Volumes:    vols,
				Entrypoint: strslice.New("/bin/true"),
			}, container.HostConfig{
				Binds:           binds,
				PublishAllPorts: true,
			},
			"volumes",
		)
	}
	c, err := create()
	if dockerclient.IsErrImageNotFound(err) && *cockroachImage == builderImage {
		if err := pullImage(l, types.ImagePullOptions{ImageID: *cockroachImage}); err != nil {
			log.Fatal(err)
		}
		c, err = create()
	}
	maybePanic(err)

	maybePanic(c.Start())
	maybePanic(c.Wait())
	l.vols = c
}