func (daemon *Daemon) mountVolumes(container *Container) error {
	mounts, err := daemon.setupMounts(container)
	if err != nil {
		return err
	}

	for _, m := range mounts {
		dest, err := container.GetResourcePath(m.Destination)
		if err != nil {
			return err
		}

		var stat os.FileInfo
		stat, err = os.Stat(m.Source)
		if err != nil {
			return err
		}
		if err = fileutils.CreateIfNotExists(dest, stat.IsDir()); err != nil {
			return err
		}

		opts := "rbind,ro"
		if m.Writable {
			opts = "rbind,rw"
		}

		if err := mount.Mount(m.Source, dest, "bind", opts); err != nil {
			return err
		}
	}

	return nil
}

func (container *Container) mountVolumes() error {
	for dest, source := range container.Volumes {
		v := container.daemon.volumes.Get(source)
		if v == nil {
			return fmt.Errorf("could not find volume for %s:%s, impossible to mount", source, dest)
		}

		destPath, err := container.GetResourcePath(dest)
		if err != nil {
			return err
		}

		if err := mount.Mount(source, destPath, "bind", "rbind,rw"); err != nil {
			return fmt.Errorf("error while mounting volume %s: %v", source, err)
		}
	}

	for _, mnt := range container.specialMounts() {
		destPath, err := container.GetResourcePath(mnt.Destination)
		if err != nil {
			return err
		}
		if err := mount.Mount(mnt.Source, destPath, "bind", "bind,rw"); err != nil {
			return fmt.Errorf("error while mounting volume %s: %v", mnt.Source, err)
		}
	}
	return nil
}

func bindMount(bindDir string, rootDir string, readonly bool) error {
	var srcDir, destDir string

	d := strings.SplitN(bindDir, ":", 2)
	if len(d) < 2 {
		srcDir = d[0]
	} else {
		srcDir, destDir = d[0], d[1]
	}
	if destDir == "" {
		destDir = srcDir
	}

	ok, err := osutil.IsDirEmpty(srcDir)
	if err != nil {
		return err
	}
	if ok {
		if _, err := os.Create(fp.Join(srcDir, ".droot.keep")); err != nil {
			return errwrap.Wrapf(err, "Failed to create .droot.keep: {{err}}")
		}
	}

	containerDir := fp.Join(rootDir, destDir)

	if err := fileutils.CreateIfNotExists(containerDir, true); err != nil { // mkdir -p
		return errwrap.Wrapff(err, "Failed to create directory: %s: {{err}}", containerDir)
	}

	ok, err = osutil.IsDirEmpty(containerDir)
	if err != nil {
		return err
	}
	if ok {
		log.Debug("bind mount", bindDir, "to", containerDir)
		if err := mount.Mount(srcDir, containerDir, "none", "bind,rw"); err != nil {
			return errwrap.Wrapff(err, "Failed to bind mount %s: {{err}}", containerDir)
		}

		if readonly {
			log.Debug("robind mount", bindDir, "to", containerDir)
			if err := mount.Mount(srcDir, containerDir, "none", "remount,ro,bind"); err != nil {
				return errwrap.Wrapff(err, "Failed to robind mount %s: {{err}}", containerDir)
			}
		}
	}

	return nil
}

func (d *driver) Mount(
	device, target, mountOptions, mountLabel string) error {

	if d.isNfsDevice(device) {

		if err := d.nfsMount(device, target); err != nil {
			return err
		}

		os.MkdirAll(d.volumeMountPath(target), d.fileModeMountPath())
		os.Chmod(d.volumeMountPath(target), d.fileModeMountPath())

		return nil
	}

	fsType, err := probeFsType(device)
	if err != nil {
		return err
	}

	options := label.FormatMountLabel("", mountLabel)
	options = fmt.Sprintf("%s,%s", mountOptions, mountLabel)
	if fsType == "xfs" {
		options = fmt.Sprintf("%s,nouuid", mountOptions)
	}

	if err := mount.Mount(device, target, fsType, options); err != nil {
		return fmt.Errorf("Couldn't mount directory %s at %s: %s", device, target, err)
	}

	os.MkdirAll(d.volumeMountPath(target), d.fileModeMountPath())
	os.Chmod(d.volumeMountPath(target), d.fileModeMountPath())

	return nil
}

func (container *Container) Copy(resource string) (io.ReadCloser, error) {
	container.Lock()
	defer container.Unlock()
	var err error
	if err := container.Mount(); err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			// unmount any volumes
			container.UnmountVolumes(true)
			// unmount the container's rootfs
			container.Unmount()
		}
	}()
	mounts, err := container.setupMounts()
	if err != nil {
		return nil, err
	}
	for _, m := range mounts {
		dest, err := container.GetResourcePath(m.Destination)
		if err != nil {
			return nil, err
		}
		if err := mount.Mount(m.Source, dest, "bind", "rbind,ro"); err != nil {
			return nil, err
		}
	}
	basePath, err := container.GetResourcePath(resource)
	if err != nil {
		return nil, err
	}
	stat, err := os.Stat(basePath)
	if err != nil {
		return nil, err
	}
	var filter []string
	if !stat.IsDir() {
		d, f := filepath.Split(basePath)
		basePath = d
		filter = []string{f}
	} else {
		filter = []string{filepath.Base(basePath)}
		basePath = filepath.Dir(basePath)
	}
	archive, err := archive.TarWithOptions(basePath, &archive.TarOptions{
		Compression:  archive.Uncompressed,
		IncludeFiles: filter,
	})
	if err != nil {
		return nil, err
	}
	return ioutils.NewReadCloserWrapper(archive, func() error {
			err := archive.Close()
			container.UnmountVolumes(true)
			container.Unmount()
			return err
		}),
		nil
}

func (v *localVolume) mount() error {
	if v.opts.MountDevice == "" {
		return fmt.Errorf("missing device in volume options")
	}
	err := mount.Mount(v.opts.MountDevice, v.path, v.opts.MountType, v.opts.MountOpts)
	return errors.Wrapf(err, "error while mounting volume with options: %s", v.opts)
}

// Test recursive bind mount works by default
func (s *DockerSuite) TestRunWithVolumesIsRecursive(c *check.C) {
	// /tmp gets permission denied
	testRequires(c, NotUserNamespace)
	tmpDir, err := ioutil.TempDir("", "docker_recursive_mount_test")
	if err != nil {
		c.Fatal(err)
	}

	defer os.RemoveAll(tmpDir)

	// Create a temporary tmpfs mount.
	tmpfsDir := filepath.Join(tmpDir, "tmpfs")
	if err := os.MkdirAll(tmpfsDir, 0777); err != nil {
		c.Fatalf("failed to mkdir at %s - %s", tmpfsDir, err)
	}
	if err := mount.Mount("tmpfs", tmpfsDir, "tmpfs", ""); err != nil {
		c.Fatalf("failed to create a tmpfs mount at %s - %s", tmpfsDir, err)
	}

	f, err := ioutil.TempFile(tmpfsDir, "touch-me")
	if err != nil {
		c.Fatal(err)
	}
	defer f.Close()

	runCmd := exec.Command(dockerBinary, "run", "--name", "test-data", "--volume", fmt.Sprintf("%s:/tmp:ro", tmpDir), "busybox:latest", "ls", "/tmp/tmpfs")
	out, stderr, exitCode, err := runCommandWithStdoutStderr(runCmd)
	if err != nil && exitCode != 0 {
		c.Fatal(out, stderr, err)
	}
	if !strings.Contains(out, filepath.Base(f.Name())) {
		c.Fatal("Recursive bind mount test failed. Expected file not found")
	}
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		return "", err
	}

	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}
	// this could be our first mount after creation of the filesystem, and the root dir may still have root
	// permissions instead of the remapped root uid:gid (if user namespaces are enabled):
	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
	}

	return mountpoint, nil
}

func mountToRootfs(m *specs.Mount, rootfs, mountLabel string) error {
	// TODO: we don't use mountLabel here because it looks like mountLabel is
	// only significant when SELinux is enabled.
	var (
		dest = m.Destination
	)
	if !strings.HasPrefix(dest, rootfs) {
		dest = filepath.Join(rootfs, dest)
	}

	switch m.Type {
	case "proc", "sysfs", "mqueue", "tmpfs", "cgroup", "devpts":
		glog.V(3).Infof("Skip mount point %q of type %s", m.Destination, m.Type)
		return nil
	case "bind":
		stat, err := os.Stat(m.Source)
		if err != nil {
			// error out if the source of a bind mount does not exist as we will be
			// unable to bind anything to it.
			return err
		}
		// ensure that the destination of the bind mount is resolved of symlinks at mount time because
		// any previous mounts can invalidate the next mount's destination.
		// this can happen when a user specifies mounts within other mounts to cause breakouts or other
		// evil stuff to try to escape the container's rootfs.
		if dest, err = symlink.FollowSymlinkInScope(filepath.Join(rootfs, m.Destination), rootfs); err != nil {
			return err
		}
		if err := checkMountDestination(rootfs, dest); err != nil {
			return err
		}
		// update the mount with the correct dest after symlinks are resolved.
		m.Destination = dest
		if err := createIfNotExists(dest, stat.IsDir()); err != nil {
			return err
		}
		if err := mount.Mount(m.Source, dest, m.Type, strings.Join(m.Options, ",")); err != nil {
			return err
		}
	default:
		if err := os.MkdirAll(dest, 0755); err != nil {
			return err
		}
		return mount.Mount(m.Source, dest, m.Type, strings.Join(m.Options, ","))
	}
	return nil
}

func bindMount(bindDir string, rootDir string, readonly bool) error {
	var srcDir, destDir string

	d := strings.SplitN(bindDir, ":", 2)
	if len(d) < 2 {
		srcDir = d[0]
	} else {
		srcDir, destDir = d[0], d[1]
	}
	if destDir == "" {
		destDir = srcDir
	}

	ok, err := IsDirEmpty(srcDir)
	if err != nil {
		return err
	}

	containerDir := fp.Join(rootDir, destDir)

	if err := fileutils.CreateIfNotExists(containerDir, true); err != nil { // mkdir -p
		log.FATAL.Fatalln("Failed to create directory:", containerDir, err)
	}

	ok, err = IsDirEmpty(containerDir)
	if err != nil {
		return err
	}
	if ok {
		log.DEBUG.Println("bind mount", bindDir, "to", containerDir)
		if err := mount.Mount(srcDir, containerDir, "none", "bind,rw"); err != nil {
			log.FATAL.Fatalln("Failed to bind mount:", containerDir, err)
		}

		if readonly {
			log.DEBUG.Println("robind mount", bindDir, "to", containerDir)
			if err := mount.Mount(srcDir, containerDir, "none", "remount,ro,bind"); err != nil {
				log.FATAL.Fatalln("Failed to robind mount:", containerDir, err)
			}
		}
	}

	return nil
}

func MakePrivate(mountPoint string) error {
	mounted, err := mount.Mounted(mountPoint)
	if err != nil {
		return err
	}

	if !mounted {
		if err := mount.Mount(mountPoint, mountPoint, "none", "bind,rw"); err != nil {
			return err
		}
	}

	return mount.ForceMount("", mountPoint, "none", "private")
}

func Mount(device, directory, fsType, options string) error {
	if err := mountProc(); err != nil {
		return nil
	}

	if _, err := os.Stat(directory); os.IsNotExist(err) {
		err = os.MkdirAll(directory, 0755)
		if err != nil {
			return err
		}
	}

	return mount.Mount(device, directory, fsType, options)
}

func MountIfNotMounted(device, target, mType, options string) error {
	mounted, err := mount.Mounted(target)
	if err != nil {
		return err
	}

	if !mounted {
		log.Debug("mount", device, target, mType, options)
		if err := mount.Mount(device, target, mType, options); err != nil {
			return err
		}
	}

	return nil
}

func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.MountPath(id)
	filesystem := d.ZfsPath(id)
	log.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, mountLabel)

	// Create the target directories if they don't exist
	if err := os.MkdirAll(mountpoint, 0755); err != nil && !os.IsExist(err) {
		return "", err
	}

	err := mount.Mount(filesystem, mountpoint, "zfs", mountLabel)
	if err != nil {
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}

	return mountpoint, nil
}

func (daemon *Daemon) mountVolumes(container *container.Container) error {
	mounts, err := daemon.setupMounts(container)
	if err != nil {
		return err
	}

	for _, m := range mounts {
		dest, err := container.GetResourcePath(m.Destination)
		if err != nil {
			return err
		}

		var stat os.FileInfo
		stat, err = os.Stat(m.Source)
		if err != nil {
			return err
		}
		if err = fileutils.CreateIfNotExists(dest, stat.IsDir()); err != nil {
			return err
		}

		opts := "rbind,ro"
		if m.Writable {
			opts = "rbind,rw"
		}

		if err := mount.Mount(m.Source, dest, bindMountType, opts); err != nil {
			return err
		}

		// mountVolumes() seems to be called for temporary mounts
		// outside the container. Soon these will be unmounted with
		// lazy unmount option and given we have mounted the rbind,
		// all the submounts will propagate if these are shared. If
		// daemon is running in host namespace and has / as shared
		// then these unmounts will propagate and unmount original
		// mount as well. So make all these mounts rprivate.
		// Do not use propagation property of volume as that should
		// apply only when mounting happen inside the container.
		if err := mount.MakeRPrivate(dest); err != nil {
			return err
		}
	}

	return nil
}

func (v *localVolume) mount() error {
	if v.opts.MountDevice == "" {
		return fmt.Errorf("missing device in volume options")
	}
	mountOpts := v.opts.MountOpts
	if v.opts.MountType == "nfs" {
		if addrValue := getAddress(v.opts.MountOpts); addrValue != "" && net.ParseIP(addrValue).To4() == nil {
			ipAddr, err := net.ResolveIPAddr("ip", addrValue)
			if err != nil {
				return errors.Wrapf(err, "error resolving passed in nfs address")
			}
			mountOpts = strings.Replace(mountOpts, "addr="+addrValue, "addr="+ipAddr.String(), 1)
		}
	}
	err := mount.Mount(v.opts.MountDevice, v.path, v.opts.MountType, mountOpts)
	return errors.Wrapf(err, "error while mounting volume with options: %s", v.opts)
}

func (driver *Driver) Mount(device, target, mountOptions, mountLabel string) error {

	fsType, err := probeFsType(device)
	if err != nil {
		return err
	}

	options := label.FormatMountLabel("", mountLabel)
	options = fmt.Sprintf("%s,%s", mountOptions, mountLabel)
	if fsType == "xfs" {
		options = fmt.Sprintf("%s,nouuid", mountOptions)
	}

	if err := mount.Mount(device, target, fsType, options); err != nil {
		return fmt.Errorf("Couldn't mount directory %s at %s: %s", device, target, err)
	}

	return nil
}

func setupSharedRoot(c *config.CloudConfig) (*config.CloudConfig, error) {
	if c.Rancher.NoSharedRoot {
		return c, nil
	}

	if isInitrd() {
		for _, i := range []string{"/mnt", "/media"} {
			if err := os.Mkdir(i, 0755); err != nil {
				return c, err
			}
			if err := mount.Mount("tmpfs", i, "tmpfs", "rw"); err != nil {
				return c, err
			}
			if err := mount.MakeShared(i); err != nil {
				return c, err
			}
		}
		return c, nil
	}

	return c, mount.MakeShared("/")
}

// Test recursive bind mount works by default
func (s *DockerSuite) TestRunWithVolumesIsRecursive(c *check.C) {
	// /tmp gets permission denied
	testRequires(c, NotUserNamespace, SameHostDaemon)
	tmpDir, err := ioutil.TempDir("", "docker_recursive_mount_test")
	c.Assert(err, checker.IsNil)

	defer os.RemoveAll(tmpDir)

	// Create a temporary tmpfs mount.
	tmpfsDir := filepath.Join(tmpDir, "tmpfs")
	c.Assert(os.MkdirAll(tmpfsDir, 0777), checker.IsNil, check.Commentf("failed to mkdir at %s", tmpfsDir))
	c.Assert(mount.Mount("tmpfs", tmpfsDir, "tmpfs", ""), checker.IsNil, check.Commentf("failed to create a tmpfs mount at %s", tmpfsDir))

	f, err := ioutil.TempFile(tmpfsDir, "touch-me")
	c.Assert(err, checker.IsNil)
	defer f.Close()

	runCmd := exec.Command(dockerBinary, "run", "--name", "test-data", "--volume", fmt.Sprintf("%s:/tmp:ro", tmpDir), "busybox:latest", "ls", "/tmp/tmpfs")
	out, _, _, err := runCommandWithStdoutStderr(runCmd)
	c.Assert(err, checker.IsNil)
	c.Assert(out, checker.Contains, filepath.Base(f.Name()), check.Commentf("Recursive bind mount test failed. Expected file not found"))
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		return "", err
	}

	err = mount.Mount(filesystem, mountpoint, "zfs", options)
	if err != nil {
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}

	return mountpoint, nil
}