func (container *Container) startPty() error {
	ptyMaster, ptySlave, err := pty.Open()
	if err != nil {
		return err
	}
	container.ptyMaster = ptyMaster
	container.cmd.Stdout = ptySlave
	container.cmd.Stderr = ptySlave

	// Copy the PTYs to our broadcasters
	go func() {
		defer container.stdout.CloseWriters()
		utils.Debugf("[startPty] Begin of stdout pipe")
		io.Copy(container.stdout, ptyMaster)
		utils.Debugf("[startPty] End of stdout pipe")
	}()

	// stdin
	if container.Config.OpenStdin {
		container.cmd.Stdin = ptySlave
		container.cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}
		go func() {
			defer container.stdin.Close()
			utils.Debugf("[startPty] Begin of stdin pipe")
			io.Copy(ptyMaster, container.stdin)
			utils.Debugf("[startPty] End of stdin pipe")
		}()
	}
	if err := container.cmd.Start(); err != nil {
		return err
	}
	ptySlave.Close()
	return nil
}

// #5979
func (s *DockerSuite) TestEventsRedirectStdout(c *check.C) {
	since := daemonTime(c).Unix()
	dockerCmd(c, "run", "busybox", "true")

	file, err := ioutil.TempFile("", "")
	c.Assert(err, checker.IsNil, check.Commentf("could not create temp file"))
	defer os.Remove(file.Name())

	command := fmt.Sprintf("%s events --since=%d --until=%d > %s", dockerBinary, since, daemonTime(c).Unix(), file.Name())
	_, tty, err := pty.Open()
	c.Assert(err, checker.IsNil, check.Commentf("Could not open pty"))
	cmd := exec.Command("sh", "-c", command)
	cmd.Stdin = tty
	cmd.Stdout = tty
	cmd.Stderr = tty
	c.Assert(cmd.Run(), checker.IsNil, check.Commentf("run err for command %q", command))

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		for _, ch := range scanner.Text() {
			c.Assert(unicode.IsControl(ch), checker.False, check.Commentf("found control character %v", []byte(string(ch))))
		}
	}
	c.Assert(scanner.Err(), checker.IsNil, check.Commentf("Scan err for command %q", command))

}

func (container *Container) startPty() error {
	stdoutMaster, stdoutSlave, err := pty.Open()
	if err != nil {
		return err
	}
	container.ptyStdoutMaster = stdoutMaster
	container.cmd.Stdout = stdoutSlave

	stderrMaster, stderrSlave, err := pty.Open()
	if err != nil {
		return err
	}
	container.ptyStderrMaster = stderrMaster
	container.cmd.Stderr = stderrSlave

	// Copy the PTYs to our broadcasters
	go func() {
		defer container.stdout.CloseWriters()
		Debugf("[startPty] Begin of stdout pipe")
		io.Copy(container.stdout, stdoutMaster)
		Debugf("[startPty] End of stdout pipe")
	}()

	go func() {
		defer container.stderr.CloseWriters()
		Debugf("[startPty] Begin of stderr pipe")
		io.Copy(container.stderr, stderrMaster)
		Debugf("[startPty] End of stderr pipe")
	}()

	// stdin
	var stdinSlave io.ReadCloser
	if container.Config.OpenStdin {
		var stdinMaster io.WriteCloser
		stdinMaster, stdinSlave, err = pty.Open()
		if err != nil {
			return err
		}
		container.ptyStdinMaster = stdinMaster
		container.cmd.Stdin = stdinSlave
		// FIXME: The following appears to be broken.
		// "cannot set terminal process group (-1): Inappropriate ioctl for device"
		// container.cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}
		go func() {
			defer container.stdin.Close()
			Debugf("[startPty] Begin of stdin pipe")
			io.Copy(stdinMaster, container.stdin)
			Debugf("[startPty] End of stdin pipe")
		}()
	}
	if err := container.cmd.Start(); err != nil {
		return err
	}
	stdoutSlave.Close()
	stderrSlave.Close()
	if stdinSlave != nil {
		stdinSlave.Close()
	}
	return nil
}

func createJob(client *beam.Client) *beam.Job {
	job, err := client.NewJob("tty")
	if err != nil {
		panic(err)
	}

	master, slave, err := pty.Open()
	if err != nil {
		panic(err)
	}

	_, err = term.SetRawTerminal(master.Fd())
	if err != nil {
		panic(err)
	}

	go job.WriteTo(slave, "stdout")
	go job.WriteTo(slave, "stderr")
	go job.ReadFrom(slave, "stdin")

	go io.Copy(os.Stdout, master)
	go io.Copy(os.Stderr, master)
	go io.Copy(master, os.Stdin)

	return job
}

func worker() {
	worker := beam.NewWorker(&beam.NetTransport{"tcp", ":6379"}, "/jobs")
	worker.RegisterJob("tty", func(name string, args []string, env map[string]string, streams beam.Streamer, db beam.DB) error {
		p := exec.Command("bash")

		master, slave, err := pty.Open()
		if err != nil {
			return err
		}
		if slave == nil {
			return fmt.Errorf("tty is nil")
		}

		term.SetWinsize(master.Fd(), winsize)

		p.Stdout = slave
		p.Stderr = slave
		p.Stdin = slave
		p.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}

		go streams.ReadFrom(master, "stdout")
		go streams.ReadFrom(master, "stderr")
		go streams.WriteTo(master, "stdin")

		return p.Run()
	})
	worker.Work()
}

func (s *GitDeploySuite) TestDevStdout(t *c.C) {
	r := s.newGitRepo(t, "empty-release")
	t.Assert(r.flynn("create"), Succeeds)
	t.Assert(r.flynn("env", "set", "BUILDPACK_URL=https://github.com/kr/heroku-buildpack-inline"), Succeeds)
	t.Assert(r.git("push", "flynn", "master"), Succeeds)

	// check slug jobs can write to /dev/stdout and /dev/stderr
	for _, dev := range []string{"/dev/stdout", "/dev/stderr"} {
		// check without a TTY
		echoFoo := fmt.Sprintf("echo foo > %s", dev)
		t.Assert(r.flynn("run", "bash", "-c", echoFoo), SuccessfulOutputContains, "foo")

		// check with a TTY
		cmd := flynnCmd(r.dir, "run", "bash", "-c", echoFoo)
		master, slave, err := pty.Open()
		t.Assert(err, c.IsNil)
		defer master.Close()
		defer slave.Close()
		t.Assert(term.SetWinsize(slave.Fd(), &term.Winsize{Height: 24, Width: 80}), c.IsNil)
		cmd.Stdin = slave
		cmd.Stdout = slave
		cmd.Stderr = slave
		t.Assert(cmd.Run(), c.IsNil)
		out := make([]byte, 3)
		_, err = io.ReadFull(master, out)
		t.Assert(err, c.IsNil)
		t.Assert(string(out), c.Equals, "foo")
	}
}

func (pipe *Pipe) runPtyCmd(cmd *exec.Cmd) error {
	py, tty, err := pty.Open()
	if err != nil {
		return err
	}
	defer tty.Close()
	env := os.Environ()
	env = append(env, "TERM=xterm")
	cmd.Env = env
	cmd.Stdout = tty
	cmd.Stdin = tty
	cmd.Stderr = tty
	cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}
	ws := &Winsize{
		Width:  uint16(pipe.opts.Width),
		Height: uint16(pipe.opts.Height),
	}
	_, _, syserr := syscall.Syscall(syscall.SYS_IOCTL, py.Fd(), uintptr(syscall.TIOCSWINSZ), uintptr(unsafe.Pointer(ws)))
	if syserr != 0 {
		return syserr
	}
	pCmd := pipedCmd{
		pipe:   pipe,
		cmd:    cmd,
		stdin:  py,
		stdout: py,
		stderr: nil,
	}
	return pCmd.run()
}

// TestRunDetach checks attaching and detaching with the escape sequence specified via flags.
func (s *DockerSuite) TestRunAttachDetachFromFlag(c *check.C) {
	name := "attach-detach"
	keyCtrlA := []byte{1}
	keyA := []byte{97}

	dockerCmd(c, "run", "--name", name, "-itd", "busybox", "cat")

	cmd := exec.Command(dockerBinary, "attach", "--detach-keys='ctrl-a,a'", name)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		c.Fatal(err)
	}
	cpty, tty, err := pty.Open()
	if err != nil {
		c.Fatal(err)
	}
	defer cpty.Close()
	cmd.Stdin = tty
	if err := cmd.Start(); err != nil {
		c.Fatal(err)
	}
	c.Assert(waitRun(name), check.IsNil)

	if _, err := cpty.Write([]byte("hello\n")); err != nil {
		c.Fatal(err)
	}

	out, err := bufio.NewReader(stdout).ReadString('\n')
	if err != nil {
		c.Fatal(err)
	}
	if strings.TrimSpace(out) != "hello" {
		c.Fatalf("expected 'hello', got %q", out)
	}

	// escape sequence
	if _, err := cpty.Write(keyCtrlA); err != nil {
		c.Fatal(err)
	}
	time.Sleep(100 * time.Millisecond)
	if _, err := cpty.Write(keyA); err != nil {
		c.Fatal(err)
	}

	ch := make(chan struct{})
	go func() {
		cmd.Wait()
		ch <- struct{}{}
	}()

	select {
	case <-ch:
	case <-time.After(10 * time.Second):
		c.Fatal("timed out waiting for container to exit")
	}

	running, err := inspectField(name, "State.Running")
	c.Assert(err, checker.IsNil)
	c.Assert(running, checker.Equals, "true", check.Commentf("expected container to still be running"))
}

func createTtyPty(windowColumns int, windowRows int) (stdinR, stdinW, stdoutR, stdoutW, stderrR, stderrW *os.File, err error) {
	// stderr will not be assigned in the case of a tty, so ensure it will return EOF on read
	stderrR, err = os.Open("/dev/null")
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}

	pty, tty, err := pty.Open()
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}

	// do NOT assign stderrR to pty; the receiving end should only receive one
	// pty output stream, as they're both the same fd

	stdinW = pty
	stdoutR = pty

	stdinR = tty
	stdoutW = tty
	stderrW = tty

	setWinSize(stdinW, windowColumns, windowRows)

	return
}

// #6509
func (s *DockerSuite) TestRunRedirectStdout(c *check.C) {
	checkRedirect := func(command string) {
		_, tty, err := pty.Open()
		c.Assert(err, checker.IsNil, check.Commentf("Could not open pty"))
		cmd := exec.Command("sh", "-c", command)
		cmd.Stdin = tty
		cmd.Stdout = tty
		cmd.Stderr = tty
		c.Assert(cmd.Start(), checker.IsNil)
		ch := make(chan error)
		go func() {
			ch <- cmd.Wait()
			close(ch)
		}()

		select {
		case <-time.After(10 * time.Second):
			c.Fatal("command timeout")
		case err := <-ch:
			c.Assert(err, checker.IsNil, check.Commentf("wait err"))
		}
	}

	checkRedirect(dockerBinary + " run -i busybox cat /etc/passwd | grep -q root")
	checkRedirect(dockerBinary + " run busybox cat /etc/passwd | grep -q root")
}

func (s *DockerSuite) TestUpdateNotAffectMonitorRestartPolicy(c *check.C) {
	testRequires(c, DaemonIsLinux, cpuShare)

	out, _ := dockerCmd(c, "run", "-tid", "--restart=always", "busybox", "sh")
	id := strings.TrimSpace(string(out))
	dockerCmd(c, "update", "--cpu-shares", "512", id)

	cpty, tty, err := pty.Open()
	c.Assert(err, checker.IsNil)
	defer cpty.Close()

	cmd := exec.Command(dockerBinary, "attach", id)
	cmd.Stdin = tty

	c.Assert(cmd.Start(), checker.IsNil)
	defer cmd.Process.Kill()

	_, err = cpty.Write([]byte("exit\n"))
	c.Assert(err, checker.IsNil)

	c.Assert(cmd.Wait(), checker.IsNil)

	// container should restart again and keep running
	err = waitInspect(id, "{{.RestartCount}}", "1", 30*time.Second)
	c.Assert(err, checker.IsNil)
	c.Assert(waitRun(id), checker.IsNil)
}

func TestIsTerminal(t *testing.T) {
	mpty, mtty, err := pty.Open()
	if err != nil {
		t.Fatal(err)
	}
	defer mtty.Close()
	defer mpty.Close()

	if !ttyutils.IsTerminal(mtty.Fd()) {
		t.Error("tty should be reported as a terminal")
	}
	if !ttyutils.IsTerminal(mpty.Fd()) {
		t.Error("pty should be reported as a terminal")
	}

	null, err := os.Open(os.DevNull)
	if err != nil {
		t.Fatal(err)
	}
	defer null.Close()

	if ttyutils.IsTerminal(null.Fd()) {
		t.Error("/dev/null should not be reported as a terminal")
	}
}

func record() {
	C.ttySave()
	defer C.ttyRestore()
	C.ttyRaw()

	master, slave, err := pty.Open()
	if err != nil {
		log.Fatalln("open:", err)
	}
	C.ttyFixWinsz(C.int(slave.Fd()))
	script, err := os.OpenFile("ttyrecord", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatalln(err)
	}

	sh := os.Getenv("SHELL")
	if sh == "" {
		sh = "/bin/sh"
	}
	attr := &syscall.SysProcAttr{
		Setsid:  true,
		Setctty: true,
		Ctty:    int(slave.Fd()),
	}
	shell := &exec.Cmd{
		Path:        sh,
		SysProcAttr: attr,
		Stdin:       slave,
		Stdout:      slave,
		Stderr:      slave,
	}
	go io.Copy(master, os.Stdin)
	go io.Copy(io.MultiWriter(newGobWriter(script), os.Stdout), master)
	shell.Run()
}

func (container *Container) setupPty() error {
	ptyMaster, ptySlave, err := pty.Open()
	if err != nil {
		return err
	}
	container.ptyMaster = ptyMaster
	container.command.Stdout = ptySlave
	container.command.Stderr = ptySlave

	// Copy the PTYs to our broadcasters
	go func() {
		defer container.stdout.CloseWriters()
		utils.Debugf("startPty: begin of stdout pipe")
		io.Copy(container.stdout, ptyMaster)
		utils.Debugf("startPty: end of stdout pipe")
	}()

	// stdin
	if container.Config.OpenStdin {
		container.command.Stdin = ptySlave
		container.command.SysProcAttr.Setctty = true
		go func() {
			defer container.stdin.Close()
			utils.Debugf("startPty: begin of stdin pipe")
			io.Copy(ptyMaster, container.stdin)
			utils.Debugf("startPty: end of stdin pipe")
		}()
	}
	return nil
}

func NewTtyConsole(processConfig *execdriver.ProcessConfig, pipes *execdriver.Pipes) (*TtyConsole, error) {
	// lxc is special in that we cannot create the master outside of the container without
	// opening the slave because we have nothing to provide to the cmd.  We have to open both then do
	// the crazy setup on command right now instead of passing the console path to lxc and telling it
	// to open up that console.  we save a couple of openfiles in the native driver because we can do
	// this.
	ptyMaster, ptySlave, err := pty.Open()
	if err != nil {
		return nil, err
	}

	tty := &TtyConsole{
		MasterPty: ptyMaster,
		SlavePty:  ptySlave,
	}

	if err := tty.AttachPipes(&processConfig.Cmd, pipes); err != nil {
		tty.Close()
		return nil, err
	}

	processConfig.Console = tty.SlavePty.Name()

	return tty, nil
}

// HandleChannel handles one SSH channel
func (c *Client) HandleChannel(newChannel ssh.NewChannel) error {
	if newChannel.ChannelType() != "session" {
		log.Debugf("Unknown channel type: %s", newChannel.ChannelType())
		newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
		return nil
	}

	channel, requests, err := newChannel.Accept()
	if err != nil {
		log.Errorf("newChannel.Accept failed: %v", err)
		return err
	}
	c.ChannelIdx++
	log.Debugf("HandleChannel.channel (client=%d channel=%d)", c.Idx, c.ChannelIdx)

	log.Debug("Creating pty...")
	c.Pty, c.Tty, err = pty.Open()
	if err != nil {
		log.Errorf("pty.Open failed: %v", err)
		return nil
	}

	c.HandleChannelRequests(channel, requests)

	return nil
}

// TestRunDetach checks attaching and detaching with the escape sequence specified via flags.
func (s *DockerSuite) TestRunAttachDetachFromInvalidFlag(c *check.C) {
	name := "attach-detach"
	dockerCmd(c, "run", "--name", name, "-itd", "busybox", "top")
	c.Assert(waitRun(name), check.IsNil)

	// specify an invalid detach key, container will ignore it and use default
	cmd := exec.Command(dockerBinary, "attach", "--detach-keys='ctrl-A,a'", name)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		c.Fatal(err)
	}
	cpty, tty, err := pty.Open()
	if err != nil {
		c.Fatal(err)
	}
	defer cpty.Close()
	cmd.Stdin = tty
	if err := cmd.Start(); err != nil {
		c.Fatal(err)
	}

	bufReader := bufio.NewReader(stdout)
	out, err := bufReader.ReadString('\n')
	if err != nil {
		c.Fatal(err)
	}
	// it should print a warning to indicate the detach key flag is invalid
	errStr := "Invalid escape keys (ctrl-A,a) provided"
	c.Assert(strings.TrimSpace(out), checker.Equals, errStr)
}

func handleConnection(c net.Conn) {
	defer c.Close()

	// Start the command
	cmd := exec.Command(flagCommand)

	// Create PTY
	pty, tty, err := pty.Open()
	if err != nil {
		log.Printf("error: could not open PTY: %s", err)
		return
	}
	defer tty.Close()
	defer pty.Close()

	// Put the TTY into raw mode
	_, err = terminal.MakeRaw(int(tty.Fd()))
	if err != nil {
		log.Printf("warn: could not make TTY raw: %s", err)
	}

	// Hook everything up
	cmd.Stdout = tty
	cmd.Stdin = tty
	cmd.Stderr = tty
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}

	cmd.SysProcAttr.Setctty = true
	cmd.SysProcAttr.Setsid = true

	// Start command
	err = cmd.Start()
	if err != nil {
		log.Printf("error: could not start command: %s", err)
		return
	}

	errs := make(chan error, 3)

	go func() {
		_, err := io.Copy(c, pty)
		errs <- err
	}()
	go func() {
		_, err := io.Copy(pty, c)
		errs <- err
	}()
	go func() {
		errs <- cmd.Wait()
	}()

	// Wait for a single error, then shut everything down. Since returning from
	// this function closes the connection, we just read a single error and
	// then continue.
	<-errs
	log.Printf("info: connection from %s finished", c.RemoteAddr())
}

// NewTty creates a pseudo-TTY for a command and modifies it appropriately so
// the command thinks it's a real terminal
func NewTty(cmd *exec.Cmd) *Tty {
	tty := &Tty{}
	tty.cmd = cmd
	// Assign pty/tty so git thinks it's a real terminal
	tty.outpty, tty.outtty, _ = pty.Open()
	cmd.Stdin = tty.outtty
	cmd.Stdout = tty.outtty
	tty.errpty, tty.errtty, _ = pty.Open()
	cmd.Stderr = tty.errtty
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	cmd.SysProcAttr.Setctty = true
	cmd.SysProcAttr.Setsid = true

	return tty
}

// TestRunDetach checks attaching and detaching with the escape sequence.
func TestRunDetach(t *testing.T) {
	stdout, stdoutPipe := io.Pipe()
	cpty, tty, err := pty.Open()
	if err != nil {
		t.Fatal(err)
	}

	key, err := libtrust.GenerateECP256PrivateKey()
	if err != nil {
		t.Fatal(err)
	}

	cli := client.NewDockerCli(tty, stdoutPipe, ioutil.Discard, key, testDaemonProto, testDaemonAddr, nil)
	defer cleanup(globalEngine, t)

	ch := make(chan struct{})
	go func() {
		defer close(ch)
		cli.CmdRun("-i", "-t", unitTestImageID, "cat")
	}()

	container := waitContainerStart(t, 10*time.Second)

	state := setRaw(t, container)
	defer unsetRaw(t, container, state)

	setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() {
		if err := assertPipe("hello\n", "hello", stdout, cpty, 150); err != nil {
			t.Fatal(err)
		}
	})

	setTimeout(t, "Escape sequence timeout", 5*time.Second, func() {
		cpty.Write([]byte{16})
		time.Sleep(100 * time.Millisecond)
		cpty.Write([]byte{17})
	})

	// wait for CmdRun to return
	setTimeout(t, "Waiting for CmdRun timed out", 15*time.Second, func() {
		<-ch
	})
	closeWrap(cpty, stdout, stdoutPipe)

	time.Sleep(500 * time.Millisecond)
	if !container.IsRunning() {
		t.Fatal("The detached container should be still running")
	}

	setTimeout(t, "Waiting for container to die timed out", 20*time.Second, func() {
		container.Kill()
	})
}