// addInstance creates a new service state and host instance
func addInstance(conn client.Connection, state ss.ServiceState) error {
	glog.V(2).Infof("Adding instance %+v", state)
	// check the object
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}

	// CC-1050: we need to trigger the scheduler in case we only have a
	// partial create.
	svclock := newStateLock(conn, state.ServiceID)
	if err := svclock.Lock(); err != nil {
		glog.Errorf("Could not set lock on service %s: %s", state.ServiceID, err)
		return err
	}
	defer svclock.Unlock()

	lock := newInstanceLock(conn, state.ID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
		return err
	}
	glog.V(2).Infof("Acquired lock for instance %s", state.ID)
	defer lock.Unlock()

	var err error
	defer func() {
		if err != nil {
			conn.Delete(hostpath(state.HostID, state.ID))
			conn.Delete(servicepath(state.ServiceID, state.ID))
			rmInstanceLock(conn, state.ID)
		}
	}()

	// Create node on the service
	spath := servicepath(state.ServiceID, state.ID)
	snode := &ServiceStateNode{ServiceState: &state}
	if err = conn.Create(spath, snode); err != nil {
		glog.Errorf("Could not create service state %s for service %s: %s", state.ID, state.ServiceID, err)
		return err
	} else if err = conn.Set(spath, snode); err != nil {
		glog.Errorf("Could not set service state %s for node %+v: %s", state.ID, snode, err)
		return err
	}

	// Create node on the host
	hpath := hostpath(state.HostID, state.ID)
	hnode := NewHostState(&state)
	glog.V(2).Infof("Host node: %+v", hnode)
	if err = conn.Create(hpath, hnode); err != nil {
		glog.Errorf("Could not create host state %s for host %s: %s", state.ID, state.HostID, err)
		return err
	} else if err = conn.Set(hpath, hnode); err != nil {
		glog.Errorf("Could not set host state %s for node %+v: %s", state.ID, hnode, err)
		return err
	}

	glog.V(2).Infof("Releasing lock for instance %s", state.ID)
	return nil
}

// UpdateServiceState does a full update of a service state
func UpdateServiceState(conn client.Connection, state *ss.ServiceState) error {
	if err := state.ValidEntity(); err != nil {
		glog.Errorf("Could not validate service state %+v: %s", state, err)
		return err
	}
	return updateInstance(conn, state.HostID, state.ID, func(_ *HostState, ssdata *ss.ServiceState) {
		*ssdata = *state
	})
}

// getStatus computes the status of a service state
func getStatus(conn client.Connection, state *servicestate.ServiceState) (dao.Status, error) {
	var status dao.Status

	// Set the state based on the service state object
	if !state.IsRunning() {
		status = dao.Stopped
	} else if state.IsPaused() {
		status = dao.Paused
	} else {
		status = dao.Running
	}

	// Set the state based on the host state object
	var hostState HostState
	if err := conn.Get(hostpath(state.HostID, state.ID), &hostState); err != nil && err != client.ErrNoNode {
		return dao.Status{}, err
	}

	if hostState.DesiredState == int(service.SVCStop) {
		switch status {
		case dao.Running, dao.Paused:
			status = dao.Stopping
		case dao.Stopped:
			// pass
		default:
			return dao.Status{}, ErrUnknownState
		}
	} else if hostState.DesiredState == int(service.SVCRun) {
		switch status {
		case dao.Stopped:
			status = dao.Starting
		case dao.Paused:
			status = dao.Resuming
		case dao.Running:
			// pass
		default:
			return dao.Status{}, ErrUnknownState
		}
	} else if hostState.DesiredState == int(service.SVCPause) {
		switch status {
		case dao.Running:
			status = dao.Pausing
		case dao.Paused, dao.Stopped:
			// pass
		default:
			return dao.Status{}, ErrUnknownState
		}
	} else {
		return dao.Status{}, ErrUnknownState
	}

	return status, nil
}

func updateInstance(state *servicestate.ServiceState, ctr *docker.Container) error {
	if _, err := ctr.Inspect(); err != nil {
		return err
	}
	state.DockerID = ctr.ID
	state.Started = ctr.Created
	state.PrivateIP = ctr.NetworkSettings.IPAddress
	state.PortMapping = make(map[string][]domain.HostIPAndPort)
	for k, v := range ctr.NetworkSettings.Ports {
		pm := []domain.HostIPAndPort{}
		for _, pb := range v {
			pm = append(pm, domain.HostIPAndPort{HostIP: pb.HostIp, HostPort: pb.HostPort})
			state.PortMapping[string(k)] = pm
		}
	}
	return nil
}

// AttachService attempts to attach to a running container
func (a *HostAgent) AttachService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
	ctr, err := docker.FindContainer(state.DockerID)
	if err != nil {
		return err
	}

	if !ctr.IsRunning() {
		defer exited(state.ID)
		return nil
	}

	ctr.OnEvent(docker.Die, func(cid string) {
		defer exited(state.ID)
		glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
		state.DockerID = cid
		a.removeInstance(state.ID, ctr)
	})

	go a.setProxy(svc, ctr)
	return nil
}

// StartService starts a new instance of the specified service and updates the control center state accordingly.
func (a *HostAgent) StartService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
	glog.V(2).Infof("About to start service %s with name %s", svc.ID, svc.Name)
	client, err := NewControlClient(a.master)
	if err != nil {
		glog.Errorf("Could not start ControlPlane client %v", err)
		return err
	}
	defer client.Close()

	// start from a known good state
	if state.DockerID != "" {
		if ctr, err := docker.FindContainer(state.DockerID); err != nil {
			glog.Errorf("Could not find container %s for %s", state.DockerID, state.ID)
		} else if err := ctr.Delete(true); err != nil {
			glog.Errorf("Could not delete container %s for %s", state.DockerID, state.ID)
		}
	}

	// create the docker client Config and HostConfig structures necessary to create and start the service
	config, hostconfig, err := configureContainer(a, client, svc, state, a.virtualAddressSubnet)
	if err != nil {
		glog.Errorf("can't configure container: %v", err)
		return err
	}

	cjson, _ := json.MarshalIndent(config, "", "     ")
	glog.V(3).Infof(">>> CreateContainerOptions:\n%s", string(cjson))

	hcjson, _ := json.MarshalIndent(hostconfig, "", "     ")
	glog.V(3).Infof(">>> HostConfigOptions:\n%s", string(hcjson))

	cd := &docker.ContainerDefinition{
		dockerclient.CreateContainerOptions{Name: state.ID, Config: config},
		*hostconfig,
	}

	ctr, err := docker.NewContainer(cd, false, 10*time.Second, nil, nil)
	if err != nil {
		glog.Errorf("Error trying to create container %v: %v", config, err)
		return err
	}

	var started sync.WaitGroup
	started.Add(1)
	ctr.OnEvent(docker.Start, func(cid string) {
		glog.Infof("Instance %s (%s) for %s (%s) has started", state.ID, ctr.ID, svc.Name, svc.ID)
		started.Done()
	})

	ctr.OnEvent(docker.Die, func(cid string) {
		defer exited(state.ID)
		glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
		state.DockerID = cid
		a.removeInstance(state.ID, ctr)
	})

	if err := ctr.Start(time.Hour); err != nil {
		glog.Errorf("Could not start service state %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
		a.removeInstance(state.ID, ctr)
		return err
	}

	started.Wait()
	if err := updateInstance(state, ctr); err != nil {
		glog.Errorf("Could not update instance %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
		ctr.Stop(45 * time.Second)
		return err
	}

	go a.setProxy(svc, ctr)
	return nil
}

// Spawn listens for changes in the host state and manages running instances
func (l *HostStateListener) Spawn(shutdown <-chan interface{}, stateID string) {
	var processDone <-chan struct{}

	// Let's have exclusive access to this node
	lock := newInstanceLock(l.conn, stateID)
	if err := lock.Lock(); err != nil {
		glog.Errorf("Could not lock service instance %s on host %s: %s", stateID, l.hostID, err)
		return
	}
	// Get the HostState node
	var hs HostState
	if err := l.conn.Get(hostpath(l.hostID, stateID), &hs); err != nil {
		glog.Errorf("Could not load host instance %s on host %s: %s", stateID, l.hostID, err)
		l.conn.Delete(hostpath(l.hostID, stateID))
		lock.Unlock()
		return
	}
	defer removeInstance(l.conn, hs.ServiceID, hs.HostID, hs.ServiceStateID)
	// Get the ServiceState node
	var ss servicestate.ServiceState
	if err := l.conn.Get(servicepath(hs.ServiceID, hs.ServiceStateID), &ServiceStateNode{ServiceState: &ss}); err != nil {
		glog.Errorf("Could not load service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
		lock.Unlock()
		return
	}
	defer l.stopInstance(processDone, &ss)
	lock.Unlock()

	for {
		// Get the HostState instance
		hsEvt, err := l.conn.GetW(hostpath(l.hostID, stateID), &hs)
		if err != nil {
			glog.Errorf("Could not load host instance %s on host %s: %s", stateID, l.hostID, err)
			return
		}
		// Get the ServiceState instance
		ssEvt, err := l.conn.GetW(servicepath(hs.ServiceID, stateID), &ServiceStateNode{ServiceState: &ss})
		if err != nil {
			glog.Errorf("Could not load service state %s for service %s on host %s: %s", stateID, hs.ServiceID, l.hostID, err)
			return
		}
		// Get the service
		var svc service.Service
		if err := l.conn.Get(servicepath(hs.ServiceID), &ServiceNode{Service: &svc}); err != nil {
			glog.Errorf("Could not load service %s for service instance %s on host %s: %s", hs.ServiceID, stateID, l.hostID, err)
			return
		}

		// Process the desired state
		glog.V(2).Infof("Processing %s (%s); Desired State: %d", svc.Name, svc.ID, hs.DesiredState)
		switch service.DesiredState(hs.DesiredState) {
		case service.SVCRun:
			var err error
			if !ss.IsRunning() {
				// process has stopped
				glog.Infof("Starting a new instance for %s (%s): %s", svc.Name, svc.ID, stateID)
				if processDone, err = l.startInstance(&svc, &ss); err != nil {
					glog.Errorf("Could not start service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
					return
				}
			} else if processDone == nil {
				glog.Infof("Attaching to instance %s for %s (%s) via %s", stateID, svc.Name, svc.ID, ss.DockerID)
				if processDone, err = l.attachInstance(&svc, &ss); err != nil {
					glog.Errorf("Could not start service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
					return
				}
			}
			if ss.IsPaused() {
				glog.Infof("Resuming paused instance %s for service %s (%s)", stateID, svc.Name, svc.ID)
				if err := l.resumeInstance(&svc, &ss); err != nil {
					glog.Errorf("Could not resume paused instance %s for service %s (%s): %s", stateID, svc.Name, svc.ID, err)
					return
				}
			}
		case service.SVCPause:
			if !ss.IsPaused() {
				if err := l.pauseInstance(&svc, &ss); err != nil {
					glog.Errorf("Could not pause instance %s for service %s (%s): %s", stateID, svc.Name, svc.ID, err)
					return
				}
			}
		case service.SVCStop:
			return
		default:
			glog.V(2).Infof("Unhandled state (%d) of instance %s for service %s (%s)", hs.DesiredState, stateID, svc.Name, svc.ID, err)
		}

		select {
		case <-processDone:
			glog.V(2).Infof("Process ended for instance %s for service %s (%s)", stateID, svc.Name, svc.ID)
		case e := <-hsEvt:
			glog.V(3).Infof("Host instance %s for service %s (%s) received an event: %+v", stateID, svc.Name, svc.ID, e)
			if e.Type == client.EventNodeDeleted {
				return
			}
		case e := <-ssEvt:
			glog.V(3).Infof("Service instance %s for service %s (%s) received an event: %+v", stateID, svc.Name, svc.ID, e)
			if e.Type == client.EventNodeDeleted {
				return
//.........这里部分代码省略.........