// GetHealthCheck returns the health check configuration for a service, if it exists
func (a *HostAgent) GetHealthCheck(req HealthCheckRequest, healthChecks *map[string]domain.HealthCheck) error {
	glog.V(4).Infof("ControlPlaneAgent.GetHealthCheck()")
	*healthChecks = make(map[string]domain.HealthCheck, 0)

	controlClient, err := NewControlClient(a.master)
	if err != nil {
		glog.Errorf("Could not start ControlPlane client %v", err)
		return err
	}
	defer controlClient.Close()

	var svc service.Service
	err = controlClient.GetService(req.ServiceID, &svc)
	if err != nil {
		return err
	}
	getSvc := func(svcID string) (service.Service, error) {
		svc := service.Service{}
		err := controlClient.GetService(svcID, &svc)
		return svc, err
	}

	findChild := func(svcID, childName string) (service.Service, error) {
		svc := service.Service{}
		err := controlClient.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
		return svc, err
	}
	svc.EvaluateHealthCheckTemplate(getSvc, findChild, req.InstanceID)
	if svc.HealthChecks != nil {
		*healthChecks = svc.HealthChecks
	}
	return nil
}

func (a *HostAgent) GetService(serviceID string, response *service.Service) (err error) {
	*response = service.Service{}

	controlClient, err := NewControlClient(a.master)
	if err != nil {
		glog.Errorf("Could not start ControlPlane client %v", err)
		return nil
	}
	defer controlClient.Close()

	err = controlClient.GetService(serviceID, response)
	if response == nil {
		*response = service.Service{}
	}
	if err != nil {
		return err
	}

	getSvc := func(svcID string) (service.Service, error) {
		svc := service.Service{}
		err := controlClient.GetService(svcID, &svc)
		return svc, err
	}

	findChild := func(svcID, childName string) (service.Service, error) {
		svc := service.Service{}
		err := controlClient.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
		return svc, err
	}

	return response.Evaluate(getSvc, findChild, 0)
}

func (dt *DaoTest) TestDao_NewSnapshot(t *C) {
	t.Skip("TODO: fix this test")
	// this is technically not a unit test since it depends on the leader
	// starting a watch for snapshot requests and the code here is time
	// dependent waiting for that leader to start the watch
	return

	glog.V(0).Infof("TestDao_NewSnapshot started")
	defer glog.V(0).Infof("TestDao_NewSnapshot finished")

	time.Sleep(2 * time.Second) // wait for Leader to start watching for snapshot requests

	service := service.Service{}
	service.ID = "service-without-quiesce"
	dt.Dao.RemoveService(service.ID, &unused)
	// snapshot should work for services without Snapshot Pause/Resume
	err := dt.Dao.AddService(service, &id)
	if err != nil {
		t.Fatalf("Failure creating service %+v with error: %s", service, err)
	}

	service.ID = "service1-quiesce"
	dt.Dao.RemoveService(service.ID, &unused)
	service.Snapshot.Pause = fmt.Sprintf("STATE=paused echo %s quiesce $STATE", service.ID)
	service.Snapshot.Resume = fmt.Sprintf("STATE=resumed echo %s quiesce $STATE", service.ID)
	err = dt.Dao.AddService(service, &id)
	if err != nil {
		t.Fatalf("Failure creating service %+v with error: %s", service, err)
	}

	service.ID = "service2-quiesce"
	dt.Dao.RemoveService(service.ID, &unused)
	service.Snapshot.Pause = fmt.Sprintf("STATE=paused echo %s quiesce $STATE", service.ID)
	service.Snapshot.Resume = fmt.Sprintf("STATE=resumed echo %s quiesce $STATE", service.ID)
	err = dt.Dao.AddService(service, &id)
	if err != nil {
		t.Fatalf("Failure creating service %+v with error: %s", service, err)
	}

	err = dt.Dao.Snapshot(service.ID, &id)
	if err != nil {
		t.Fatalf("Failure creating snapshot for service %+v with error: %s", service, err)
	}
	if id == "" {
		t.Fatalf("Failure creating snapshot for service %+v - label is empty", service)
	}
	glog.V(0).Infof("successfully created 1st snapshot with label:%s", id)

	err = dt.Dao.Snapshot(service.ID, &id)
	if err != nil {
		t.Fatalf("Failure creating snapshot for service %+v with error: %s", service, err)
	}
	if id == "" {
		t.Fatalf("Failure creating snapshot for service %+v - label is empty", service)
	}
	glog.V(0).Infof("successfully created 2nd snapshot with label:%s", id)

	time.Sleep(10 * time.Second)
}

// Use the Context field of the given template to fill in all the templates in
// the Command fields of the template's ServiceDefinitions
func injectContext(s *service.Service, svcState *servicestate.ServiceState, cp dao.ControlPlane) error {
	getSvc := func(svcID string) (service.Service, error) {
		svc := service.Service{}
		err := cp.GetService(svcID, &svc)
		return svc, err
	}
	findChild := func(svcID, childName string) (service.Service, error) {
		svc := service.Service{}
		err := cp.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
		return svc, err
	}

	return s.Evaluate(getSvc, findChild, svcState.InstanceID)
}

func (dt *DaoTest) TestDao_UpdateService(t *C) {
	dt.Dao.RemoveService("default", &unused)

	svc, _ := service.NewService()
	svc.ID = "default0"
	svc.Name = "default0"
	svc.PoolID = "default"
	svc.Launch = "auto"
	svc.DeploymentID = "deployment_id"
	err := dt.Dao.AddService(*svc, &id)
	t.Assert(err, IsNil)

	svc.Name = "name"
	err = dt.Dao.UpdateService(*svc, &unused)
	if err != nil {
		t.Errorf("Failure updating service %-v with error: %s", svc, err)
		t.Fail()
	}

	result := service.Service{}
	dt.Dao.GetService("default0", &result)
	//XXX the time.Time types fail comparison despite being equal...
	//	  as far as I can tell this is a limitation with Go
	result.UpdatedAt = svc.UpdatedAt
	result.CreatedAt = svc.CreatedAt
	if !svc.Equals(&result) {
		t.Errorf("Expected Service %+v, Actual Service %+v", result, *svc)
		t.Fail()
	}

	svc, _ = service.NewService()
	svc.ID = "default1"
	svc.Name = "default1"
	svc.PoolID = "default"
	svc.Launch = "auto"
	svc.DeploymentID = "deployment_id"
	err = dt.Dao.AddService(*svc, &id)
	t.Assert(err, IsNil)

	svc.Name = "name"
	err = dt.Dao.UpdateService(*svc, &unused)
	if err == nil {
		t.Errorf("Expected error updating service with same name and parent", svc)
		t.Fail()
	}
}

// restRemoveVirtualHost removes a vhost name from provided service and endpoint. Parameters are defined in path.
func restRemoveVirtualHost(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	serviceID, err := url.QueryUnescape(r.PathParam("serviceId"))
	if err != nil {
		glog.Errorf("Failed getting serviceId: %v", err)
		restBadRequest(w, err)
		return
	}
	application, err := url.QueryUnescape(r.PathParam("application"))
	if err != nil {
		glog.Errorf("Failed getting application: %v", err)
		restBadRequest(w, err)
		return
	}

	hostname, err := url.QueryUnescape(r.PathParam("name"))
	if err != nil {
		glog.Errorf("Failed getting hostname: %v", err)
		restBadRequest(w, err)
		return
	}

	var service service.Service
	err = client.GetService(serviceID, &service)
	if err != nil {
		glog.Errorf("Unexpected error getting service (%s): %v", serviceID, err)
		restServerError(w, err)
		return
	}

	err = service.RemoveVirtualHost(application, hostname)
	if err != nil {
		glog.Errorf("Unexpected error removing vhost, %s, from service (%s): %v", hostname, serviceID, err)
		restServerError(w, err)
		return
	}

	var unused int
	err = client.UpdateService(service, &unused)
	if err != nil {
		glog.Errorf("Unexpected error removing vhost, %s, from service (%s): %v", hostname, serviceID, err)
		restServerError(w, err)
		return
	}

	restSuccess(w)
}

func (f *Facade) getTenantIDAndPath(ctx datastore.Context, svc service.Service) (string, string, error) {
	gs := func(id string) (service.Service, error) {
		return f.getService(ctx, id)
	}

	tenantID, err := f.GetTenantID(ctx, svc.ID)
	if err != nil {
		return "", "", err
	}

	path, err := svc.GetPath(gs)
	if err != nil {
		return "", "", err
	}

	return tenantID, path, err
}

func (dt *DaoTest) TestDao_NewService(t *C) {
	svc := service.Service{}
	err := dt.Dao.AddService(svc, &id)
	if err == nil {
		t.Errorf("Expected failure to create service %-v", svc)
		t.Fail()
	}

	svc.ID = "default"
	svc.Name = "default"
	svc.PoolID = "default"
	svc.Launch = "auto"
	svc.DeploymentID = "deployment_id"
	err = dt.Dao.AddService(svc, &id)
	if err != nil {
		t.Errorf("Failure creating service %-v with error: %s", svc, err)
		t.Fail()
	}

	err = dt.Dao.AddService(svc, &id)
	if err == nil {
		t.Errorf("Expected error creating redundant service %-v", svc)
		t.Fail()
	}

	svc.ID = ""
	err = dt.Dao.AddService(svc, &id)
	if err == nil {
		t.Errorf("Expected error creating service with same name and parent", svc)
		t.Fail()
	}
}

// AddService adds a service; return error if service already exists
func (f *Facade) AddService(ctx datastore.Context, svc service.Service) error {
	glog.V(2).Infof("Facade.AddService: %+v", svc)
	store := f.serviceStore

	_, err := store.Get(ctx, svc.ID)
	if err != nil && !datastore.IsErrNoSuchEntity(err) {
		return err
	} else if err == nil {
		return fmt.Errorf("error adding service; %v already exists", svc.ID)
	}

	// verify the service with parent ID does not exist with the given name
	if s, err := store.FindChildService(ctx, svc.DeploymentID, svc.ParentServiceID, svc.Name); err != nil {
		glog.Errorf("Could not verify service path for %s: %s", svc.Name, err)
		return err
	} else if s != nil {
		err := fmt.Errorf("service %s found at %s", svc.Name, svc.ParentServiceID)
		glog.Errorf("Cannot create service %s: %s", svc.Name, err)
		return err
	}

	// Strip the database version; we already know this is a create
	svc.DatabaseVersion = 0

	// Save a copy for checking configs later
	svcCopy := svc

	err = store.Put(ctx, &svc)
	if err != nil {
		glog.V(2).Infof("Facade.AddService: %+v", err)
		return err
	}
	glog.V(2).Infof("Facade.AddService: id %+v", svc.ID)

	// Compare the incoming config files to see if there are modifications from
	// the original. If there are, we need to perform an update to add those
	// modifications to the service.
	if svcCopy.OriginalConfigs != nil && !reflect.DeepEqual(svcCopy.OriginalConfigs, svcCopy.ConfigFiles) {
		// Get the current service in order to get the database version. We
		// don't save this because it won't have any of the updated config
		// files, among other things.
		cursvc, err := store.Get(ctx, svc.ID)
		if err != nil {
			glog.V(2).Infof("Facade.AddService: %+v", err)
			return err
		}
		svcCopy.DatabaseVersion = cursvc.DatabaseVersion

		for key, _ := range svcCopy.OriginalConfigs {
			glog.V(2).Infof("Facade.AddService: calling updateService for %s due to OriginalConfigs of %+v", svc.Name, key)
		}
		return f.updateService(ctx, &svcCopy)
	}

	glog.V(2).Infof("Facade.AddService: calling zk.updateService for %s %d ConfigFiles", svc.Name, len(svc.ConfigFiles))
	return zkAPI(f).UpdateService(&svc)
}

func TestServicedCLI_CmdServiceList_one(t *testing.T) {
	serviceID := "test-service-1"

	expected, err := DefaultServiceAPITest.GetService(serviceID)
	if err != nil {
		t.Fatal(err)
	}

	var actual service.Service
	output := pipe(InitServiceAPITest, "serviced", "service", "list", serviceID)
	if err := json.Unmarshal(output, &actual); err != nil {
		t.Fatalf("error unmarshaling resource: %s", err)
	}

	// Did you remember to update Service.Equals?
	if !actual.Equals(expected) {
		t.Fatalf("\ngot:\n%+v\nwant:\n%+v", actual, expected)
	}
}

func (dt *DaoTest) TestDao_GetService(t *C) {
	svc, _ := service.NewService()
	svc.Name = "testname"
	svc.PoolID = "default"
	svc.Launch = "auto"
	svc.DeploymentID = "deployment_id"
	err := dt.Dao.AddService(*svc, &id)
	t.Assert(err, IsNil)

	var result service.Service
	err = dt.Dao.GetService(svc.ID, &result)
	t.Assert(err, IsNil)
	//XXX the time.Time types fail comparison despite being equal...
	//	  as far as I can tell this is a limitation with Go
	result.UpdatedAt = svc.UpdatedAt
	result.CreatedAt = svc.CreatedAt
	if !svc.Equals(&result) {
		t.Errorf("GetService Failed: expected=%+v, actual=%+v", svc, result)
	}
}

func (f *Facade) UpdateService(ctx datastore.Context, svc service.Service) error {
	glog.V(2).Infof("Facade.UpdateService: %+v", svc)
	//cannot update service without validating it.
	if svc.DesiredState != int(service.SVCStop) {
		if err := f.validateServicesForStarting(ctx, &svc); err != nil {
			glog.Warningf("Could not validate service %s (%s) for starting: %s", svc.Name, svc.ID, err)
			svc.DesiredState = int(service.SVCStop)
		}

		for _, ep := range svc.GetServiceVHosts() {
			for _, vh := range ep.VHosts {
				//check that vhosts aren't already started elsewhere
				if err := zkAPI(f).CheckRunningVHost(vh, svc.ID); err != nil {
					return err
				}
			}
		}
	}

	return f.updateService(ctx, &svc)
}

// UpdateServiceVhosts updates vhosts of a service
func UpdateServiceVhosts(conn client.Connection, svc *service.Service) error {
	glog.V(2).Infof("UpdateServiceVhosts for ID:%s Name:%s", svc.ID, svc.Name)

	// generate map of current vhosts
	currentvhosts := map[string]string{}
	if svcvhosts, err := conn.Children(zkServiceVhosts); err == client.ErrNoNode {
		/*
			// do not do this, otherwise, nodes aren't deleted when calling RemoveServiceVhost

			if exists, err := zzk.PathExists(conn, zkServiceVhosts); err != nil {
				return err
			} else if !exists {
				err := conn.CreateDir(zkServiceVhosts)
				if err != client.ErrNodeExists && err != nil {
					return err
				}
			}
		*/
	} else if err != nil {
		glog.Errorf("UpdateServiceVhosts unable to retrieve vhost children at path %s %s", zkServiceVhosts, err)
		return err
	} else {
		for _, svcvhost := range svcvhosts {
			parts := strings.SplitN(svcvhost, "_", 2)
			vhostname := parts[1]
			currentvhosts[svcvhost] = vhostname
		}
	}
	glog.V(2).Infof("  currentvhosts %+v", currentvhosts)

	// generate map of vhosts in the service
	svcvhosts := map[string]string{}
	for _, ep := range svc.GetServiceVHosts() {
		for _, vhostname := range ep.VHosts {
			svcvhosts[fmt.Sprintf("%s_%s", svc.ID, vhostname)] = vhostname
		}
	}
	glog.V(2).Infof("  svcvhosts %+v", svcvhosts)

	// remove vhosts in current not in svc that match serviceid
	for sv, vhostname := range currentvhosts {
		svcID := strings.SplitN(sv, "_", 2)[0]
		if svcID != svc.ID {
			continue
		}

		if _, ok := svcvhosts[sv]; !ok {
			if err := RemoveServiceVhost(conn, svc.ID, vhostname); err != nil {
				return err
			}
		}
	}

	// add vhosts from svc not in current
	for sv, vhostname := range svcvhosts {
		if _, ok := currentvhosts[sv]; !ok {
			if err := UpdateServiceVhost(conn, svc.ID, vhostname); err != nil {
				return err
			}
		}
	}

	return nil
}

// restAddVirtualHost parses payload, adds the vhost to the service, then updates the service
func restAddVirtualHost(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	var request virtualHostRequest
	err := r.DecodeJsonPayload(&request)
	if err != nil {
		restBadRequest(w, err)
		return
	}

	var services []service.Service
	var serviceRequest dao.ServiceRequest
	if err := client.GetServices(serviceRequest, &services); err != nil {
		glog.Errorf("Could not get services: %v", err)
		restServerError(w, err)
		return
	}

	var service *service.Service
	for _, _service := range services {
		if _service.ID == request.ServiceID {
			service = &_service
			break
		}
	}

	if service == nil {
		glog.Errorf("Could not find service: %s", services)
		restServerError(w, err)
		return
	}

	//checkout other virtual hosts for redundancy
	_vhost := strings.ToLower(request.VirtualHostName)
	for _, service := range services {
		if service.Endpoints == nil {
			continue
		}

		for _, endpoint := range service.Endpoints {
			for _, host := range endpoint.VHosts {
				if host == _vhost {
					glog.Errorf("vhost %s already defined for service: %s", request.VirtualHostName, service.ID)
					restServerError(w, err)
					return
				}
			}
		}
	}

	err = service.AddVirtualHost(request.Application, request.VirtualHostName)
	if err != nil {
		glog.Errorf("Unexpected error adding vhost to service (%s): %v", service.Name, err)
		restServerError(w, err)
		return
	}

	var unused int
	err = client.UpdateService(*service, &unused)
	if err != nil {
		glog.Errorf("Unexpected error adding vhost to service (%s): %v", service.Name, err)
		restServerError(w, err)
		return
	}

	restSuccess(w)
}

// updateService internal method to use when service has been validated
func (f *Facade) updateService(ctx datastore.Context, svc *service.Service) error {
	id := strings.TrimSpace(svc.ID)
	if id == "" {
		return errors.New("empty Service.ID not allowed")
	}
	svc.ID = id
	//add assignment info to service so it is availble in zk
	f.fillServiceAddr(ctx, svc)

	svcStore := f.serviceStore

	// verify the service with name and parent does not collide with another existing service
	if s, err := svcStore.FindChildService(ctx, svc.DeploymentID, svc.ParentServiceID, svc.Name); err != nil {
		glog.Errorf("Could not verify service path for %s: %s", svc.Name, err)
		return err
	} else if s != nil {
		if s.ID != svc.ID {
			err := fmt.Errorf("service %s found at %s", svc.Name, svc.ParentServiceID)
			glog.Errorf("Cannot update service %s: %s", svc.Name, err)
			return err
		}
	}

	oldSvc, err := svcStore.Get(ctx, svc.ID)
	if err != nil {
		return err
	}

	//Deal with Service Config Files
	//For now always make sure originalConfigs stay the same, essentially they are immutable
	svc.OriginalConfigs = oldSvc.OriginalConfigs

	//check if config files haven't changed
	if !reflect.DeepEqual(oldSvc.OriginalConfigs, svc.ConfigFiles) {
		//lets validate Service before doing more work....
		if err := svc.ValidEntity(); err != nil {
			return err
		}

		tenantID, servicePath, err := f.getTenantIDAndPath(ctx, *svc)
		if err != nil {
			return err
		}

		newConfs := make(map[string]*serviceconfigfile.SvcConfigFile)
		//config files are different, for each one that is different validate and add to newConfs
		for key, oldConf := range oldSvc.OriginalConfigs {
			if conf, found := svc.ConfigFiles[key]; found {
				if !reflect.DeepEqual(oldConf, conf) {
					newConf, err := serviceconfigfile.New(tenantID, servicePath, conf)
					if err != nil {
						return err
					}
					newConfs[key] = newConf
				}
			}
		}

		//Get current stored conf files and replace as needed
		configStore := serviceconfigfile.NewStore()
		existingConfs, err := configStore.GetConfigFiles(ctx, tenantID, servicePath)
		if err != nil {
			return err
		}
		foundConfs := make(map[string]*serviceconfigfile.SvcConfigFile)
		for _, svcConfig := range existingConfs {
			foundConfs[svcConfig.ConfFile.Filename] = svcConfig
		}
		//add or replace stored service config
		for _, newConf := range newConfs {
			if existing, found := foundConfs[newConf.ConfFile.Filename]; found {
				newConf.ID = existing.ID
				//delete it from stored confs, left overs will be deleted from DB
				delete(foundConfs, newConf.ConfFile.Filename)
			}
			configStore.Put(ctx, serviceconfigfile.Key(newConf.ID), newConf)
		}
		//remove leftover non-updated stored confs, conf was probably reverted to original or no longer exists
		for _, confToDelete := range foundConfs {
			configStore.Delete(ctx, serviceconfigfile.Key(confToDelete.ID))
		}
	}

	svc.UpdatedAt = time.Now()
	if err := svcStore.Put(ctx, svc); err != nil {
		return err
	}

	// Remove the service from zookeeper if the pool ID has changed
	if oldSvc.PoolID != svc.PoolID {
		if err := zkAPI(f).RemoveService(oldSvc); err != nil {
			// Synchronizer will eventually clean this service up
			glog.Warningf("ZK: Could not delete service %s (%s) from pool %s: %s", svc.Name, svc.ID, oldSvc.PoolID, err)
			oldSvc.DesiredState = int(service.SVCStop)
			zkAPI(f).UpdateService(oldSvc)
		}
	}

	return zkAPI(f).UpdateService(svc)
//.........这里部分代码省略.........

func restAddService(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	var svc service.Service
	var serviceID string
	err := r.DecodeJsonPayload(&svc)
	if err != nil {
		glog.V(1).Info("Could not decode service payload: ", err)
		restBadRequest(w, err)
		return

	}
	if id, err := utils.NewUUID36(); err != nil {
		restBadRequest(w, err)
		return
	} else {
		svc.ID = id
	}
	now := time.Now()
	svc.CreatedAt = now
	svc.UpdatedAt = now

	//for each endpoint, evaluate its EndpointTemplates
	getSvc := func(svcID string) (service.Service, error) {
		svc := service.Service{}
		err := client.GetService(svcID, &svc)
		return svc, err
	}
	findChild := func(svcID, childName string) (service.Service, error) {
		svc := service.Service{}
		err := client.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
		return svc, err
	}
	if err = svc.EvaluateEndpointTemplates(getSvc, findChild); err != nil {
		glog.Errorf("Unable to evaluate service endpoints: %v", err)
		restServerError(w, err)
		return
	}

	tags := map[string][]string{
		"controlplane_service_id": []string{svc.ID},
	}
	profile, err := svc.MonitoringProfile.ReBuild("1h-ago", tags)
	if err != nil {
		glog.Errorf("Unable to rebuild service monitoring profile: %v", err)
		restServerError(w, err)
		return
	}
	svc.MonitoringProfile = *profile

	//add the service to the data store
	err = client.AddService(svc, &serviceID)
	if err != nil {
		glog.Errorf("Unable to add service: %v", err)
		restServerError(w, err)
		return
	}

	//automatically assign virtual ips to new service
	request := dao.AssignmentRequest{ServiceID: svc.ID, IPAddress: "", AutoAssignment: true}
	if err := client.AssignIPs(request, nil); err != nil {
		glog.Errorf("Failed to automatically assign IPs: %+v -> %v", request, err)
		restServerError(w, err)
		return
	}

	glog.V(0).Info("Added service ", serviceID)
	w.WriteJson(&simpleResponse{"Added service", serviceLinks(serviceID)})
}

// sync synchronizes the number of running instances for this service
func (l *ServiceListener) sync(svc *service.Service, rss []dao.RunningService) bool {
	// sort running services by instance ID, so that you stop instances by the
	// lowest instance ID first and start instances with the greatest instance
	// ID last.
	sort.Sort(instances(rss))

	// resume any paused running services
	for _, state := range rss {
		// resumeInstance updates the service state ONLY if it has a PAUSED DesiredState
		if err := resumeInstance(l.conn, state.HostID, state.ID); err != nil {
			glog.Warningf("Could not resume paused service instance %s (%s) for service %s on host %s: %s", state.ID, state.Name, state.ServiceID, state.HostID, err)
		}
	}

	// if the service has a change option for restart all on changed, stop all
	// instances and wait for the nodes to stop.  Once all service instances
	// have been stopped (deleted), then go ahead and start the instances back
	// up.
	if count := len(rss); count > 0 && count != svc.Instances && utils.StringInSlice("restartAllOnInstanceChanged", svc.ChangeOptions) {
		svc.Instances = 0 // NOTE: this will not update the node in zk or elastic
	}

	// netInstances is the difference between the number of instances that
	// should be running, as described by the service from the number of
	// instances that are currently running
	netInstances := svc.Instances - len(rss)

	if netInstances > 0 {
		// If the service lock is enabled, do not try to start any service instances
		// This will prevent the retry restart from activating
		if locked, err := IsServiceLocked(l.conn); err != nil {
			glog.Errorf("Could not check service lock: %s", err)
			return true
		} else if locked {
			glog.Warningf("Could not start %d instances; service %s (%s) is locked", netInstances, svc.Name, svc.ID)
			return true
		}

		// the number of running instances is *less* than the number of
		// instances that need to be running, so schedule instances to start
		glog.V(2).Infof("Starting %d instances of service %s (%s)", netInstances, svc.Name, svc.ID)
		var (
			last        = 0
			instanceIDs = make([]int, netInstances)
		)

		// Find which instances IDs are being unused and add those instances
		// first.  All SERVICES must have an instance ID of 0, if instance ID
		// zero dies for whatever reason, then the service must schedule
		// another 0-id instance to take its place.
		j := 0
		for i := range instanceIDs {
			for j < len(rss) && last == rss[j].InstanceID {
				// if instance ID exists, then keep searching the list for
				// the next unique instance ID
				last += 1
				j += 1
			}
			instanceIDs[i] = last
			last += 1
		}

		return netInstances == l.start(svc, instanceIDs)
	} else if netInstances = -netInstances; netInstances > 0 {
		// the number of running instances is *greater* than the number of
		// instances that need to be running, so schedule instances to stop of
		// the highest instance IDs.
		glog.V(2).Infof("Stopping %d of %d instances of service %s (%s)", netInstances, len(rss), svc.Name, svc.ID)
		l.stop(rss[svc.Instances:])
	}

	return true
}