func (s *UpgradeSuite) TestDowngradeOnMasterWhenOtherStateServerDoesntStartUpgrade(c *gc.C) {
	coretesting.SkipIfWindowsBug(c, "lp:1446885")
	// This test checks that the master triggers a downgrade if one of
	// the other state server fails to signal it is ready for upgrade.
	//
	// This test is functional, ensuring that the upgrader worker
	// terminates the machine agent with the UpgradeReadyError which
	// makes the downgrade happen.

	// Speed up the watcher frequency to make the test much faster.
	s.PatchValue(&watcher.Period, 200*time.Millisecond)

	// Provide (fake) tools so that the upgrader has something to downgrade to.
	envtesting.AssertUploadFakeToolsVersions(
		c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)

	// Only the first machine is going to be ready for upgrade.
	machineIdA, machineIdB, _ := s.createUpgradingStateServers(c)

	// One of the other state servers is ready for upgrade (but machine C doesn't).
	info, err := s.State.EnsureUpgradeInfo(machineIdB, s.oldVersion.Number, version.Current)
	c.Assert(err, jc.ErrorIsNil)

	agent := s.newAgentFromMachineId(c, machineIdA)
	defer agent.Stop()

	s.machineIsMaster = true

	var agentErr error
	agentDone := make(chan bool)
	go func() {
		agentErr = agent.Run(nil)
		close(agentDone)
	}()

	select {
	case <-agentDone:
		upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
		if !ok {
			c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
		}
		// Confirm that the downgrade is back to the previous version.
		current := version.Binary{
			Number: version.Current,
			Arch:   arch.HostArch(),
			Series: series.HostSeries(),
		}
		c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
		c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)

	case <-time.After(coretesting.LongWait):
		c.Fatal("machine agent did not exit as expected")
	}

	// UpgradeInfo doc should now be archived.
	err = info.Refresh()
	c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}

func (s *upgradeSuite) TestLoginsDuringUpgrade(c *gc.C) {
	coretesting.SkipIfWindowsBug(c, "lp:1446885")

	// Create machine agent to upgrade
	machine, machine0Conf := s.makeStateAgentVersion(c, s.oldVersion)

	// Set up a second machine to log in as. API logins are tested
	// manually so there's no need to actually start this machine.
	machine1, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
		Nonce: agent.BootstrapNonce,
	})
	machine1Conf, _ := s.PrimeAgent(c, machine1.Tag(), password)

	// Mock out upgrade logic, using a channel so that the test knows
	// when upgrades have started and can control when upgrades
	// should finish.
	upgradeCh := make(chan bool)
	abort := make(chan bool)
	fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
		// Signal that upgrade has started.
		select {
		case upgradeCh <- true:
		case <-abort:
			return nil
		}

		// Wait for signal that upgrades should finish.
		select {
		case <-upgradeCh:
		case <-abort:
			return nil
		}
		return nil
	}
	s.PatchValue(&upgradesteps.PerformUpgrade, fakePerformUpgrade)

	a := s.newAgent(c, machine)
	go func() { c.Check(a.Run(nil), jc.ErrorIsNil) }()
	defer func() { c.Check(a.Stop(), jc.ErrorIsNil) }()

	c.Assert(waitForUpgradeToStart(upgradeCh), jc.IsTrue)

	// Only user and local logins are allowed during upgrade. Users get a restricted API.
	s.checkLoginToAPIAsUser(c, machine0Conf, RestrictedAPIExposed)
	c.Assert(canLoginToAPIAsMachine(c, machine0Conf, machine0Conf), jc.IsTrue)
	c.Assert(canLoginToAPIAsMachine(c, machine1Conf, machine0Conf), jc.IsFalse)

	close(upgradeCh) // Allow upgrade to complete

	waitForUpgradeToFinish(c, machine0Conf)

	// All logins are allowed after upgrade
	s.checkLoginToAPIAsUser(c, machine0Conf, FullAPIExposed)
	c.Assert(canLoginToAPIAsMachine(c, machine0Conf, machine0Conf), jc.IsTrue)
	c.Assert(canLoginToAPIAsMachine(c, machine1Conf, machine0Conf), jc.IsTrue)
}

func (s *UpgradeSuite) TestUpgradeStepsHostMachine(c *gc.C) {
	coretesting.SkipIfPPC64EL(c, "lp:1444576")
	coretesting.SkipIfWindowsBug(c, "lp:1446885")
	s.setInstantRetryStrategy(c)
	// We need to first start up a state server that thinks it has already been upgraded.
	ss, _, _ := s.primeAgent(c, state.JobManageEnviron)
	a := s.newAgent(c, ss)
	go func() { c.Check(a.Run(nil), gc.IsNil) }()
	defer func() { c.Check(a.Stop(), gc.IsNil) }()
	// Now run the test.
	s.assertUpgradeSteps(c, state.JobHostUnits)
	s.assertHostUpgrades(c)
}

func (s *UpgradeSuite) TestUpgradeStepsStateServer(c *gc.C) {
	coretesting.SkipIfI386(c, "lp:1444576")
	coretesting.SkipIfPPC64EL(c, "lp:1444576")
	coretesting.SkipIfWindowsBug(c, "lp:1446885")
	s.setInstantRetryStrategy(c)
	// Upload tools to provider storage, so they can be migrated to environment storage.
	stor, err := environs.LegacyStorage(s.State)
	if !errors.IsNotSupported(err) {
		c.Assert(err, jc.ErrorIsNil)
		envtesting.AssertUploadFakeToolsVersions(
			c, stor, "releases", s.Environ.Config().AgentStream(), s.oldVersion)
	}
	s.assertUpgradeSteps(c, state.JobManageEnviron)
	s.assertStateServerUpgrades(c)
}

func (s *upgradeSuite) TestDowngradeOnMasterWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
	coretesting.SkipIfWindowsBug(c, "lp:1446885")

	// This test checks that the master triggers a downgrade if one of
	// the other controller fails to signal it is ready for upgrade.
	//
	// This test is functional, ensuring that the upgrader worker
	// terminates the machine agent with the UpgradeReadyError which
	// makes the downgrade happen.

	// Speed up the watcher frequency to make the test much faster.
	s.PatchValue(&watcher.Period, 200*time.Millisecond)

	// Provide (fake) tools so that the upgrader has something to downgrade to.
	envtesting.AssertUploadFakeToolsVersions(
		c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), s.oldVersion)

	// Create 3 controllers
	machineA, _ := s.makeStateAgentVersion(c, s.oldVersion)
	changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(len(changes.Added), gc.Equals, 2)
	machineB, _, _ := s.configureMachine(c, changes.Added[0], s.oldVersion)
	s.configureMachine(c, changes.Added[1], s.oldVersion)

	// One of the other controllers is ready for upgrade (but machine C isn't).
	info, err := s.State.EnsureUpgradeInfo(machineB.Id(), s.oldVersion.Number, jujuversion.Current)
	c.Assert(err, jc.ErrorIsNil)

	// Ensure the agent will think it's the master controller.
	fakeIsMachineMaster := func(*state.State, string) (bool, error) {
		return true, nil
	}
	s.PatchValue(&upgradesteps.IsMachineMaster, fakeIsMachineMaster)

	// Start the agent
	agent := s.newAgent(c, machineA)
	defer agent.Stop()
	agentDone := make(chan error)
	go func() {
		agentDone <- agent.Run(nil)
	}()

	select {
	case agentErr := <-agentDone:
		upgradeReadyErr, ok := agentErr.(*upgrader.UpgradeReadyError)
		if !ok {
			c.Fatalf("didn't see UpgradeReadyError, instead got: %v", agentErr)
		}
		// Confirm that the downgrade is back to the previous version.
		current := version.Binary{
			Number: jujuversion.Current,
			Arch:   arch.HostArch(),
			Series: series.HostSeries(),
		}
		c.Assert(upgradeReadyErr.OldTools, gc.Equals, current)
		c.Assert(upgradeReadyErr.NewTools, gc.Equals, s.oldVersion)

	case <-time.After(coretesting.LongWait):
		c.Fatal("machine agent did not exit as expected")
	}

	// UpgradeInfo doc should now be archived.
	err = info.Refresh()
	c.Assert(err, gc.ErrorMatches, "current upgrade info not found")
}