func mongoRestoreCall(runCommand utilsRun, tmpDir, mongoPath, adminPassword, migrationName string,
	dbs []string, statePort int, invalidSSL bool, batchSize int, callArgs retry.CallArgs) error {
	mongorestore := filepath.Join(mongoPath, "mongorestore")
	restoreParams := []string{
		"--ssl",
		"--port", strconv.Itoa(statePort),
		"--host", "localhost",
	}

	if invalidSSL {
		restoreParams = append(restoreParams, "--sslAllowInvalidCertificates")
	}
	if batchSize > 0 {
		restoreParams = append(restoreParams, "--batchSize", strconv.Itoa(batchSize))
	}
	if adminPassword != "" {
		restoreParams = append(restoreParams, "-u", "admin", "-p", adminPassword)
	}
	var out string
	if len(dbs) == 0 || dbs == nil {
		restoreParams = append(restoreParams, filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName)))
		restoreCallArgs := callArgs
		restoreCallArgs.Func = func() error {
			var err error
			out, err = runCommand(mongorestore, restoreParams...)
			if err == nil {
				return nil
			}
			logger.Errorf("cannot restore %v: %s", err, out)
			return err
		}
		if err := retry.Call(restoreCallArgs); err != nil {
			err := errors.Annotatef(err, "cannot restore dbs got: %s", out)
			logger.Errorf("%#v", err)
			return err
		}
	}
	for i := range dbs {
		restoreDbParams := append(restoreParams,
			fmt.Sprintf("--db=%s", dbs[i]),
			filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName), dbs[i]))
		restoreCallArgs := callArgs
		restoreCallArgs.Func = func() error {
			var err error
			out, err = runCommand(mongorestore, restoreDbParams...)
			if err == nil {
				return nil
			}
			logger.Errorf("cannot restore db %q: %v: got %s", dbs[i], err, out)
			return err
		}
		if err := retry.Call(restoreCallArgs); err != nil {
			return errors.Annotatef(err, "cannot restore db %q got: %s", dbs[i], out)
		}
		logger.Infof("Succesfully restored db %q", dbs[i])
	}
	return nil
}

// deleteSecurityGroup attempts to delete the security group. Should it fail,
// the deletion is retried due to timing issues in openstack. A security group
// cannot be deleted while it is in use. Theoretically we terminate all the
// instances before we attempt to delete the associated security groups, but
// in practice nova hasn't always finished with the instance before it
// returns, so there is a race condition where we think the instance is
// terminated and hence attempt to delete the security groups but nova still
// has it around internally. To attempt to catch this timing issue, deletion
// of the groups is tried multiple times.
func deleteSecurityGroup(novaclient *nova.Client, name, id string) {
	logger.Debugf("deleting security group %q", name)
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			return novaclient.DeleteSecurityGroup(id)
		},
		NotifyFunc: func(err error, attempt int) {
			if attempt%4 == 0 {
				message := fmt.Sprintf("waiting to delete security group %q", name)
				if attempt != 4 {
					message = "still " + message
				}
				logger.Debugf(message)
			}
		},
		Attempts: 30,
		Delay:    time.Second,
		// TODO(dimitern): This should be fixed to take a clock.Clock arg, not
		// hard-coded WallClock, like in provider/ec2/securitygroups_test.go!
		// See PR juju:#5197, especially the code around autoAdvancingClock.
		// LP Bug: http://pad.lv/1580626.
		Clock: clock.WallClock,
	})
	if err != nil {
		logger.Warningf("cannot delete security group %q. Used by another model?", name)
	}
}

// GetResource returns a reader for the resource's data.
func (client CSRetryClient) GetResource(req charmstore.ResourceRequest) (charmstore.ResourceData, error) {
	args := client.retryArgs // a copy

	var data charmstore.ResourceData
	args.Func = func() error {
		var err error
		data, err = client.Client.GetResource(req)
		if err != nil {
			return errors.Trace(err)
		}
		return nil
	}

	var lastErr error
	args.NotifyFunc = func(err error, i int) {
		// Remember the error we're hiding and then retry!
		logger.Debugf("(attempt %d) retrying resource download from charm store due to error: %v", i, err)
		lastErr = err
	}

	err := retry.Call(args)
	if retry.IsAttemptsExceeded(err) {
		return data, errors.Annotate(lastErr, "failed after retrying")
	}
	if err != nil {
		return data, errors.Trace(err)
	}

	return data, nil
}

// dialAndLogin returns a mongo session logged in as a user with administrative
// privileges
func dialAndLogin(mongoInfo *mongo.MongoInfo, callArgs retry.CallArgs) (mgoSession, mgoDb, error) {
	var session *mgo.Session
	opts := mongo.DefaultDialOpts()
	callArgs.Func = func() error {
		// Try to connect, retry a few times until the db comes up.
		var err error
		session, err = mongo.DialWithInfo(mongoInfo.Info, opts)
		if err == nil {
			return nil
		}
		logger.Errorf("cannot open mongo connection: %v", err)
		return err
	}
	if err := retry.Call(callArgs); err != nil {
		return nil, nil, errors.Annotate(err, "error dialing mongo to resume HA")
	}
	admin := session.DB("admin")
	if mongoInfo.Tag != nil {
		if err := admin.Login(mongoInfo.Tag.String(), mongoInfo.Password); err != nil {
			return nil, nil, errors.Annotatef(err, "cannot log in to admin database as %q", mongoInfo.Tag)
		}
	} else if mongoInfo.Password != "" {
		if err := admin.Login(mongo.AdminUser, mongoInfo.Password); err != nil {
			return nil, nil, errors.Annotate(err, "cannot log in to admin database")
		}
	}
	return session, admin, nil
}

func mongoDumpCall(
	runCommand utilsRun, tmpDir, mongoPath, adminPassword, migrationName string,
	statePort int, callArgs retry.CallArgs,
) (string, error) {
	mongodump := filepath.Join(mongoPath, "mongodump")
	dumpParams := []string{
		"--ssl",
		"-u", "admin",
		"-p", adminPassword,
		"--port", strconv.Itoa(statePort),
		"--host", "localhost",
		"--out", filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName)),
	}
	var out string
	callArgs.Func = func() error {
		var err error
		out, err = runCommand(mongodump, dumpParams...)
		if err == nil {
			return nil
		}
		logger.Errorf("cannot dump db %v: %s", err, out)
		return err
	}
	if err := retry.Call(callArgs); err != nil {
		logger.Errorf(out)
		return out, errors.Annotate(err, "cannot dump mongo db")
	}
	return out, nil
}

// APICall places a call to the remote machine.
//
// This fills out the rpc.Request on the given facade, version for a given
// object id, and the specific RPC method. It marshalls the Arguments, and will
// unmarshall the result into the response object that is supplied.
func (s *state) APICall(facade string, version int, id, method string, args, response interface{}) error {
	retrySpec := retry.CallArgs{
		Func: func() error {
			return s.client.Call(rpc.Request{
				Type:    facade,
				Version: version,
				Id:      id,
				Action:  method,
			}, args, response)
		},
		IsFatalError: func(err error) bool {
			err = errors.Cause(err)
			ec, ok := err.(hasErrorCode)
			if !ok {
				return true
			}
			return ec.ErrorCode() != params.CodeRetry
		},
		Delay:       100 * time.Millisecond,
		MaxDelay:    1500 * time.Millisecond,
		MaxDuration: 10 * time.Second,
		BackoffFunc: retry.DoubleDelay,
		Clock:       s.clock,
	}
	err := retry.Call(retrySpec)
	return errors.Trace(err)
}

func (*retrySuite) TestMissingFuncNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Attempts: 5,
		Delay:    time.Minute,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Func not valid`)
}

func (*retrySuite) TestMissingDelayNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return errors.New("bah") },
		Attempts: 5,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Delay not valid`)
}

func (*retrySuite) TestMissingAttemptsNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Func:  func() error { return errors.New("bah") },
		Delay: time.Minute,
		Clock: clock.WallClock,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Attempts or MaxDuration not valid`)
}

func (*retrySuite) TestSuccessHasNoDelay(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return nil },
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(clock.delays, gc.HasLen, 0)
}

func (*retrySuite) TestAttemptsExceededError(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return funcErr },
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, gc.ErrorMatches, `attempt count exceeded: bah`)
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(retry.LastError(err), gc.Equals, funcErr)
}

func (*retrySuite) TestFatalErrorsNotRetried(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:         func() error { return funcErr },
		IsFatalError: func(error) bool { return true },
		Attempts:     5,
		Delay:        time.Minute,
		Clock:        clock,
	})
	c.Assert(errors.Cause(err), gc.Equals, funcErr)
	c.Assert(clock.delays, gc.HasLen, 0)
}

func (*retrySuite) TestBackoffErrors(c *gc.C) {
	// Backoff values of less than one are a validation error.
	for _, factor := range []float64{-2, 0.5} {
		err := retry.Call(retry.CallArgs{
			Func:          func() error { return errors.New("bah") },
			Attempts:      5,
			Delay:         time.Minute,
			BackoffFactor: factor,
		})
		c.Check(err, jc.Satisfies, errors.IsNotValid)
		c.Check(err, gc.ErrorMatches, `BackoffFactor of .* not valid`)
	}
}

func (*retrySuite) TestWithWallClock(c *gc.C) {
	var attempts []int
	err := retry.Call(retry.CallArgs{
		Func: func() error { return errors.New("bah") },
		NotifyFunc: func(lastError error, attempt int) {
			attempts = append(attempts, attempt)
		},
		Attempts: 5,
		Delay:    time.Microsecond,
	})
	c.Assert(errors.Cause(err), jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(attempts, jc.DeepEquals, []int{1, 2, 3, 4, 5})
}

// It appears that sometimes the lock is not cleared when we expect it to be.
// Capture and log any errors from the Unlock method and retry a few times.
func (s *store) unlock(lock *fslock.Lock) {
	err := retry.Call(retry.CallArgs{
		Func: lock.Unlock,
		NotifyFunc: func(err error, attempt int) {
			logger.Debugf("failed to unlock jujuclient lock: %s", err)
		},
		Attempts: 10,
		Delay:    50 * time.Millisecond,
		Clock:    clock.WallClock,
	})
	if err != nil {
		logger.Errorf("unable to unlock jujuclient lock: %s", err)
	}
}

func (*retrySuite) TestAttempts(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return funcErr },
		Attempts: 4,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	// We delay between attempts, and don't delay after the last one.
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute,
		time.Minute,
	})
}

func (*retrySuite) TestMaxDuration(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Delay:       time.Minute,
		MaxDuration: 5 * time.Minute,
		Clock:       clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsDurationExceeded)
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute,
		time.Minute,
		time.Minute,
		time.Minute,
	})
}

func (*retrySuite) TestBackoffFactor(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Clock:       clock,
		Attempts:    5,
		Delay:       time.Minute,
		BackoffFunc: retry.DoubleDelay,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute * 2,
		time.Minute * 4,
		time.Minute * 8,
	})
}

func (*retrySuite) TestMaxDurationDoubling(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Delay:       time.Minute,
		MaxDuration: 10 * time.Minute,
		BackoffFunc: retry.DoubleDelay,
		Clock:       clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsDurationExceeded)
	// Stops after seven minutes, because the next wait time
	// would take it to 15 minutes.
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		2 * time.Minute,
		4 * time.Minute,
	})
}

// Dial returns a new connection to the MongoDB server.
func (inst *MgoInstance) Dial() (*mgo.Session, error) {
	var session *mgo.Session
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			var err error
			session, err = mgo.DialWithInfo(inst.DialInfo())
			return err
		},
		// Only interested in retrying the intermittent
		// 'unexpected message'.
		IsFatalError: func(err error) bool {
			return !strings.HasSuffix(err.Error(), "unexpected message")
		},
		Delay:    time.Millisecond,
		Clock:    clock.WallClock,
		Attempts: 5,
	})
	return session, err
}