func (s *ImageSuite) TestAddImageExcessiveContention(c *gc.C) {
	metadata := []*imagestorage.Metadata{
		{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path"},
		{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path"},
		{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "2", SourceURL: "http://path"},
		{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "3", SourceURL: "http://path"},
	}

	i := 1
	addMetadata := func() {
		err := s.storage.AddImage(strings.NewReader(metadata[i].SHA256), metadata[i])
		c.Assert(err, gc.IsNil)
		i++
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata, addMetadata, addMetadata).Check()

	err := s.storage.AddImage(strings.NewReader(metadata[0].SHA256), metadata[0])
	c.Assert(err, gc.ErrorMatches, "cannot store image metadata: state changing too quickly; try again soon")

	// There should be no blobs apart from the last one added by the before-hook.
	for _, metadata := range metadata[:3] {
		path := fmt.Sprintf("images/%s-%s-%s:%s", metadata.Kind, metadata.Series, metadata.Arch, metadata.SHA256)
		managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
		_, _, err = managedStorage.GetForBucket("my-uuid", path)
		c.Assert(err, jc.Satisfies, errors.IsNotFound)
	}

	s.assertImage(c, metadata[3], "3")
}

func (s *ClientSimpleRaceSuite) TestNewClient_WorksDespite_CreateClockRace(c *gc.C) {
	config := func(id string) lease.ClientConfig {
		return lease.ClientConfig{
			Id:         id,
			Namespace:  "ns",
			Collection: "leases",
			Mongo:      NewMongo(s.db),
			Clock:      lease.SystemClock{},
		}
	}
	sutConfig := config("sut")
	sutRunner := sutConfig.Mongo.(*Mongo).runner

	// Set up a hook to create the clock doc (and write some important data to
	// it)  by creating another client before the SUT gets a chance.
	defer txntesting.SetBeforeHooks(c, sutRunner, func() {
		client, err := lease.NewClient(config("blocker"))
		c.Check(err, jc.ErrorIsNil)
		err = client.ClaimLease("somewhere", lease.Request{"someone", time.Minute})
		c.Check(err, jc.ErrorIsNil)
	})()

	// Create a client against an apparently-empty namespace.
	client, err := lease.NewClient(sutConfig)
	c.Check(err, jc.ErrorIsNil)

	// Despite the scramble, it's generated with recent lease data and no error.
	leases := client.Leases()
	info, found := leases["somewhere"]
	c.Check(found, jc.IsTrue)
	c.Check(info.Holder, gc.Equals, "someone")
}

func (s *ToolsSuite) TestAddToolsExcessiveContention(c *gc.C) {
	metadata := []toolstorage.Metadata{
		{Version: version.Current, Size: 1, SHA256: "0"},
		{Version: version.Current, Size: 1, SHA256: "1"},
		{Version: version.Current, Size: 1, SHA256: "2"},
		{Version: version.Current, Size: 1, SHA256: "3"},
	}

	i := 1
	addMetadata := func() {
		err := s.storage.AddTools(strings.NewReader(metadata[i].SHA256), metadata[i])
		c.Assert(err, jc.ErrorIsNil)
		i++
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata, addMetadata, addMetadata).Check()

	err := s.storage.AddTools(strings.NewReader(metadata[0].SHA256), metadata[0])
	c.Assert(err, gc.ErrorMatches, "cannot store tools metadata: state changing too quickly; try again soon")

	// There should be no blobs apart from the last one added by the before-hook.
	for _, metadata := range metadata[:3] {
		path := fmt.Sprintf("tools/%s-%s", metadata.Version, metadata.SHA256)
		_, _, err = s.managedStorage.GetForEnvironment("my-uuid", path)
		c.Assert(err, jc.Satisfies, errors.IsNotFound)
	}

	s.assertTools(c, metadata[3], "3")
}

func (s *ImageSuite) TestAddImageConcurrent(c *gc.C) {
	metadata0 := &imagestorage.Metadata{
		ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path",
	}
	metadata1 := &imagestorage.Metadata{
		ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path",
	}

	addMetadata := func() {
		err := s.storage.AddImage(strings.NewReader("0"), metadata0)
		c.Assert(err, gc.IsNil)
		managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
		r, _, err := managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
		c.Assert(err, gc.IsNil)
		r.Close()
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata).Check()

	err := s.storage.AddImage(strings.NewReader("1"), metadata1)
	c.Assert(err, gc.IsNil)

	// Blob added in before-hook should be removed.
	managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
	_, _, err = managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)

	s.assertImage(c, metadata1, "1")
}

func (s *txnSuite) TestBeforeHooks(c *gc.C) {
	s.insertDoc(c, "1", "Simple")
	changeFuncs := []func(){
		func() { s.setDocName(c, "1", "FooBar") },
		func() { s.setDocName(c, "1", "Foo") },
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, changeFuncs...).Check()
	maxAttempt := 0
	buildTxn := func(attempt int) ([]txn.Op, error) {
		maxAttempt = attempt
		ops := []txn.Op{{
			C:      s.collection.Name,
			Id:     "1",
			Assert: bson.D{{"name", "Foo"}},
			Update: bson.D{{"$set", bson.D{{"name", "Bar"}}}},
		}}
		return ops, nil
	}
	err := s.txnRunner.Run(buildTxn)
	c.Assert(err, gc.IsNil)
	var found simpleDoc
	err = s.collection.FindId("1").One(&found)
	c.Assert(err, gc.IsNil)
	c.Assert(maxAttempt, gc.Equals, 1)
	doc := simpleDoc{"1", "Bar"}
	c.Assert(found, gc.DeepEquals, doc)
}

func (s *resourceCatalogSuite) TestPutDeletedResourceRace(c *gc.C) {
	firstId, _ := s.assertPut(c, true, "md5foo", "sha256foo")
	err := s.rCatalog.UploadComplete(firstId)
	c.Assert(err, gc.IsNil)
	beforeFuncs := []func(){
		func() {
			_, _, err := s.rCatalog.Remove(firstId)
			c.Assert(err, gc.IsNil)
		},
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
	rh := &blobstore.ResourceHash{"md5foo", "sha256foo"}
	id, _, isNew, err := s.rCatalog.Put(rh, 200)
	c.Assert(err, gc.IsNil)
	c.Assert(isNew, jc.IsTrue)
	c.Assert(firstId, gc.Equals, id)
	err = s.rCatalog.UploadComplete(id)
	c.Assert(err, gc.IsNil)
	r, err := s.rCatalog.Get(id)
	c.Assert(err, gc.IsNil)
	s.assertRefCount(c, id, 1)
	c.Assert(r.MD5Hash, gc.Equals, "md5foo")
	c.Assert(r.SHA256Hash, gc.Equals, "sha256foo")
	c.Assert(r.Length, gc.Equals, int64(200))
}

func (s *cloudImageMetadataSuite) assertConcurrentDelete(c *gc.C, imageId0, imageId1 string) {
	deleteMetadata := func() {
		s.assertDeleteMetadata(c, imageId0)
	}
	defer txntesting.SetBeforeHooks(c, s.access.runner, deleteMetadata).Check()
	s.assertDeleteMetadata(c, imageId1)
	s.assertNoMetadata(c)
}

func (s *cloudImageMetadataSuite) assertConcurrentSave(c *gc.C, metadata0, metadata1 cloudimagemetadata.Metadata, expected ...cloudimagemetadata.Metadata) {
	addMetadata := func() {
		s.assertRecordMetadata(c, metadata0)
	}
	defer txntesting.SetBeforeHooks(c, s.access.runner, addMetadata).Check()
	s.assertRecordMetadata(c, metadata1)
	s.assertMetadataRecorded(c, cloudimagemetadata.MetadataAttributes{}, expected...)
}

func (s *managedStorageSuite) TestPutRace(c *gc.C) {
	blob := []byte("some resource")
	beforeFunc := func() {
		s.assertPut(c, "/path/to/blob", blob)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	anotherblob := []byte("another resource")
	s.assertPut(c, "/path/to/blob", anotherblob)
	s.assertResourceCatalogCount(c, 1)
}

func (s *resourceCatalogSuite) TestUploadCompleteDeleted(c *gc.C) {
	id, _, err := s.rCatalog.Put("sha384foo", 100)
	c.Assert(err, gc.IsNil)
	remove := func() {
		_, _, err := s.rCatalog.Remove(id)
		c.Assert(err, gc.IsNil)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, remove).Check()
	err = s.rCatalog.UploadComplete(id, "wherever")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *managedStorageSuite) TestPutDeleteRace(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	beforeFunc := func() {
		err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
		c.Assert(err, gc.IsNil)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	anotherblob := []byte("another resource")
	s.assertPut(c, "/path/to/blob", anotherblob)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestRemoveRace(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	beforeFunc := func() {
		err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
		c.Assert(err, gc.IsNil)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
	_, _, err = s.managedStorage.GetForEnvironment("env", "/path/to/blob")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *resourceCatalogSuite) TestDeleteResourceRace(c *gc.C) {
	id, _ := s.assertPut(c, true, "md5foo", "sha256foo")
	s.assertPut(c, false, "md5foo", "sha256foo")
	beforeFuncs := []func(){
		func() {
			_, _, err := s.rCatalog.Remove(id)
			c.Assert(err, gc.IsNil)
		},
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
	_, _, err := s.rCatalog.Remove(id)
	c.Assert(err, gc.IsNil)
	_, err = s.rCatalog.Get(id)
	c.Assert(err, gc.ErrorMatches, `resource with id ".*" not found`)
}

func (s *ClientTrickyRaceSuite) TestExtendLease_BlockedBy_ExpireLease(c *gc.C) {

	// Set up a hook to expire the lease before the extend gets a chance.
	defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
		s.blocker.Clock.Advance(90 * time.Second)
		err := s.blocker.Client.ExpireLease("name")
		c.Check(err, jc.ErrorIsNil)
	})()

	// Try to extend; check it aborts.
	err := s.sut.Client.ExtendLease("name", lease.Request{"holder", 2 * time.Minute})
	c.Check(err, gc.Equals, lease.ErrInvalid)

	// The SUT has been refreshed, and you can see why the operation was invalid.
	c.Check("name", s.sut.Holder(), "")
}

func (s *resourceCatalogSuite) TestPutNewResourceRace(c *gc.C) {
	var firstId string
	beforeFuncs := []func(){
		func() { firstId, _ = s.assertPut(c, true, "sha384foo") },
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
	id, _, err := s.rCatalog.Put("sha384foo", 200)
	c.Assert(err, gc.IsNil)
	c.Assert(id, gc.Equals, firstId)
	err = s.rCatalog.UploadComplete(id, "wherever")
	c.Assert(err, gc.IsNil)
	r, err := s.rCatalog.Get(id)
	c.Assert(err, gc.IsNil)
	s.assertRefCount(c, id, 2)
	c.Assert(r.SHA384Hash, gc.Equals, "sha384foo")
	c.Assert(int(r.Length), gc.Equals, 200)
}

func (s *ClientTrickyRaceSuite) TestExtendLease_WorksDespite_LongerExtendLease(c *gc.C) {

	shorterRequest := 90 * time.Second
	longerRequest := 120 * time.Second

	// Set up hooks to extend the lease by a lot, before the SUT's extend can.
	defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
		err := s.blocker.Client.ExtendLease("name", lease.Request{"holder", longerRequest})
		c.Check(err, jc.ErrorIsNil)
	})()

	// Extend the lease by a little.
	err := s.sut.Client.ExtendLease("name", lease.Request{"holder", shorterRequest})
	c.Check(err, jc.ErrorIsNil)

	// The SUT was refreshed, and knows that the lease is really valid for longer.
	c.Check("name", s.sut.Expiry(), s.sut.Zero.Add(longerRequest))
}

func (s *ClientSimpleRaceSuite) TestClaimLease_BlockedBy_ClaimLease(c *gc.C) {
	sut := s.EasyFixture(c)
	blocker := s.NewFixture(c, FixtureParams{Id: "blocker"})

	// Set up a hook to grab the lease "name" just before the next txn runs.
	defer txntesting.SetBeforeHooks(c, sut.Runner, func() {
		err := blocker.Client.ClaimLease("name", lease.Request{"ha-haa", time.Minute})
		c.Check(err, jc.ErrorIsNil)
	})()

	// Try to grab the lease "name", and fail.
	err := sut.Client.ClaimLease("name", lease.Request{"trying", time.Second})
	c.Check(err, gc.Equals, lease.ErrInvalid)

	// The client that failed has refreshed state (as it had to, in order
	// to discover the reason for the invalidity).
	c.Check("name", sut.Holder(), "ha-haa")
	c.Check("name", sut.Expiry(), sut.Zero.Add(time.Minute))
}

func (s *ClientTrickyRaceSuite) TestExpireLease_BlockedBy_ExpireThenReclaim(c *gc.C) {

	// Set up a hook to expire the lease and then reclaim it.
	defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
		s.blocker.Clock.Advance(90 * time.Second)
		err := s.blocker.Client.ExpireLease("name")
		c.Check(err, jc.ErrorIsNil)
		err = s.blocker.Client.ClaimLease("name", lease.Request{"holder", time.Minute})
		c.Check(err, jc.ErrorIsNil)
	})()

	// Try to expire; check it aborts.
	s.sut.Clock.Advance(90 * time.Second)
	err := s.sut.Client.ExpireLease("name")
	c.Check(err, gc.Equals, lease.ErrInvalid)

	// The SUT has been refreshed, and you can see why the operation was invalid.
	c.Check("name", s.sut.Expiry(), s.sut.Zero.Add(150*time.Second))
}

func (s *resourceCatalogSuite) TestPutNewResourceRace(c *gc.C) {
	var firstId string
	beforeFuncs := []func(){
		func() { firstId, _ = s.assertPut(c, true, "md5foo", "sha256foo") },
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
	rh := &blobstore.ResourceHash{"md5foo", "sha256foo"}
	id, _, isNew, err := s.rCatalog.Put(rh, 200)
	c.Assert(err, gc.IsNil)
	c.Assert(id, gc.Equals, firstId)
	c.Assert(isNew, jc.IsFalse)
	err = s.rCatalog.UploadComplete(id)
	c.Assert(err, gc.IsNil)
	r, err := s.rCatalog.Get(id)
	c.Assert(err, gc.IsNil)
	s.assertRefCount(c, id, 2)
	c.Assert(r.MD5Hash, gc.Equals, "md5foo")
	c.Assert(r.SHA256Hash, gc.Equals, "sha256foo")
	c.Assert(int(r.Length), gc.Equals, 200)
}

func (s *managedStorageSuite) TestPutRaceWhereCatalogEntryRemoved(c *gc.C) {
	blob := []byte("some resource")
	// Remove the resource catalog entry with the resourceId that we are about
	// to write to a managed resource entry.
	beforeFunc := []func(){
		nil, //  resourceCatalog Put()
		nil, //  managedResource Put()
		func() {
			// Shamelessly exploit our knowledge of how ids are made.
			sha384Hash := calculateCheckSum(c, 0, int64(len(blob)), blob)
			_, _, err := blobstore.GetResourceCatalog(s.managedStorage).Remove(sha384Hash)
			c.Assert(err, gc.IsNil)
		},
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc...).Check()
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", "/path/to/blob", rdr, int64(len(blob)))
	c.Assert(err, gc.ErrorMatches, "unexpected deletion .*")
	s.assertResourceCatalogCount(c, 0)
}