func (s *URLsSuite) TestToolsMetadataURLsRegisteredFuncs(c *gc.C) {
	tools.RegisterToolsDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames), nil
	})
	tools.RegisterToolsDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames), nil
	})
	// overwrite the one previously registered against id1
	tools.RegisterToolsDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		// NotSupported errors do not cause GetMetadataSources to fail,
		// they just cause the datasource function to be ignored.
		return nil, errors.NewNotSupported(nil, "oyvey")
	})
	defer tools.UnregisterToolsDataSourceFunc("id0")
	defer tools.UnregisterToolsDataSourceFunc("id1")

	env := s.env(c, "config-tools-metadata-url")
	sources, err := tools.GetMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"config-tools-metadata-url/",
		"betwixt/releases/",
		"https://streams.canonical.com/juju/tools/",
	})
}

func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
	environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false), nil
	})
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames, simplestreams.SPECIFIC_CLOUD_DATA, false), nil
	})
	// overwrite the one previously registered against id1
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return nil, errors.NewNotSupported(nil, "oyvey")
	})
	environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false), nil
	})
	defer environs.UnregisterImageDataSourceFunc("id0")
	defer environs.UnregisterImageDataSourceFunc("id1")
	defer environs.UnregisterImageDataSourceFunc("id2")

	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{
		{"config-image-metadata-url/", ""},
		{"foobar/", ""},
		{"betwixt/releases/", ""},
		{"https://streams.canonical.com/juju/images/releases/", imagemetadata.SimplestreamsImagesPublicKey},
		{"http://cloud-images.ubuntu.com/releases/", imagemetadata.SimplestreamsImagesPublicKey},
	})
}

// GetMetadataSources returns the sources to use when looking for
// simplestreams image id metadata for the given stream. If env implements
// SupportsCustomSources, the sources returned from that method will also
// be considered.
func GetMetadataSources(env environs.ConfigGetter) ([]simplestreams.DataSource, error) {
	var sources []simplestreams.DataSource
	config := env.Config()
	if userURL, ok := config.ImageMetadataURL(); ok {
		verify := utils.VerifySSLHostnames
		if !config.SSLHostnameVerification() {
			verify = utils.NoVerifySSLHostnames
		}
		sources = append(sources, simplestreams.NewURLDataSource("image-metadata-url", userURL, verify))
	}
	if custom, ok := env.(SupportsCustomSources); ok {
		customSources, err := custom.GetImageSources()
		if err != nil {
			return nil, err
		}
		sources = append(sources, customSources...)
	}

	defaultURL, err := ImageMetadataURL(DefaultBaseURL, config.ImageStream())
	if err != nil {
		return nil, err
	}
	if defaultURL != "" {
		sources = append(sources,
			simplestreams.NewURLDataSource("default cloud images", defaultURL, utils.VerifySSLHostnames))
	}
	return sources, nil
}

// GetMetadataSourcesWithRetries returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
// The sources are configured to use retries according to the value of allowRetry.
func GetMetadataSourcesWithRetries(env environs.ConfigGetter, allowRetry bool) ([]simplestreams.DataSource, error) {
	var sources []simplestreams.DataSource
	config := env.Config()
	if userURL, ok := config.ToolsURL(); ok {
		verify := utils.VerifySSLHostnames
		if !config.SSLHostnameVerification() {
			verify = utils.NoVerifySSLHostnames
		}
		sources = append(sources, simplestreams.NewURLDataSource("tools-metadata-url", userURL, verify))
	}
	if custom, ok := env.(SupportsCustomSources); ok {
		customSources, err := custom.GetToolsSources()
		if err != nil {
			return nil, err
		}
		sources = append(sources, customSources...)
	}

	defaultURL, err := ToolsURL(DefaultBaseURL)
	if err != nil {
		return nil, err
	}
	if defaultURL != "" {
		sources = append(sources,
			simplestreams.NewURLDataSource("default simplestreams", defaultURL, utils.VerifySSLHostnames))
	}
	for _, source := range sources {
		source.SetAllowRetry(allowRetry)
	}
	return sources, nil
}

func (s *ImageMetadataSuite) TestImageMetadataURLsRegisteredFuncs(c *gc.C) {
	environs.RegisterImageDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames), nil
	})
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames), nil
	})
	// overwrite the one previously registered against id1
	environs.RegisterImageDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
		return nil, errors.NewNotSupported(nil, "oyvey")
	})
	environs.RegisterUserImageDataSourceFunc("id2", func(environs.Environ) (simplestreams.DataSource, error) {
		return simplestreams.NewURLDataSource("id2", "foobar", utils.NoVerifySSLHostnames), nil
	})
	defer environs.UnregisterImageDataSourceFunc("id0")
	defer environs.UnregisterImageDataSourceFunc("id1")
	defer environs.UnregisterImageDataSourceFunc("id2")

	env := s.env(c, "config-image-metadata-url", "")
	sources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	sstesting.AssertExpectedSources(c, sources, []string{
		"config-image-metadata-url/",
		"foobar/",
		"betwixt/releases/",
		"http://cloud-images.ubuntu.com/releases/",
	})
}

// ImageMetadataSources returns the sources to use when looking for
// simplestreams image id metadata for the given stream.
func ImageMetadataSources(env Environ) ([]simplestreams.DataSource, error) {
	config := env.Config()

	// Add configured and environment-specific datasources.
	var sources []simplestreams.DataSource
	if userURL, ok := config.ImageMetadataURL(); ok {
		verify := utils.VerifySSLHostnames
		if !config.SSLHostnameVerification() {
			verify = utils.NoVerifySSLHostnames
		}
		sources = append(sources, simplestreams.NewURLDataSource("image-metadata-url", userURL, verify))
	}

	envDataSources, err := environmentDataSources(env)
	if err != nil {
		return nil, err
	}
	sources = append(sources, envDataSources...)

	// Add the default, public datasource.
	defaultURL, err := imagemetadata.ImageMetadataURL(imagemetadata.DefaultBaseURL, config.ImageStream())
	if err != nil {
		return nil, err
	}
	if defaultURL != "" {
		sources = append(sources,
			simplestreams.NewURLDataSource("default cloud images", defaultURL, utils.VerifySSLHostnames))
	}
	for _, ds := range sources {
		logger.Debugf("using image datasource %q", ds.Description())
	}
	return sources, nil
}

func (s *ValidateSuite) TestMajorVersionMatch(c *gc.C) {
	s.makeLocalMetadata(c, "released", "1.11.2", "raring")
	params := &ToolsMetadataLookupParams{
		Major: 1,
		Minor: -1,
		MetadataLookupParams: simplestreams.MetadataLookupParams{
			Region:        "region-2",
			Series:        "raring",
			Architectures: []string{"amd64"},
			Endpoint:      "some-auth-url",
			Stream:        "released",
			Sources: []simplestreams.DataSource{
				simplestreams.NewURLDataSource("test", s.toolsURL(), utils.VerifySSLHostnames)},
		},
	}
	versions, resolveInfo, err := ValidateToolsMetadata(params)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(versions, gc.DeepEquals, []string{"1.11.2-raring-amd64"})
	c.Check(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
		Source:    "test",
		Signed:    false,
		IndexURL:  utils.MakeFileURL(path.Join(s.metadataDir, "tools/streams/v1/index2.json")),
		MirrorURL: "",
	})
}

func (s *simplestreamsSuite) TestFetchMetadata(c *gc.C) {
	for i, test := range fetchMetadataTests {
		c.Logf("\ntest %d: %s", i, test.about)

		// Patch the current Juju version.
		jujuVersion := version.MustParse(test.jujuVersion)
		s.PatchValue(&jujuversion.Current, jujuVersion)

		// Add invalid datasource and check later that resolveInfo is correct.
		invalidSource := simplestreams.NewURLDataSource(
			"invalid", "file://invalid", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, s.RequireSigned)

		// Fetch the Juju GUI archives.
		allMeta, err := gui.FetchMetadata(test.stream, invalidSource, s.Source)
		for i, meta := range allMeta {
			c.Logf("metadata %d:\n%#v", i, meta)
		}
		if test.expectedError != "" {
			c.Assert(err, gc.ErrorMatches, test.expectedError)
			c.Assert(allMeta, gc.IsNil)
			continue
		}

		// Populate the expected metadata with missing fields.
		for _, meta := range test.expectedMetadata {
			meta.JujuMajorVersion = jujuVersion.Major
			meta.FullPath = "test:/" + meta.Path
			meta.Source = s.Source
			meta.StringVersion = meta.Version.String()
		}
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(allMeta, jc.DeepEquals, test.expectedMetadata)
	}
}

// GetMetadataSources returns the sources to use when looking for
// simplestreams tools metadata for the given stream.
func GetMetadataSources(env environs.Environ) ([]simplestreams.DataSource, error) {
	config := env.Config()

	// Add configured and environment-specific datasources.
	var sources []simplestreams.DataSource
	if userURL, ok := config.AgentMetadataURL(); ok {
		verify := utils.VerifySSLHostnames
		if !config.SSLHostnameVerification() {
			verify = utils.NoVerifySSLHostnames
		}
		sources = append(sources, simplestreams.NewURLDataSource(conf.AgentMetadataURLKey, userURL, verify, simplestreams.SPECIFIC_CLOUD_DATA, false))
	}

	envDataSources, err := environmentDataSources(env)
	if err != nil {
		return nil, err
	}
	sources = append(sources, envDataSources...)

	// Add the default, public datasource.
	defaultURL, err := ToolsURL(DefaultBaseURL)
	if err != nil {
		return nil, err
	}
	if defaultURL != "" {
		sources = append(sources,
			simplestreams.NewURLSignedDataSource("default simplestreams", defaultURL, simplestreams.SimplestreamsJujuPublicKey, utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, true))
	}
	return sources, nil
}

// ImageMetadataSources returns the sources to use when looking for
// simplestreams image id metadata for the given stream.
func ImageMetadataSources(env Environ) ([]simplestreams.DataSource, error) {
	config := env.Config()

	// Add configured and environment-specific datasources.
	var sources []simplestreams.DataSource
	if userURL, ok := config.ImageMetadataURL(); ok {
		verify := utils.VerifySSLHostnames
		if !config.SSLHostnameVerification() {
			verify = utils.NoVerifySSLHostnames
		}
		sources = append(sources, simplestreams.NewURLDataSource("image-metadata-url", userURL, verify, simplestreams.SPECIFIC_CLOUD_DATA, false))
	}

	envDataSources, err := environmentDataSources(env)
	if err != nil {
		return nil, err
	}
	sources = append(sources, envDataSources...)

	// Add the official image metadata datasources.
	officialDataSources, err := imagemetadata.OfficialDataSources(config.ImageStream())
	if err != nil {
		return nil, err
	}
	for _, source := range officialDataSources {
		sources = append(sources, source)
	}
	for _, ds := range sources {
		logger.Debugf("using image datasource %q", ds.Description())
	}
	return sources, nil
}

func (s *simplestreamsSuite) TestGetMetadataNoMatching(c *gc.C) {
	source := &countingSource{
		DataSource: simplestreams.NewURLDataSource(
			"test", "test:/daily", utils.VerifySSLHostnames,
		),
	}
	sources := []simplestreams.DataSource{source, source, source}
	constraint := sstesting.NewTestConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{
			Region:   "us-east-1",
			Endpoint: "https://ec2.us-east-1.amazonaws.com",
		},
		Series: []string{"precise"},
		Arches: []string{"not-a-real-arch"}, // never matches
	})
	params := simplestreams.GetMetadataParams{
		StreamsVersion:   s.StreamsVersion,
		LookupConstraint: constraint,
		ValueParams:      simplestreams.ValueParams{DataType: "image-ids"},
	}

	items, resolveInfo, err := simplestreams.GetMetadata(sources, params)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(items, gc.HasLen, 0)
	c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
		Source:    "test",
		Signed:    false,
		IndexURL:  "test:/daily/streams/v1/index.json",
		MirrorURL: "",
	})

	// There should be 4 calls to each data-source:
	// one for .sjson, one for .json, repeated for legacy vs new index files.
	c.Assert(source.count, gc.Equals, 4*len(sources))
}

// setPrivateMetadataSources sets the default tools metadata source
// for tools syncing, and adds an image metadata source after verifying
// the contents.
func setPrivateMetadataSources(env environs.Environ, metadataDir string) ([]*imagemetadata.ImageMetadata, error) {
	logger.Infof("Setting default tools and image metadata sources: %s", metadataDir)
	tools.DefaultBaseURL = metadataDir

	imageMetadataDir := filepath.Join(metadataDir, storage.BaseImagesPath)
	if _, err := os.Stat(imageMetadataDir); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Annotate(err, "cannot access image metadata")
		}
		return nil, nil
	}

	baseURL := fmt.Sprintf("file://%s", filepath.ToSlash(imageMetadataDir))
	datasource := simplestreams.NewURLDataSource("bootstrap metadata", baseURL, utils.NoVerifySSLHostnames)

	// Read the image metadata, as we'll want to upload it to the environment.
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{})
	existingMetadata, _, err := imagemetadata.Fetch(
		[]simplestreams.DataSource{datasource}, imageConstraint, false)
	if err != nil && !errors.IsNotFound(err) {
		return nil, errors.Annotate(err, "cannot read image metadata")
	}

	// Add an image metadata datasource for constraint validation, etc.
	// TODO (anastasiamac 2015-09-26) Delete when search path is modified to look
	// into state first.
	environs.RegisterUserImageDataSourceFunc("bootstrap metadata", func(environs.Environ) (simplestreams.DataSource, error) {
		return datasource, nil
	})
	logger.Infof("custom image metadata added to search path")
	return existingMetadata, nil
}

func getImageSource(env environs.Environ) (simplestreams.DataSource, error) {
	e, ok := env.(*environ)
	if !ok {
		return nil, errors.NotSupportedf("non-cloudsigma environment")
	}
	return simplestreams.NewURLDataSource("cloud images", fmt.Sprintf(CloudsigmaCloudImagesURLTemplate, e.ecfg.region()), utils.VerifySSLHostnames), nil
}

func (s *simplestreamsSuite) TestFetch(c *gc.C) {
	for i, t := range fetchTests {
		c.Logf("test %d", i)
		cloudSpec := simplestreams.CloudSpec{t.region, "https://ec2.us-east-1.amazonaws.com"}
		if t.region == "" {
			cloudSpec = simplestreams.EmptyCloudSpec
		}
		imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
			CloudSpec: cloudSpec,
			Series:    []string{"precise"},
			Arches:    t.arches,
		})
		// Add invalid datasource and check later that resolveInfo is correct.
		invalidSource := simplestreams.NewURLDataSource("invalid", "file://invalid", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, s.RequireSigned)
		images, resolveInfo, err := imagemetadata.Fetch(
			[]simplestreams.DataSource{invalidSource, s.Source}, imageConstraint)
		if !c.Check(err, jc.ErrorIsNil) {
			continue
		}
		for _, testImage := range t.images {
			testImage.Version = t.version
		}
		c.Check(images, gc.DeepEquals, t.images)
		c.Check(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
			Source:    "test roundtripper",
			Signed:    s.RequireSigned,
			IndexURL:  "test:/streams/v1/index.json",
			MirrorURL: "",
		})
	}
}

func registerLiveSimpleStreamsTests(baseURL string, validImageConstraint simplestreams.LookupConstraint, requireSigned bool) {
	gc.Suite(&sstesting.LocalLiveSimplestreamsSuite{
		Source:          simplestreams.NewURLDataSource("test", baseURL, utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, requireSigned),
		RequireSigned:   requireSigned,
		DataType:        imagemetadata.ImageIds,
		ValidConstraint: validImageConstraint,
	})
}

func registerLiveSimpleStreamsTests(baseURL string, validToolsConstraint simplestreams.LookupConstraint, requireSigned bool) {
	gc.Suite(&sstesting.LocalLiveSimplestreamsSuite{
		Source:          simplestreams.NewURLDataSource("test", baseURL, utils.VerifySSLHostnames),
		RequireSigned:   requireSigned,
		DataType:        tools.ContentDownload,
		ValidConstraint: validToolsConstraint,
	})
}

func (s *signedSuite) TestSignedImageMetadataInvalidSignature(c *gc.C) {
	signedSource := simplestreams.NewURLDataSource("test", "test://host/signed", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, true)
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{"us-east-1", "https://ec2.us-east-1.amazonaws.com"},
		Series:    []string{"precise"},
		Arches:    []string{"amd64"},
	})
	imagemetadata.SetSigningPublicKey(s.origKey)
	_, _, err := imagemetadata.Fetch([]simplestreams.DataSource{signedSource}, imageConstraint)
	c.Assert(err, gc.ErrorMatches, "cannot read index data.*")
}

func (s *datasourceSuite) TestFetch(c *gc.C) {
	ds := simplestreams.NewURLDataSource("test", "test:", utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false)
	rc, url, err := ds.Fetch("streams/v1/tools_metadata.json")
	c.Assert(err, jc.ErrorIsNil)
	defer rc.Close()
	c.Assert(url, gc.Equals, "test:/streams/v1/tools_metadata.json")
	data, err := ioutil.ReadAll(rc)
	cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(len(cloudMetadata.Products), jc.GreaterThan, 0)
}

func (s *datasourceHTTPSSuite) TestNormalClientFails(c *gc.C) {
	ds := simplestreams.NewURLDataSource("test", s.Server.URL, utils.VerifySSLHostnames, simplestreams.DEFAULT_CLOUD_DATA, false)
	url, err := ds.URL("bar")
	c.Assert(err, jc.ErrorIsNil)
	c.Check(url, gc.Equals, s.Server.URL+"/bar")
	reader, _, err := ds.Fetch("bar")
	// The underlying failure is a x509: certificate signed by unknown authority
	// However, the urlDataSource abstraction hides that as a simple NotFound
	c.Assert(err, gc.ErrorMatches, "invalid URL \".*/bar\" not found")
	c.Check(reader, gc.IsNil)
}

// selectSourceDatasource returns a storage reader based on the source setting.
func selectSourceDatasource(syncContext *SyncContext) (simplestreams.DataSource, error) {
	source := syncContext.Source
	if source == "" {
		source = envtools.DefaultBaseURL
	}
	sourceURL, err := envtools.ToolsURL(source)
	if err != nil {
		return nil, err
	}
	logger.Infof("using sync tools source: %v", sourceURL)
	return simplestreams.NewURLDataSource("sync tools source", sourceURL, utils.VerifySSLHostnames), nil
}