func (r CIRepository) Pull() (ImportRec, bool, error) {
	var relSourceRecs []relSourceRec

	err := r.index.ListKeys(&relSourceRecs)
	if err != nil {
		return ImportRec{}, false, bosherr.WrapError(err, "Finding release sources")
	}

	if len(relSourceRecs) == 0 {
		return ImportRec{}, false, nil
	}

	rec := relSourceRecs[0]

	r.logger.Debug(ciRepositoryLogTag, "Pulling import '%v'", rec)

	importRec := ImportRec{
		RelSource: rec.RelSource,
		Version:   rec.Version,
	}

	err = r.index.Remove(relSourceRec{rec.RelSource, rec.Version})
	if err != nil {
		return ImportRec{}, false, bosherr.WrapError(err, "Removing import")
	}

	return importRec, true, nil
}

func (p sfdiskPartitioner) diskMatchesPartitions(devicePath string, partitionsToMatch []Partition) (result bool) {
	existingPartitions, err := p.getPartitions(devicePath)
	if err != nil {
		err = bosherr.WrapError(err, "Getting partitions for %s", devicePath)
		return
	}

	if len(existingPartitions) < len(partitionsToMatch) {
		return
	}

	remainingDiskSpace, err := p.GetDeviceSizeInMb(devicePath)
	if err != nil {
		err = bosherr.WrapError(err, "Getting device size for %s", devicePath)
		return
	}

	for index, partitionToMatch := range partitionsToMatch {
		if index == len(partitionsToMatch)-1 {
			partitionToMatch.SizeInMb = remainingDiskSpace
		}

		existingPartition := existingPartitions[index]
		switch {
		case existingPartition.Type != partitionToMatch.Type:
			return
		case notWithinDelta(existingPartition.SizeInMb, partitionToMatch.SizeInMb, 20):
			return
		}

		remainingDiskSpace = remainingDiskSpace - partitionToMatch.SizeInMb
	}

	return true
}

func (c Config) validate() error {
	if c.AssetsDir == "" {
		return bosherr.New("Must provide non-empty assets_dir")
	}

	if c.ReposDir == "" {
		return bosherr.New("Must provide non-empty repos_dir")
	}

	err := c.EventLog.Validate()
	if err != nil {
		return bosherr.WrapError(err, "Validating event_log configuration")
	}

	if c.Blobstore.Type != bpprov.BlobstoreConfigTypeLocal {
		return bosherr.New("Blobstore type must be local")
	}

	err = c.Blobstore.Validate()
	if err != nil {
		return bosherr.WrapError(err, "Validating blobstore configuration")
	}

	return nil
}

func (p ReleaseCompiler) compileRelease(pkgsCompiler bppkgscomp.PackagesCompiler, depRelease bpdep.Release) error {
	relReader := p.releaseReaderFactory.NewReader(
		depRelease.Name,
		depRelease.Version,
		depRelease.URL,
	)

	relRelease, err := relReader.Read()
	if err != nil {
		return bosherr.WrapError(err, "Reading release")
	}

	defer relReader.Close()

	err = pkgsCompiler.Compile(relRelease)
	if err != nil {
		return bosherr.WrapError(err, "Compiling release packages")
	}

	err = p.templatesCompiler.Precompile(relRelease)
	if err != nil {
		return bosherr.WrapError(err, "Precompiling release job templates")
	}

	return nil
}

func (i PeriodicGithubNoteImporter) importNotes() error {
	// todo getting all releases; could be big
	sources, err := i.releasesRepo.ListAll()
	if err != nil {
		return bosherr.WrapError(err, "Listing releases")
	}

	for _, source := range sources {
		ghSource, valid := newGithubSource(source)
		if !valid {
			continue
		}

		// todo convert Source to string; argh
		relVerRecs, err := i.releasesRepo.FindAll(source.Full)
		if err != nil {
			return bosherr.WrapError(err, "Listing all versions for release source '%s'", source)
		}

		err = i.importNotesForRelease(ghSource, relVerRecs)
		if err != nil {
			return bosherr.WrapError(err, "Importing notes for release source '%s'", source)
		}
	}

	return nil
}

func (p BlobstoreProvider) Get(provider string, options map[string]interface{}) (boshblob.Blobstore, error) {
	configDir, err := p.fs.TempDir("blobstore-s3-config")
	if err != nil {
		return nil, bosherr.WrapError(err, "Cerating tmp dir for blobstore config")
	}

	configPath := filepath.Join(configDir, "config.json")

	blobstore := boshblob.NewExternalBlobstore(
		provider,
		options,
		p.fs,
		p.runner,
		p.uuidGen,
		configPath,
	)

	blobstore = boshblob.NewSHA1VerifiableBlobstore(blobstore)

	blobstore = boshblob.NewRetryableBlobstore(blobstore, 3, p.logger)

	err = blobstore.Validate()
	if err != nil {
		return nil, bosherr.WrapError(err, "Validating blobstore")
	}

	return blobstore, nil
}

func (p *PlainBucketPage) fetchOnce() (plainBucketPage_ListBucketResult, error) {
	// Return memoized results; not thread safe
	if p.fetchedResult != nil || p.fetchedErr != nil {
		return *p.fetchedResult, p.fetchedErr
	}

	fullURL := p.fullURL()

	p.logger.Debug(p.logTag, "Fetching bucket page from url '%s'", fullURL)

	var result plainBucketPage_ListBucketResult

	resp, err := http.Get(fullURL)
	if err != nil {
		p.fetchedErr = err
		return result, bosherr.WrapError(err, "Requesting bucket page")
	}

	defer resp.Body.Close()

	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader

	err = decoder.Decode(&result)
	if err != nil {
		p.fetchedErr = err
		return result, bosherr.WrapError(err, "Parsing bucket page")
	}

	p.logger.Debug(p.logTag, "Saving bucket page result '%v'", result)

	p.fetchedResult = &result

	return result, nil
}

func (r S3StemcellsRepository) SaveAll(s3Files []bhs3.File) error {
	var s3StemcellRecs []s3StemcellRec

	for _, s3File := range s3Files {
		url, err := s3File.URL()
		if err != nil {
			return bosherr.WrapError(err, "Generating S3 stemcell URL")
		}

		rec := s3StemcellRec{
			Key:  s3File.Key(),
			ETag: s3File.ETag(),

			Size:         s3File.Size(),
			LastModified: s3File.LastModified(),

			URL: url,
		}

		s3StemcellRecs = append(s3StemcellRecs, rec)
	}

	err := r.index.Save(s3StemcellsKeyLatest, s3StemcellRecs)
	if err != nil {
		return bosherr.WrapError(err, "Saving all S3 stemcell recs")
	}

	return nil
}

func (a MountDiskAction) Run(diskCid string) (interface{}, error) {
	err := a.settingsService.LoadSettings()
	if err != nil {
		return nil, bosherr.WrapError(err, "Refreshing the settings")
	}

	settings := a.settingsService.GetSettings()

	devicePath, found := settings.Disks.Persistent[diskCid]
	if !found {
		return nil, bosherr.New("Persistent disk with volume id '%s' could not be found", diskCid)
	}

	mountPoint := a.dirProvider.StoreDir()

	isMountPoint, err := a.mountPoints.IsMountPoint(mountPoint)
	if err != nil {
		return nil, bosherr.WrapError(err, "Checking mount point")
	}
	if isMountPoint {
		mountPoint = a.dirProvider.StoreMigrationDir()
	}

	err = a.diskMounter.MountPersistentDisk(devicePath, mountPoint)
	if err != nil {
		return nil, bosherr.WrapError(err, "Mounting persistent disk")
	}

	return map[string]string{}, nil
}

func (e Extractor) extractReleaseAndJobs(tgzPath string) (bprel.Release, []bpreljob.Job, error) {
	var rel bprel.Release

	relReader := e.releaseReaderFactory.NewTarReader("file://" + tgzPath)

	rel, err := relReader.Read()
	if err != nil {
		return rel, nil, bosherr.WrapError(err, "Reading release")
	}

	defer relReader.Close()

	var relJobs []bpreljob.Job

	for _, j := range rel.Jobs {
		relJobReader := e.jobReaderFactory.NewTarReader("file://" + j.TarPath)

		relJob, err := relJobReader.Read()
		if err != nil {
			return rel, nil, bosherr.WrapError(err, "Reading release job '%s'", j.Name)
		}

		defer relJobReader.Close()

		relJobs = append(relJobs, relJob)
	}

	return rel, relJobs, nil
}

func (s JobState) buildPackageSpecs() (map[string]boshas.PackageSpec, error) {
	specs := map[string]boshas.PackageSpec{}

	for _, template := range s.depJob.Templates {
		pkgs, err := s.templatesCompiler.FindPackages(template)
		if err != nil {
			return specs, bosherr.WrapError(err, "Finding packages for template %s", template.Name)
		}

		for _, pkg := range pkgs {
			rec, err := s.packagesCompiler.FindCompiledPackage(pkg)
			if err != nil {
				return specs, bosherr.WrapError(err, "Finding compiled package %s", pkg.Name)
			}

			specs[pkg.Name] = boshas.PackageSpec{
				Name:    pkg.Name,
				Version: pkg.Version,

				Sha1:        rec.SHA1,
				BlobstoreID: rec.BlobID,
			}
		}
	}

	return specs, nil
}

func (e Extractor) Extract(url, tgzPath string) (bhrelsrepo.ReleaseVersionRec, error) {
	e.logger.Debug(e.logTag, "Extracting from '%s' (url '%s')", tgzPath, url)

	var relVerRec bhrelsrepo.ReleaseVersionRec

	rel, relJobs, err := e.extractReleaseAndJobs(tgzPath)
	if err != nil {
		return relVerRec, bosherr.WrapError(err, "Extracting release and jobs")
	}

	relVerRec, err = e.releasesRepo.Find(url, rel.Version)
	if err != nil {
		return relVerRec, bosherr.WrapError(err, "Finding release")
	}

	err = e.jobsRepo.SaveAll(relVerRec, relJobs)
	if err != nil {
		return relVerRec, bosherr.WrapError(err, "Saving release jobs into jobs repository")
	}

	err = e.releaseVersionsRepo.Save(relVerRec, rel)
	if err != nil {
		return relVerRec, bosherr.WrapError(err, "Saving release into releases repository")
	}

	return relVerRec, nil
}

func (p RunitProvisioner) Provision(name string, stopTimeout time.Duration) error {
	p.logger.Info(runitProvisionerLogTag, "Provisioning %s service", name)

	err := p.depsProvisioner.InstallRunit()
	if err != nil {
		return bosherr.WrapError(err, "Installing runit")
	}

	servicePath, enableServicePath := p.buildServicePaths(name)

	err = p.stopRunAndLog(servicePath, enableServicePath, name, stopTimeout)
	if err != nil {
		return bosherr.WrapError(err, "Stopping run and log")
	}

	err = p.setUpRun(servicePath, name)
	if err != nil {
		return bosherr.WrapError(err, "Setting up run")
	}

	err = p.setUpLog(servicePath, name)
	if err != nil {
		return bosherr.WrapError(err, "Setting up log")
	}

	err = p.startRunAndLog(servicePath, enableServicePath, name)
	if err != nil {
		return bosherr.WrapError(err, "Starting run and log")
	}

	return nil
}

func (tc ConcreteTemplatesCompiler) buildJobReaders(job bpdep.Job) ([]jobReader, error) {
	var readers []jobReader

	for _, template := range job.Templates {
		rec, found, err := tc.tplToJobRepo.FindByTemplate(template)
		if err != nil {
			return readers, bosherr.WrapError(err, "Finding dep-template -> release-job record %s", template.Name)
		} else if !found {
			return readers, bosherr.New("Expected to find dep-template -> release-job record %s", template.Name)
		}

		jobRec, found, err := tc.jobsRepo.FindByReleaseJob(rec)
		if err != nil {
			return readers, bosherr.WrapError(err, "Finding job source blob %s", template.Name)
		} else if !found {
			return readers, bosherr.New("Expected to find job source blob %s -- %s", template.Name, rec)
		}

		jobURL := fmt.Sprintf("blobstore:///%s?fingerprint=%s", jobRec.BlobID, jobRec.SHA1)

		reader := jobReader{
			rec:       rec,
			tarReader: tc.jobReaderFactory.NewReader(jobURL),
		}

		readers = append(readers, reader)
	}

	return readers, nil
}

func (tr TarballRelease) Import(url string) error {
	tr.logger.Debug(tr.logTag, "Importing tarball release from '%s'", url)

	tgzPath, err := tr.builder.Build(tr.manifestPath)
	if err != nil {
		return bosherr.WrapError(err, "Building release tarball from manifest '%s'", tr.manifestPath)
	}

	defer tr.builder.CleanUp(tgzPath)

	relVerRec, err := tr.extractor.Extract(url, tgzPath)
	if err != nil {
		return bosherr.WrapError(err, "Importing release version from tgz '%s'", tgzPath)
	}

	err = tr.uploader.Upload(relVerRec, tgzPath)
	if err != nil {
		return bosherr.WrapError(err, "Uploading release verison '%v'", relVerRec)
	}

	// Save release version only after everything else was successfully imported
	err = tr.releasesRepo.Add(relVerRec)
	if err != nil {
		return bosherr.WrapError(err, "Saving release version into release versions repository")
	}

	return nil
}

func (u Updater) SetUp() error {
	stage := u.eventLog.BeginStage(fmt.Sprintf("Setting up instance %s", u.instanceDesc), 3)

	task := stage.BeginTask("Applying")

	err := task.End(u.applier.Apply())
	if err != nil {
		return bosherr.WrapError(err, "Applying")
	}

	task = stage.BeginTask("Starting")

	err = task.End(u.starter.Start())
	if err != nil {
		return bosherr.WrapError(err, "Starting")
	}

	task = stage.BeginTask("Waiting")

	err = task.End(u.waiter.Wait())
	if err != nil {
		return bosherr.WrapError(err, "Waiting")
	}

	return nil
}

func (c httpClient) status() (status, error) {
	url := c.monitURL("/_status2")
	url.RawQuery = "format=xml"

	response, err := c.makeRequest(url, "GET", "")
	if err != nil {
		return status{}, bosherr.WrapError(err, "Sending status request to monit")
	}

	defer response.Body.Close()

	err = c.validateResponse(response)
	if err != nil {
		return status{}, bosherr.WrapError(err, "Getting monit status")
	}

	decoder := xml.NewDecoder(response.Body)
	decoder.CharsetReader = charset.NewReader

	var st status

	err = decoder.Decode(&st)
	if err != nil {
		return status{}, bosherr.WrapError(err, "Unmarshalling Monit status")
	}

	return st, nil
}

func (v SemanticValidator) Validate() error {
	for _, net := range v.deployment.Networks {
		err := v.validateNetwork(net)
		if err != nil {
			return bosherr.WrapError(err, "Network %s", net.Name)
		}
	}

	for _, release := range v.deployment.Releases {
		err := v.validateRelease(release)
		if err != nil {
			return bosherr.WrapError(err, "Release %s", release.Name)
		}
	}

	err := v.validateInstance(v.deployment.CompilationInstance)
	if err != nil {
		return bosherr.WrapError(err, "Compilation instance")
	}

	for _, job := range v.deployment.Jobs {
		err := v.validateJob(job)
		if err != nil {
			return bosherr.WrapError(err, "Job %s", job.Name)
		}
	}

	return nil
}

func (p MonitProvisioner) Provision() error {
	p.logger.Info(monitProvisionerLogTag, "Provisioning monit")

	path := "/var/vcap/monit"

	err := p.cmds.MkdirP(path)
	if err != nil {
		return err
	}

	err = p.configureMonitrc()
	if err != nil {
		return bosherr.WrapError(err, "Configuring monitrc")
	}

	err = p.runitProvisioner.Provision(
		monitProvisionerRunitName,
		monitProvisionerRunitStopTime,
	)
	if err != nil {
		return bosherr.WrapError(err, "Provisioning monit with runit")
	}

	return nil
}

func (r ERBRenderer) Render(srcPath, dstPath string) error {
	r.logger.Debug(erbRendererLogTag, "Rendering template %s", dstPath)

	dirPath := filepath.Dir(dstPath)

	err := r.fs.MkdirAll(dirPath, os.FileMode(0755))
	if err != nil {
		return bosherr.WrapError(err, "Creating directory %s", dirPath)
	}

	rendererScriptPath, err := r.writeRendererScript()
	if err != nil {
		return err
	}

	contextPath, err := r.writeContext()
	if err != nil {
		return err
	}

	// Use ruby to compile job templates
	command := boshsys.Command{
		Name: r.determineRubyExePath(),
		Args: []string{rendererScriptPath, contextPath, srcPath, dstPath},
	}

	_, _, _, err = r.runner.RunComplexCommand(command)
	if err != nil {
		return bosherr.WrapError(err, "Running ruby")
	}

	return nil
}