func getReferencedImgs(s *store.Store) (map[string]struct{}, error) {
	imgs := map[string]struct{}{}
	walkErrors := []error{}
	// Consider pods in preparing, prepared, run, exitedgarbage state
	if err := walkPods(includeMostDirs, func(p *pod) {
		appImgs, err := p.getAppsHashes()
		if err != nil {
			// Ignore errors reading/parsing pod file
			return
		}
		for _, appImg := range appImgs {
			key, err := s.ResolveKey(appImg.String())
			if err != nil && err != store.ErrKeyNotFound {
				walkErrors = append(walkErrors, fmt.Errorf("bad imageID %q in pod definition: %v", appImg.String(), err))
				return
			}
			imgs[key] = struct{}{}
		}
	}); err != nil {
		return nil, fmt.Errorf("failed to get pod handles: %v", err)
	}
	if len(walkErrors) > 0 {
		return nil, fmt.Errorf("errors occured walking pods. errors: %v", walkErrors)

	}
	return imgs, nil
}

// getTmpROC returns a removeOnClose instance wrapping a temporary
// file provided by the passed store. The actual file name is based on
// a hash of the passed path.
func getTmpROC(s *store.Store, path string) (*removeOnClose, error) {
	h := sha512.New()
	h.Write([]byte(path))
	pathHash := s.HashToKey(h)

	tmp, err := s.TmpNamedFile(pathHash)
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error setting up temporary file"), err)
	}
	// let's lock the file to avoid concurrent writes to the temporary file, it
	// will go away when removing the temp file
	_, err = lock.TryExclusiveLock(tmp.Name(), lock.RegFile)
	if err != nil {
		if err != lock.ErrLocked {
			return nil, errwrap.Wrap(errors.New("failed to lock temporary file"), err)
		}
		log.Printf("another rkt instance is downloading this file, waiting...")
		_, err = lock.ExclusiveLock(tmp.Name(), lock.RegFile)
		if err != nil {
			return nil, errwrap.Wrap(errors.New("failed to lock temporary file"), err)
		}
	}
	roc := &removeOnClose{File: tmp}
	return roc, nil
}

// gcTreeStore removes all treeStoreIDs not referenced by any non garbage
// collected pod from the store.
func gcTreeStore(s *store.Store) error {
	// Take an exclusive lock to block other pods being created.
	// This is needed to avoid races between the below steps (getting the
	// list of referenced treeStoreIDs, getting the list of treeStoreIDs
	// from the store, removal of unreferenced treeStoreIDs) and new
	// pods/treeStores being created/referenced
	keyLock, err := lock.ExclusiveKeyLock(lockDir(), common.PrepareLock)
	if err != nil {
		return fmt.Errorf("cannot get exclusive prepare lock: %v", err)
	}
	defer keyLock.Close()
	referencedTreeStoreIDs, err := getReferencedTreeStoreIDs()
	if err != nil {
		return fmt.Errorf("cannot get referenced treestoreIDs: %v", err)
	}
	treeStoreIDs, err := s.GetTreeStoreIDs()
	if err != nil {
		return fmt.Errorf("cannot get treestoreIDs from the store: %v", err)
	}
	for _, treeStoreID := range treeStoreIDs {
		if _, ok := referencedTreeStoreIDs[treeStoreID]; !ok {
			if err := s.RemoveTreeStore(treeStoreID); err != nil {
				stderr("rkt: error removing treestore %q: %v", treeStoreID, err)
			} else {
				stderr("rkt: removed treestore %q", treeStoreID)
			}
		}
	}
	return nil
}

// getTmpROC returns a removeOnClose instance wrapping a temporary
// file provided by the passed store. The actual file name is based on
// a hash of the passed path.
func getTmpROC(s *store.Store, path string) (*removeOnClose, error) {
	h := sha512.New()
	h.Write([]byte(path))
	pathHash := s.HashToKey(h)

	tmp, err := s.TmpNamedFile(pathHash)
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error setting up temporary file"), err)
	}
	roc := &removeOnClose{File: tmp}
	return roc, nil
}

func rmImages(s *store.Store, images []string) error {
	done := 0
	errors := 0
	staleErrors := 0
	for _, pkey := range images {
		errors++
		h, err := types.NewHash(pkey)
		if err != nil {
			stderr("rkt: wrong image ID %q: %v", pkey, err)
			continue
		}
		key, err := s.ResolveKey(h.String())
		if err != nil {
			stderr("rkt: image ID %q not valid: %v", pkey, err)
			continue
		}
		if key == "" {
			stderr("rkt: image ID %q doesn't exist", pkey)
			continue
		}

		if err = s.RemoveACI(key); err != nil {
			if serr, ok := err.(*store.StoreRemovalError); ok {
				staleErrors++
				stderr("rkt: some files cannot be removed for image ID %q: %v", pkey, serr)
			} else {
				stderr("rkt: error removing aci for image ID %q: %v", pkey, err)
				continue
			}
		}
		stdout("rkt: successfully removed aci for image ID: %q", pkey)
		errors--
		done++
	}

	if done > 0 {
		stderr("rkt: %d image(s) successfully removed", done)
	}

	// If anything didn't complete, return exit status of 1
	if (errors + staleErrors) > 0 {
		if staleErrors > 0 {
			stderr("rkt: %d image(s) removed but left some stale files", staleErrors)
		}
		if errors > 0 {
			stderr("rkt: %d image(s) cannot be removed", errors)
		}
		return fmt.Errorf("error(s) found while removing images")
	}

	return nil
}

func getStoreKeyFromApp(s *store.Store, img string) (string, error) {
	app, err := discovery.NewAppFromString(img)
	if err != nil {
		return "", fmt.Errorf("cannot parse the image name: %v", err)
	}
	labels, err := types.LabelsFromMap(app.Labels)
	if err != nil {
		return "", fmt.Errorf("invalid labels in the name: %v", err)
	}
	key, err := s.GetACI(app.Name, labels)
	if err != nil {
		return "", fmt.Errorf("cannot find image: %v", err)
	}
	return key, nil
}

func getStoreKeyFromAppOrHash(s *store.Store, input string) (string, error) {
	var key string
	if _, err := types.NewHash(input); err == nil {
		key, err = s.ResolveKey(input)
		if err != nil {
			return "", fmt.Errorf("cannot resolve key: %v", err)
		}
	} else {
		key, err = getStoreKeyFromApp(s, input)
		if err != nil {
			return "", fmt.Errorf("cannot find image: %v", err)
		}
	}
	return key, nil
}

func getStoreKeyFromAppOrHash(s *store.Store, input string) (string, error) {
	var key string
	if _, err := types.NewHash(input); err == nil {
		key, err = s.ResolveKey(input)
		if err != nil {
			return "", errwrap.Wrap(errors.New("cannot resolve image ID"), err)
		}
	} else {
		key, err = getStoreKeyFromApp(s, input)
		if err != nil {
			return "", errwrap.Wrap(errors.New("cannot find image"), err)
		}
	}
	return key, nil
}

// aciInfoToV1AlphaAPIImage takes an aciInfo object and construct the v1alpha.Image object.
// It also returns the image manifest of the image.
func aciInfoToV1AlphaAPIImage(store *store.Store, aciInfo *store.ACIInfo) (*v1alpha.Image, *schema.ImageManifest, error) {
	manifest, err := store.GetImageManifestJSON(aciInfo.BlobKey)
	if err != nil {
		log.Printf("Failed to read the image manifest: %v", err)
		return nil, nil, err
	}

	var im schema.ImageManifest
	if err = json.Unmarshal(manifest, &im); err != nil {
		log.Printf("Failed to unmarshal image manifest: %v", err)
		return nil, nil, err
	}

	version, ok := im.Labels.Get("version")
	if !ok {
		version = "latest"
	}

	return &v1alpha.Image{
		BaseFormat: &v1alpha.ImageFormat{
			// Only support appc image now. If it's a docker image, then it
			// will be transformed to appc before storing in the disk store.
			Type:    v1alpha.ImageType_IMAGE_TYPE_APPC,
			Version: schema.AppContainerVersion.String(),
		},
		Id:              aciInfo.BlobKey,
		Name:            im.Name.String(),
		Version:         version,
		ImportTimestamp: aciInfo.ImportTime.Unix(),
		Manifest:        manifest,
	}, &im, nil
}

// ExtractLayerInfo extracts the image name and ID from a path to an ACI
func ExtractLayerInfo(store *store.Store, in string) (types.Dependency, error) {
	im, err := GetManifestFromImage(in)
	if err != nil {
		return types.Dependency{}, fmt.Errorf("error getting manifest from image (%v): %v", in, err)
	}

	inFile, err := os.Open(in)
	if err != nil {
		return types.Dependency{}, fmt.Errorf("error opening ACI: %v", err)
	}
	defer inFile.Close()

	inImageID, err := store.WriteACI(inFile, false)
	if err != nil {
		return types.Dependency{}, fmt.Errorf("error writing ACI into the tree store: %v", err)
	}

	hash, err := types.NewHash(inImageID)
	if err != nil {
		return types.Dependency{}, fmt.Errorf("error creating hash from an image ID (%s): %v", hash, err)
	}

	return types.Dependency{
		ImageName: im.Name,
		ImageID:   hash,
	}, nil
}

// aciInfoToV1AlphaAPIImage takes an aciInfo object and construct the v1alpha.Image
// object. It will also get and return the image manifest.
// Note that v1alpha.Image.Manifest field is not set by this function.
func aciInfoToV1AlphaAPIImage(store *store.Store, aciInfo *store.ACIInfo) (*v1alpha.Image, *schema.ImageManifest, error) {
	imgManifest, err := store.GetImageManifest(aciInfo.BlobKey)
	if err != nil {
		return nil, nil, err
	}

	data, err := json.Marshal(imgManifest)
	if err != nil {
		return nil, nil, err
	}

	version, ok := imgManifest.Labels.Get("version")
	if !ok {
		version = "latest"
	}

	return &v1alpha.Image{
		BaseFormat: &v1alpha.ImageFormat{
			// Only support appc image now. If it's a docker image, then it
			// will be transformed to appc before storing in the disk store.
			Type:    v1alpha.ImageType_IMAGE_TYPE_APPC,
			Version: schema.AppContainerVersion.String(),
		},
		Id:              aciInfo.BlobKey,
		Name:            imgManifest.Name.String(),
		Version:         version,
		ImportTimestamp: aciInfo.ImportTime.Unix(),
		Manifest:        data,
	}, imgManifest, nil
}

// renderInStore renders a ACI specified by `filename` in the given tree store,
// and returns the hash (image ID) of the rendered ACI.
func renderInStore(s *store.Store, filename string) (string, error) {
	// Put the ACI into the store
	f, err := os.Open(filename)
	if err != nil {
		return "", fmt.Errorf("Could not open ACI image: %s", err)
	}

	key, err := s.WriteACI(f, false)
	if err != nil {
		return "", fmt.Errorf("Could not open ACI: %s", key)
	}

	// Render the ACI
	if err := s.RenderTreeStore(key, false); err != nil {
		return "", fmt.Errorf("Could not render tree store: %s", err)
	}

	return key, err
}

func gcStore(s *store.Store, gracePeriod time.Duration) error {
	var imagesToRemove []string
	aciinfos, err := s.GetAllACIInfos([]string{"lastused"}, true)
	if err != nil {
		return fmt.Errorf("Failed to get aciinfos: %v", err)
	}
	for _, ai := range aciinfos {
		if time.Now().Sub(ai.LastUsed) <= gracePeriod {
			break
		}
		imagesToRemove = append(imagesToRemove, ai.BlobKey)
	}

	if err := rmImages(s, imagesToRemove); err != nil {
		return err
	}

	return nil
}

// getImageInfo for a given image ID, returns the *v1alpha.Image object.
func getImageInfo(store *store.Store, imageID string) (*v1alpha.Image, error) {
	key, err := store.ResolveKey(imageID)
	if err != nil {
		stderr.PrintE(fmt.Sprintf("failed to resolve the image ID %q", imageID), err)
		return nil, err
	}

	aciInfo, err := store.GetACIInfoWithBlobKey(key)
	if err != nil {
		stderr.PrintE(fmt.Sprintf("failed to get ACI info for image %q", key), err)
		return nil, err
	}

	image, _, err := aciInfoToV1AlphaAPIImage(store, aciInfo)
	if err != nil {
		stderr.PrintE(fmt.Sprintf("failed to convert ACI to v1alphaAPIImage for image %q", key), err)
		return nil, err
	}
	return image, nil
}

func getStoreKeyFromApp(s *store.Store, img string) (string, error) {
	app, err := discovery.NewAppFromString(img)
	if err != nil {
		return "", fmt.Errorf("cannot parse the image name %q: %v", img, err)
	}
	labels, err := types.LabelsFromMap(app.Labels)
	if err != nil {
		return "", fmt.Errorf("invalid labels in the image %q: %v", img, err)
	}
	key, err := s.GetACI(app.Name, labels)
	if err != nil {
		switch err.(type) {
		case store.ACINotFoundError:
			return "", err
		default:
			return "", fmt.Errorf("cannot find image %q: %v", img, err)
		}
	}
	return key, nil
}

// getImageInfo for a given image ID, returns the *v1alpha.Image object.
//
// FIXME(yifan): We should get the image manifest from the tree store.
// See https://github.com/coreos/rkt/issues/1659
func getImageInfo(store *store.Store, imageID string) (*v1alpha.Image, error) {
	aciInfo, err := store.GetACIInfoWithBlobKey(imageID)
	if err != nil {
		log.Printf("Failed to get ACI info for image ID %q: %v", imageID, err)
		return nil, err
	}

	image, _, err := aciInfoToV1AlphaAPIImage(store, aciInfo)
	if err != nil {
		log.Printf("Failed to convert ACI to v1alphaAPIImage for image ID %q: %v", imageID, err)
		return nil, err
	}
	return image, nil
}

func getKeyFromAppOrHash(s *store.Store, input string) (string, error) {
	var key string
	if _, err := types.NewHash(input); err == nil {
		key, err = s.ResolveKey(input)
		if err != nil {
			return "", fmt.Errorf("cannot resolve key: %v", err)
		}
	} else {
		app, err := discovery.NewAppFromString(input)
		if err != nil {
			return "", fmt.Errorf("cannot parse the image name: %v", err)
		}
		labels, err := types.LabelsFromMap(app.Labels)
		if err != nil {
			return "", fmt.Errorf("invalid labels in the name: %v", err)
		}
		key, err = s.GetACI(app.Name, labels)
		if err != nil {
			return "", fmt.Errorf("cannot find image: %v", err)
		}
	}

	return key, nil
}

func rmImages(s *store.Store, images []string) error {
	done := 0
	errors := 0
	staleErrors := 0
	for _, pkey := range images {
		var (
			keys []string
			name string
		)
		errors++
		h, err := types.NewHash(pkey)
		if err != nil {
			var found bool
			keys, found, err = s.ResolveName(pkey)
			if len(keys) > 0 {
				errors += len(keys) - 1
			}
			if err != nil {
				stderr("rkt: %v", err)
				continue
			}
			if !found {
				stderr("rkt: image name %q not found", pkey)
				continue
			}
			name = pkey
		} else {
			key, err := s.ResolveKey(h.String())
			if err != nil {
				stderr("rkt: image ID %q not valid: %v", pkey, err)
				continue
			}
			if key == "" {
				stderr("rkt: image ID %q doesn't exist", pkey)
				continue
			}

			aciinfo, err := s.GetACIInfoWithBlobKey(key)
			if err != nil {
				stderr("rkt: error retrieving aci infos for image %q: %v", key, err)
				continue
			}
			name = aciinfo.Name
			keys = append(keys, key)
		}

		for _, key := range keys {
			if err = s.RemoveACI(key); err != nil {
				if serr, ok := err.(*store.StoreRemovalError); ok {
					staleErrors++
					stderr("rkt: some files cannot be removed for image %q (%q): %v", key, name, serr)
				} else {
					stderr("rkt: error removing aci for image %q (%q): %v", key, name, err)
					continue
				}
			}
			stdout("rkt: successfully removed aci for image: %q (%q)", key, name)
			errors--
			done++
		}
	}

	if done > 0 {
		stderr("rkt: %d image(s) successfully removed", done)
	}

	// If anything didn't complete, return exit status of 1
	if (errors + staleErrors) > 0 {
		if staleErrors > 0 {
			stderr("rkt: %d image(s) removed but left some stale files", staleErrors)
		}
		if errors > 0 {
			stderr("rkt: %d image(s) cannot be removed", errors)
		}
		return fmt.Errorf("error(s) found while removing images")
	}

	return nil
}

func rmImages(s *store.Store, images []string) error {
	done := 0
	errors := 0
	staleErrors := 0
	imageMap := make(map[string]string)
	imageCounter := make(map[string]int)

	for _, pkey := range images {
		errors++
		h, err := types.NewHash(pkey)
		if err != nil {
			var found bool
			keys, found, err := s.ResolveName(pkey)
			if len(keys) > 0 {
				errors += len(keys) - 1
			}
			if err != nil {
				stderr("rkt: %v", err)
				continue
			}
			if !found {
				stderr("rkt: image name %q not found", pkey)
				continue
			}
			for _, key := range keys {
				imageMap[key] = pkey
				imageCounter[key]++
			}
		} else {
			key, err := s.ResolveKey(h.String())
			if err != nil {
				stderr("rkt: image ID %q not valid: %v", pkey, err)
				continue
			}
			if key == "" {
				stderr("rkt: image ID %q doesn't exist", pkey)
				continue
			}

			aciinfo, err := s.GetACIInfoWithBlobKey(key)
			if err != nil {
				stderr("rkt: error retrieving aci infos for image %q: %v", key, err)
				continue
			}
			imageMap[key] = aciinfo.Name
			imageCounter[key]++
		}
	}

	// Adjust the error count by subtracting duplicate IDs from it,
	// therefore allowing only one error per ID.
	for _, c := range imageCounter {
		if c > 1 {
			errors -= c - 1
		}
	}

	for key, name := range imageMap {
		if err := s.RemoveACI(key); err != nil {
			if serr, ok := err.(*store.StoreRemovalError); ok {
				staleErrors++
				stderr("rkt: some files cannot be removed for image %q (%q): %v", key, name, serr)
			} else {
				stderr("rkt: error removing aci for image %q (%q): %v", key, name, err)
				continue
			}
		}
		stdout("rkt: successfully removed aci for image: %q (%q)", key, name)
		errors--
		done++
	}

	if done > 0 {
		stderr("rkt: %d image(s) successfully removed", done)
	}

	// If anything didn't complete, return exit status of 1
	if (errors + staleErrors) > 0 {
		if staleErrors > 0 {
			stderr("rkt: %d image(s) removed but left some stale files", staleErrors)
		}
		if errors > 0 {
			stderr("rkt: %d image(s) cannot be removed", errors)
		}
		return fmt.Errorf("error(s) found while removing images")
	}

	return nil
}

// fillAppInfo fills the apps' state and image info of the pod.
func fillAppInfo(store *store.Store, p *pod, v1pod *v1alpha.Pod) error {
	statusDir, err := p.getStatusDir()
	if err != nil {
		stderr.PrintE("failed to get pod exit status directory", err)
		return err
	}

	for _, app := range v1pod.Apps {
		// Fill app's image info (id, name, version).
		fullImageID, err := store.ResolveKey(app.Image.Id)
		if err != nil {
			stderr.PrintE(fmt.Sprintf("failed to resolve the image ID %q", app.Image.Id), err)
			return err
		}

		im, err := p.getAppImageManifest(*types.MustACName(app.Name))
		if err != nil {
			stderr.PrintE(fmt.Sprintf("failed to get image manifests for app %q", app.Name), err)
			return err
		}

		version, ok := im.Labels.Get("version")
		if !ok {
			version = "latest"
		}

		app.Image = &v1alpha.Image{
			BaseFormat: &v1alpha.ImageFormat{
				// Only support appc image now. If it's a docker image, then it
				// will be transformed to appc before storing in the disk store.
				Type:    v1alpha.ImageType_IMAGE_TYPE_APPC,
				Version: schema.AppContainerVersion.String(),
			},
			Id:      fullImageID,
			Name:    im.Name.String(),
			Version: version,
			// Other information are not available because they require the image
			// info from store.
		}

		// Fill app's state and exit code.
		value, err := p.readIntFromFile(filepath.Join(statusDir, app.Name))
		if err == nil {
			app.State = v1alpha.AppState_APP_STATE_EXITED
			app.ExitCode = int32(value)
			continue
		}

		if !os.IsNotExist(err) {
			stderr.PrintE(fmt.Sprintf("failed to read status for app %q", app.Name), err)
			return err
		}
		// If status file does not exit, that means the
		// app is either running or aborted.
		//
		// FIXME(yifan): This is not acttually true, the app can be aborted while
		// the pod is still running if the spec changes.
		switch p.getState() {
		case Running:
			app.State = v1alpha.AppState_APP_STATE_RUNNING
		default:
			app.State = v1alpha.AppState_APP_STATE_UNDEFINED
		}

	}
	return nil
}