def testCompileModules(self):
		for parent, dirs, files in itertools.chain(
			os.walk(PORTAGE_BIN_PATH),
			os.walk(PORTAGE_PYM_PATH)):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-4:] in ('.pyc', '.pyo'):
					continue
				x = os.path.join(parent, x)
				st = os.lstat(x)
				if not stat.S_ISREG(st.st_mode):
					continue
				do_compile = False
				cfile = x
				if x[-3:] == '.py':
					do_compile = True
				else:
					# Check for python shebang
					f = open(_unicode_encode(x,
						encoding=_encodings['fs'], errors='strict'), 'rb')
					line = _unicode_decode(f.readline(),
						encoding=_encodings['content'], errors='replace')
					f.close()
					if line[:2] == '#!' and \
						'python' in line:
						do_compile = True
						cfile += '.py'
				if do_compile:
					cfile += (__debug__ and 'c' or 'o')
					py_compile.compile(x, cfile=cfile, doraise=True)

	def testCompileModules(self):
		for parent, dirs, files in itertools.chain(
			os.walk(PORTAGE_BIN_PATH),
			os.walk(PORTAGE_PYM_PATH)):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-4:] in ('.pyc', '.pyo'):
					continue
				x = os.path.join(parent, x)
				st = os.lstat(x)
				if not stat.S_ISREG(st.st_mode):
					continue
				do_compile = False
				if x[-3:] == '.py':
					do_compile = True
				else:
					# Check for python shebang
					with open(_unicode_encode(x,
						encoding=_encodings['fs'], errors='strict'), 'rb') as f:
						line = _unicode_decode(f.readline(),
							encoding=_encodings['content'], errors='replace')
					if line[:2] == '#!' and 'python' in line:
						do_compile = True
				if do_compile:
					with open(_unicode_encode(x,
						encoding=_encodings['fs'], errors='strict'), 'rb') as f:
						compile(f.read(), x, 'exec')

	def testCompileModules(self):
		iters = [os.walk(os.path.join(PORTAGE_PYM_PATH, x))
			for x in PORTAGE_PYM_PACKAGES]
		iters.append(os.walk(PORTAGE_BIN_PATH))

		for parent, _dirs, files in itertools.chain(*iters):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-4:] in ('.pyc', '.pyo'):
					continue
				x = os.path.join(parent, x)
				st = os.lstat(x)
				if not stat.S_ISREG(st.st_mode):
					continue

				bin_path = os.path.relpath(x, PORTAGE_BIN_PATH)
				mod_path = os.path.relpath(x, PORTAGE_PYM_PATH)

				meta = module_metadata.get(mod_path) or script_metadata.get(bin_path)
				if meta:
					req_py = tuple(int(x) for x
							in meta.get('required_python', '0.0').split('.'))
					if sys.version_info < req_py:
						continue

				do_compile = False
				if x[-3:] == '.py':
					do_compile = True
				else:
					# Check for python shebang.
					try:
						with open(_unicode_encode(x,
							encoding=_encodings['fs'], errors='strict'), 'rb') as f:
							line = _unicode_decode(f.readline(),
								encoding=_encodings['content'], errors='replace')
					except IOError as e:
						# Some tests create files that are unreadable by the
						# user (by design), so ignore EACCES issues.
						if e.errno != errno.EACCES:
							raise
						continue
					if line[:2] == '#!' and 'python' in line:
						do_compile = True
				if do_compile:
					with open(_unicode_encode(x,
						encoding=_encodings['fs'], errors='strict'), 'rb') as f:
						compile(f.read(), x, 'exec')

	def multiBuilder(self, options, settings, trees):
		rValue = {}
		directory = options.get("directory",
			os.path.join(settings["PORTAGE_CONFIGROOT"],
			USER_CONFIG_PATH, "sets"))
		name_pattern = options.get("name_pattern", "${name}")
		if not "$name" in name_pattern and not "${name}" in name_pattern:
			raise SetConfigError(_("name_pattern doesn't include ${name} placeholder"))
		greedy = get_boolean(options, "greedy", False)
		# look for repository path variables
		match = self._repopath_match.match(directory)
		if match:
			try:
				directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory)
			except KeyError:
				raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"])

		try:
			directory = _unicode_decode(directory,
				encoding=_encodings['fs'], errors='strict')
			# Now verify that we can also encode it.
			_unicode_encode(directory,
				encoding=_encodings['fs'], errors='strict')
		except UnicodeError:
			directory = _unicode_decode(directory,
				encoding=_encodings['fs'], errors='replace')
			raise SetConfigError(
				_("Directory path contains invalid character(s) for encoding '%s': '%s'") \
				% (_encodings['fs'], directory))

		if os.path.isdir(directory):
			directory = normalize_path(directory)

			for parent, dirs, files in os.walk(directory):
				try:
					parent = _unicode_decode(parent,
						encoding=_encodings['fs'], errors='strict')
				except UnicodeDecodeError:
					continue
				for d in dirs[:]:
					if d[:1] == '.':
						dirs.remove(d)
				for filename in files:
					try:
						filename = _unicode_decode(filename,
							encoding=_encodings['fs'], errors='strict')
					except UnicodeDecodeError:
						continue
					if filename[:1] == '.':
						continue
					if filename.endswith(".metadata"):
						continue
					filename = os.path.join(parent,
						filename)[1 + len(directory):]
					myname = name_pattern.replace("$name", filename)
					myname = myname.replace("${name}", filename)
					rValue[myname] = StaticFileSet(
						os.path.join(directory, filename),
						greedy=greedy, dbapi=trees["vartree"].dbapi)
		return rValue

def addtolist(mylist, curdir):
	"""(list, dir) --- Takes an array(list) and appends all files from dir down
	the directory tree. Returns nothing. list is modified."""
	curdir = normalize_path(_unicode_decode(curdir,
		encoding=_encodings['fs'], errors='strict'))
	for parent, dirs, files in os.walk(curdir):

		parent = _unicode_decode(parent,
			encoding=_encodings['fs'], errors='strict')
		if parent != curdir:
			mylist.append(parent[len(curdir) + 1:] + os.sep)

		for x in dirs:
			try:
				_unicode_decode(x, encoding=_encodings['fs'], errors='strict')
			except UnicodeDecodeError:
				dirs.remove(x)

		for x in files:
			try:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
			except UnicodeDecodeError:
				continue
			mylist.append(os.path.join(parent, x)[len(curdir) + 1:])

	def testBashSyntax(self):
		for parent, dirs, files in os.walk(PORTAGE_BIN_PATH):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				ext = x.split('.')[-1]
				if ext in ('.py', '.pyc', '.pyo'):
					continue
				x = os.path.join(parent, x)
				st = os.lstat(x)
				if not stat.S_ISREG(st.st_mode):
					continue

				# Check for bash shebang
				f = open(_unicode_encode(x,
					encoding=_encodings['fs'], errors='strict'), 'rb')
				line = _unicode_decode(f.readline(),
					encoding=_encodings['content'], errors='replace')
				f.close()
				if line[:2] == '#!' and \
					'bash' in line:
					cmd = "%s -n %s" % (_shell_quote(BASH_BINARY), _shell_quote(x))
					status, output = subprocess_getstatusoutput(cmd)
					self.assertEqual(os.WIFEXITED(status) and \
						os.WEXITSTATUS(status) == os.EX_OK, True, msg=output)

def main():

	TEST_FILE = _unicode_encode('__test__',
		encoding=_encodings['fs'], errors='strict')
	svn_dirname = _unicode_encode('.svn',
		encoding=_encodings['fs'], errors='strict')
	suite = unittest.TestSuite()
	basedir = os.path.dirname(os.path.realpath(__file__))
	testDirs = []

  # the os.walk help mentions relative paths as being quirky
	# I was tired of adding dirs to the list, so now we add __test__
	# to each dir we want tested.
	for root, dirs, files in os.walk(basedir):
		if svn_dirname in dirs:
			dirs.remove(svn_dirname)
		try:
			root = _unicode_decode(root,
				encoding=_encodings['fs'], errors='strict')
		except UnicodeDecodeError:
			continue

		if TEST_FILE in files:
			testDirs.append(root)

	for mydir in testDirs:
		suite.addTests(getTests(os.path.join(basedir, mydir), basedir) )
	return TextTestRunner(verbosity=2).run(suite)

def RecursiveFileLoader(filename):
    """
	If filename is of type file, return a generate that yields filename
	else if filename is of type directory, return a generator that fields
	files in that directory.
	
	Ignore files beginning with . or ending in ~.
	Prune CVS directories.

	@param filename: name of a file/directory to traverse
	@rtype: list
	@returns: List of files to process
	"""

    try:
        st = os.stat(filename)
    except OSError:
        return
    if stat.S_ISDIR(st.st_mode):
        for root, dirs, files in os.walk(filename):
            for d in list(dirs):
                if d[:1] == "." or d == "CVS":
                    dirs.remove(d)
            for f in files:
                try:
                    f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
                except UnicodeDecodeError:
                    continue
                if f[:1] == "." or f[-1:] == "~":
                    continue
                yield os.path.join(root, f)
    else:
        yield filename

	def testCompileModules(self):
		for parent, dirs, files in os.walk(PORTAGE_PYM_PATH):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-3:] == '.py':
					py_compile.compile(os.path.join(parent, x), doraise=True)

	def _prune_empty_dirs(self):
		all_dirs = []
		for parent, dirs, files in os.walk(self.location):
			for x in dirs:
				all_dirs.append(_os.path.join(parent, x))
		while all_dirs:
			try:
				_os.rmdir(all_dirs.pop())
			except OSError:
				pass

	def _apply_max_mtime(self, preserved_stats, entries):
		"""
		Set the Manifest mtime to the max mtime of all relevant files
		and directories. Directory mtimes account for file renames and
		removals. The existing Manifest mtime accounts for eclass
		modifications that change DIST entries. This results in a
		stable/predictable mtime, which is useful when converting thin
		manifests to thick manifests for distribution via rsync. For
		portability, the mtime is set with 1 second resolution.

		@param preserved_stats: maps paths to preserved stat results
			that should be used instead of os.stat() calls
		@type preserved_stats: dict
		@param entries: list of current Manifest2Entry instances
		@type entries: list
		"""
		# Use stat_result[stat.ST_MTIME] for 1 second resolution, since
		# it always rounds down. Note that stat_result.st_mtime will round
		# up from 0.999999999 to 1.0 when precision is lost during conversion
		# from nanosecond resolution to float.
		max_mtime = None
		_update_max = (lambda st: max_mtime if max_mtime is not None
			and max_mtime > st[stat.ST_MTIME] else st[stat.ST_MTIME])
		_stat = (lambda path: preserved_stats[path] if path in preserved_stats
			else os.stat(path))

		for stat_result in preserved_stats.values():
			max_mtime = _update_max(stat_result)

		for entry in entries:
			if entry.type == 'DIST':
				continue
			abs_path = (os.path.join(self.pkgdir, 'files', entry.name) if
				entry.type == 'AUX' else os.path.join(self.pkgdir, entry.name))
			max_mtime = _update_max(_stat(abs_path))

		if not self.thin:
			# Account for changes to all relevant nested directories.
			# This is not necessary for thin manifests because
			# self.pkgdir is already included via preserved_stats.
			for parent_dir, dirs, files in os.walk(self.pkgdir.rstrip(os.sep)):
				try:
					parent_dir = _unicode_decode(parent_dir,
						encoding=_encodings['fs'], errors='strict')
				except UnicodeDecodeError:
					# If an absolute path cannot be decoded, then it is
					# always excluded from the manifest (repoman will
					# report such problems).
					pass
				else:
					max_mtime = _update_max(_stat(parent_dir))

		if max_mtime is not None:
			for path in preserved_stats:
				os.utime(path, (max_mtime, max_mtime))

	def _update_thick_pkgdir(self, cat, pn, pkgdir):
		cpvlist = []
		for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
			break
		for f in pkgdir_files:
			try:
				f = _unicode_decode(f,
					encoding=_encodings['fs'], errors='strict')
			except UnicodeDecodeError:
				continue
			if f[:1] == ".":
				continue
			pf = self._is_cpv(cat, pn, f)
			if pf is not None:
				mytype = "EBUILD"
				cpvlist.append(pf)
			elif self._find_invalid_path_char(f) == -1 and \
				manifest2MiscfileFilter(f):
				mytype = "MISC"
			else:
				continue
			self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
		recursive_files = []

		pkgdir = self.pkgdir
		cut_len = len(os.path.join(pkgdir, "files") + os.sep)
		for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
			for f in files:
				try:
					f = _unicode_decode(f,
						encoding=_encodings['fs'], errors='strict')
				except UnicodeDecodeError:
					continue
				full_path = os.path.join(parentdir, f)
				recursive_files.append(full_path[cut_len:])
		for f in recursive_files:
			if self._find_invalid_path_char(f) != -1 or \
				not manifest2AuxfileFilter(f):
				continue
			self.fhashdict["AUX"][f] = perform_multiple_checksums(
				os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
		return cpvlist

def apply_recursive_permissions(top, uid=-1, gid=-1,
	dirmode=-1, dirmask=-1, filemode=-1, filemask=-1, onerror=None):
	"""A wrapper around apply_secpass_permissions that applies permissions
	recursively.  If optional argument onerror is specified, it should be a
	function; it will be called with one argument, a PortageException instance.
	Returns True if all permissions are applied and False if some are left
	unapplied."""

	# Avoid issues with circular symbolic links, as in bug #339670.
	follow_links = False

	if onerror is None:
		# Default behavior is to dump errors to stderr so they won't
		# go unnoticed.  Callers can pass in a quiet instance.
		def onerror(e):
			if isinstance(e, OperationNotPermitted):
				writemsg(_("Operation Not Permitted: %s\n") % str(e),
					noiselevel=-1)
			elif isinstance(e, FileNotFound):
				writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1)
			else:
				raise

	all_applied = True
	for dirpath, dirnames, filenames in os.walk(top):
		try:
			applied = apply_secpass_permissions(dirpath,
				uid=uid, gid=gid, mode=dirmode, mask=dirmask,
				follow_links=follow_links)
			if not applied:
				all_applied = False
		except PortageException as e:
			all_applied = False
			onerror(e)

		for name in filenames:
			try:
				applied = apply_secpass_permissions(os.path.join(dirpath, name),
					uid=uid, gid=gid, mode=filemode, mask=filemask,
					follow_links=follow_links)
				if not applied:
					all_applied = False
			except PortageException as e:
				# Ignore InvalidLocation exceptions such as FileNotFound
				# and DirectoryNotFound since sometimes things disappear,
				# like when adjusting permissions on DISTCC_DIR.
				if not isinstance(e, portage.exception.InvalidLocation):
					all_applied = False
					onerror(e)
	return all_applied

 def testCompileModules(self):
     for parent, dirs, files in itertools.chain(os.walk(PORTAGE_BIN_PATH), os.walk(PORTAGE_PYM_PATH)):
         parent = _unicode_decode(parent, encoding=_encodings["fs"], errors="strict")
         for x in files:
             x = _unicode_decode(x, encoding=_encodings["fs"], errors="strict")
             if x[-4:] in (".pyc", ".pyo"):
                 continue
             x = os.path.join(parent, x)
             st = os.lstat(x)
             if not stat.S_ISREG(st.st_mode):
                 continue
             do_compile = False
             if x[-3:] == ".py":
                 do_compile = True
             else:
                 # Check for python shebang
                 f = open(_unicode_encode(x, encoding=_encodings["fs"], errors="strict"), "rb")
                 line = _unicode_decode(f.readline(), encoding=_encodings["content"], errors="replace")
                 f.close()
                 if line[:2] == "#!" and "python" in line:
                     do_compile = True
             if do_compile:
                 py_compile.compile(x, cfile="/dev/null", doraise=True)

	def testCompileModules(self):
		for parent, _dirs, files in itertools.chain(
			os.walk(PORTAGE_BIN_PATH),
			os.walk(PORTAGE_PYM_PATH)):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-4:] in ('.pyc', '.pyo'):
					continue
				x = os.path.join(parent, x)
				st = os.lstat(x)
				if not stat.S_ISREG(st.st_mode):
					continue
				do_compile = False
				if x[-3:] == '.py':
					do_compile = True
				else:
					# Check for python shebang.
					try:
						with open(_unicode_encode(x,
							encoding=_encodings['fs'], errors='strict'), 'rb') as f:
							line = _unicode_decode(f.readline(),
								encoding=_encodings['content'], errors='replace')
					except IOError as e:
						# Some tests create files that are unreadable by the
						# user (by design), so ignore EACCES issues.
						if e.errno != errno.EACCES:
							raise
						continue
					if line[:2] == '#!' and 'python' in line:
						do_compile = True
				if do_compile:
					with open(_unicode_encode(x,
						encoding=_encodings['fs'], errors='strict'), 'rb') as f:
						compile(f.read(), x, 'exec')

	def __iter__(self):

		for root, dirs, files in os.walk(self.portdir):
			for file in files:
				try:
					file = _unicode_decode(file,
						encoding=_encodings['fs'], errors='strict')
				except UnicodeDecodeError:
					continue
				if file[-7:] == '.ebuild':
					cat = os.path.basename(os.path.dirname(root))
					pn_pv = file[:-7]
					path = os.path.join(root,file)
					if self.__has_cache(path):
						yield "%s/%s/%s" % (cat,os.path.basename(root),file[:-7])

 def _update_thin_pkgdir(self, cat, pn, pkgdir):
     for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
         break
     cpvlist = []
     for f in pkgdir_files:
         try:
             f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
         except UnicodeDecodeError:
             continue
         if f[:1] == ".":
             continue
         pf = self._is_cpv(cat, pn, f)
         if pf is not None:
             cpvlist.append(pf)
     return cpvlist

def install_mask_dir(base_dir, install_mask, onerror=None):
	"""
	Remove files and directories matched by INSTALL_MASK.

	@param base_dir: directory path corresponding to ${ED}
	@type base_dir: str
	@param install_mask: INSTALL_MASK configuration
	@type install_mask: InstallMask
	"""
	onerror = onerror or _raise_exc
	base_dir = normalize_path(base_dir)
	base_dir_len = len(base_dir) + 1
	dir_stack = []

	# Remove masked files.
	for parent, dirs, files in os.walk(base_dir, onerror=onerror):
		try:
			parent = _unicode_decode(parent, errors='strict')
		except UnicodeDecodeError:
			continue
		dir_stack.append(parent)
		for fname in files:
			try:
				fname = _unicode_decode(fname, errors='strict')
			except UnicodeDecodeError:
				continue
			abs_path = os.path.join(parent, fname)
			relative_path = abs_path[base_dir_len:]
			if install_mask.match(relative_path):
				try:
					os.unlink(abs_path)
				except OSError as e:
					onerror(e)

	# Remove masked dirs (unless non-empty due to exclusions).
	while True:
		try:
			dir_path = dir_stack.pop()
		except IndexError:
			break

		if install_mask.match(dir_path[base_dir_len:] + '/'):
			try:
				os.rmdir(dir_path)
			except OSError:
				pass

	def _iter_modules(self, base_dir):
		for parent, dirs, files in os.walk(base_dir):
			parent = _unicode_decode(parent,
				encoding=_encodings['fs'], errors='strict')
			parent_mod = parent[len(PORTAGE_PYM_PATH)+1:]
			parent_mod = parent_mod.replace("/", ".")
			for x in files:
				x = _unicode_decode(x,
					encoding=_encodings['fs'], errors='strict')
				if x[-3:] != '.py':
					continue
				x = x[:-3]
				if x[-8:] == '__init__':
					x = parent_mod
				else:
					x = parent_mod + "." + x
				yield x

def getTestDirs(base_path):
    TEST_FILE = b"__test__"
    testDirs = []

    # the os.walk help mentions relative paths as being quirky
    # I was tired of adding dirs to the list, so now we add __test__
    # to each dir we want tested.
    for root, dirs, files in os.walk(base_path):
        try:
            root = _unicode_decode(root, encoding=_encodings["fs"], errors="strict")
        except UnicodeDecodeError:
            continue

        if TEST_FILE in files:
            testDirs.append(root)

    testDirs.sort()
    return testDirs