def _make_ts_diff(old_ts_path, new_ts_path):
    old = factory.getobject(old_ts_path)
    new = factory.getobject(new_ts_path)

    # this is for added / removed
    old_ids = set(old.getids())
    new_ids = set(new.getids())
    added = new_ids - old_ids
    removed = old_ids - new_ids

    old_trans_count = 0
    for unit_id in old_ids:
        if old.findid(unit_id).istranslated():
            old_trans_count += 1

    new_trans_count = 0
    for unit_id in new_ids:
        if new.findid(unit_id).istranslated():
            new_trans_count += 1

    return {
        "old_trans_count": old_trans_count,
        "new_trans_count": new_trans_count,
        "old_units": len(old_ids),
        "new_units": len(new_ids),
        "added": list(added),
        "removed": list(removed),
    }

 def scanstores(self):
     """Populate the internal store data."""
     for dirname, filename in self.file_iter():
         # TODO: Here os.path.join(dirname, filename) doesn't work.....
         if filename == '':continue
         if dirname=='':
             filepath=os.path.join(dirname, filename)
         else:
             filepath=dirname+'/'+filename
         strfile = wStringIO.StringIO(self.archive.read(filepath))
         # let skip the other filetypes            
         root, ext = os.path.splitext(filename)
         if self.specifyext:
             if ext[1:].lower() != self.specifyext.lower(): 
                 strfile.filename = os.path.join(dirname, filename)
                 self.storedata.append(strfile)
             else:
                 strfile.filename = filename
                 store = factory.getobject(strfile)
                 self.storedata.append(store)
         else:
             if ext[1:] not in factory.classes_str: 
                 strfile.filename = os.path.join(dirname, filename)
                 self.storedata.append(strfile)
             else:
                 strfile.filename = filename
                 store = factory.getobject(strfile)
                 self.storedata.append(store)

    def __init__(self, tmdbfile, tmfiles, max_candidates=3, min_similarity=75,
            max_length=1000, prefix="", source_lang=None, target_lang=None):

        self.tmdb = tmdb.TMDB(tmdbfile, max_candidates, min_similarity, max_length)

        #load files into db
        if isinstance(tmfiles, list):
            [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
                    for tmfile in tmfiles]
        elif tmfiles:
            self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)

        #initialize url dispatcher
        self.rest = selector.Selector(prefix=prefix)
        self.rest.add("/{slang}/{tlang}/unit/{uid:any}",
                      GET=self.translate_unit,
                      POST=self.update_unit,
                      PUT=self.add_unit,
                      DELETE=self.forget_unit
                      )

        self.rest.add("/{slang}/{tlang}/store/{sid:any}",
                      GET=self.get_store_stats,
                      PUT=self.upload_store,
                      POST=self.add_store,
                      DELETE=self.forget_store)

    def _checkerrors(self, filename, fileid, configid, checker, store):

        def geterrors():
            self.cur.execute("""SELECT
                name,
                unitindex
                FROM uniterrors WHERE fileid=? and configid=?
                ORDER BY unitindex;""", (fileid, configid))
            return self.cur.fetchone(), self.cur

        first, cur = geterrors()
        if first is not None:
            return first, cur

        # This could happen if we haven't done the checks before, or the
        # file changed, or we are using a different configuration
        if callable(store):
            store = store()
        else:
            store = store or factory.getobject(filename)

        if os.path.exists(suggestion_filename(filename)):
            checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
        self._cachestorechecks(fileid, store, checker, configid)
        return geterrors()

def main(dir, output, pstr):
	
	import random
	percentage = float(pstr)
	from translate.storage import factory
	unitslist = []
	create_intermediates(dir)
	
	print "Creating final TMX..."
	for f in os.listdir(dir):
		if f.endswith(".tmx") and not os.path.isdir(f):
			try:
				filepath = os.path.join(dir,f)
				minicorpus = factory.getobject(filepath)
				for u in minicorpus.units:
					u.addnote("Origin: "+f)
				unitslist.append(minicorpus.units)
			except ValueError:
				print "Could not convert to factory."
				continue
	
	sample_list = create_sample(unitslist, percentage)
	sample = []
	for l in sample_list:
		sample.extend(l)
	random.shuffle(sample)
	
	if os.path.exists(output):
		os.remove(output)
	newcorpus = factory.getobject(output)
	for u in sample:
		newunit = newcorpus.UnitClass.buildfromunit(u)
		newcorpus.addunit(newunit)
	newcorpus.save()

def mergestore(inputfile, outputfile, templatefile, mergeblanks="no", mergefuzzy="no",
               mergecomments="yes"):
    try:
        mergecomments = str2bool(mergecomments)
    except ValueError:
        raise ValueError("invalid mergecomments value: %r" % mergecomments)
    try:
        mergeblanks = str2bool(mergeblanks)
    except ValueError:
        raise ValueError("invalid mergeblanks value: %r" % mergeblanks)
    try:
        mergefuzzy = str2bool(mergefuzzy)
    except ValueError:
        raise ValueError("invalid mergefuzzy value: %r" % mergefuzzy)
    inputstore = factory.getobject(inputfile)
    if templatefile is None:
        # just merge nothing
        templatestore = type(inputstore)()
    else:
        templatestore = factory.getobject(templatefile)
    outputstore = mergestores(templatestore, inputstore, mergeblanks,
                              mergefuzzy, mergecomments)
    if outputstore.isempty():
        return 0
    outputstore.serialize(outputfile)
    return 1

 def _load_files(self, tmfiles, source_lang, target_lang):
     from translate.storage import factory
     if isinstance(tmfiles, list):
         [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
                 for tmfile in tmfiles]
     elif tmfiles:
         self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)

def test_odf2xliff2_inline():
    """Test for issue #3239."""
    reference_xlf = factory.getobject(REFERENCE_XLF_INLINE)

    odf2xliff.main(args(SOURCE_ODF_INLINE, GENERATED_XLF_TOOLKIT_INLINE))
    generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT_INLINE)
    print_diff(reference_xlf, generated_xlf_toolkit)
    assert reference_xlf == generated_xlf_toolkit

def pretranslate_file(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True):
    """Pretranslate any factory supported file with old translations and translation memory."""
    input_store = factory.getobject(input_file)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file)

    output = pretranslate_store(input_store, template_store, tm, min_similarity, fuzzymatching)
    output_file.write(str(output))
    return 1

def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=factory.classes, **kwargs):
    """Main conversion function"""

    input_store = factory.getobject(input_file, classes=classes)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file, classes=classes)
    output_store = convert_stores(input_store, template_store, tm, min_similarity, fuzzymatching, **kwargs)
    output_file.write(str(output_store))
    return 1

def memory(tmfiles, max_candidates=1, min_similarity=75, max_length=1000):
    """Returns the TM store to use. Only initialises on first call."""
    global tmmatcher
    # Only initialise first time
    if tmmatcher is None:
        if isinstance(tmfiles, list):
            tmstore = [factory.getobject(tmfile) for tmfile in tmfiles]
        else:
            tmstore = factory.getobject(tmfiles)
        tmmatcher = match.matcher(tmstore, max_candidates=max_candidates, min_similarity=min_similarity, max_length=max_length)
    return tmmatcher

def overwrite_file(request, relative_root_dir, django_file, upload_path):
    """overwrite with uploaded file"""
    upload_dir = os.path.dirname(absolute_real_path(upload_path))
    # Ensure that there is a directory into which we can dump the
    # uploaded file.
    if not os.path.exists(upload_dir):
        os.makedirs(upload_dir)

    # Get the file extensions of the uploaded filename and the
    # current translation project
    _upload_base, upload_ext = os.path.splitext(django_file.name)
    _local_base, local_ext = os.path.splitext(upload_path)
    # If the extension of the uploaded file matches the extension
    # used in this translation project, then we simply write the
    # file to the disc.
    if upload_ext == local_ext:
        outfile = open(absolute_real_path(upload_path), "wb")
        try:
            outfile.write(django_file.read())
        finally:
            outfile.close()
            try:
                #FIXME: we need a way to delay reparsing
                store = Store.objects.get(file=upload_path)
                store.update(
                    update_structure=True, update_translation=True,
                    conservative=False)
            except Store.DoesNotExist:
                # newfile, delay parsing
                pass
    else:
        newstore = factory.getobject(django_file, classes=factory_classes)
        if not newstore.units:
            return

        # If the extension of the uploaded file does not match the
        # extension of the current translation project, we create
        # an empty file (with the right extension)...
        empty_store = factory.getobject(
            absolute_real_path(upload_path), classes=factory_classes)
        # And save it...
        empty_store.save()
        request.translation_project.scan_files()
        # Then we open this newly created file and merge the
        # uploaded file into it.
        store = Store.objects.get(file=upload_path)
        #FIXME: maybe there is a faster way to do this?
        store.update(
            update_structure=True, update_translation=True,
            conservative=False, store=newstore)
        store.sync(
            update_structure=True, update_translation=True,
            conservative=False)

def test_odf2xliff():
    reference_xlf = factory.getobject(REFERENCE_XLF)

    odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_TOOLKIT))
    generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT)
    print_diff(reference_xlf, generated_xlf_toolkit)
    assert reference_xlf == generated_xlf_toolkit

    odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_ITOOLS))
    generated_xlf_itools = factory.getobject(GENERATED_XLF_ITOOLS)
    print_diff(reference_xlf, generated_xlf_itools)
    assert reference_xlf == generated_xlf_itools

def convert():
    if not Editor.isValid() or Editor.currentFile()=='': return

    xliffpathname=Editor.currentFile()
    (path, filename)=os.path.split(xliffpathname)
    if not filename.endswith('.xlf'): return

    store = factory.getobject(xliffpathname)
    odfpathname=store.getfilenames()[0]
    
    if odfpathname.startswith('NoName'):
        print 'translate-toolkit is too old'
        odfpathname=os.path.splitext(xliffpathname)[0]+'.odt'
    if not os.path.exists(odfpathname): return


    translatedodfpathname=os.path.splitext(odfpathname)[0]+'-'+Project.targetLangCode()+'.odt'
    print 'translatedodfpathname %s' % translatedodfpathname
    print 'odfpathname %s' % odfpathname
    xliffinput=XliffInput(xliffpathname,Editor.currentFileContents())
    odf=open(odfpathname,'rb')

    xliff2odf.convertxliff(xliffinput, translatedodfpathname, odf)

    ourpath=([p for p in sys.path if os.path.exists(p+'/xliff2odf.py')]+[''])[0]
    os.system('python "'+ourpath+'/xliff2odf-standalone.py" "%s" "%s" &'%(translatedodfpathname, Editor.currentEntryId()))

    def test_output(self):
        for posource in posources:
            print("PO source file")
            print(posource)
            PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()

            out_file = open(PO_FILE, 'w')
            out_file.write(posource)
            out_file.close()

            subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT])
            subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE])

            store = factory.getobject(StringIO(posource))
            if store.isempty() and not os.path.exists(MO_POCOMPILE):
                # pocompile doesn't create MO files for empty PO files, so we
                # can skip the checks here.
                continue

            mo_msgfmt_f = open(MO_MSGFMT)
            mo_pocompile_f = open(MO_POCOMPILE)

            try:
                mo_msgfmt = mo_msgfmt_f.read()
                print("msgfmt output:")
                print(repr(mo_msgfmt))
                mo_pocompile = mo_pocompile_f.read()
                print("pocompile output:")
                print(repr(mo_pocompile))
                assert mo_msgfmt == mo_pocompile
            finally:
                mo_msgfmt_f.close()
                mo_pocompile_f.close()

    def recacheunit(self, filename, checker, unit):
        """Recalculate all information for a specific unit. This is necessary
        for updating all statistics when a translation of a unit took place,
        for example.

        This method assumes that everything was up to date before (file totals,
        checks, checker config, etc."""
        fileid = self._getfileid(filename, check_mod_info=False)
        configid = self._get_config_id(fileid, checker)
        unitid = unit.getid()
        # get the unit index
        totals_without_unit = self.file_totals[fileid] - \
                                   FileTotals.new_record(*self.get_unit_stats(fileid, unitid))
        self.cur.execute("""SELECT unitindex FROM units WHERE
            fileid=? AND unitid=?;""", (fileid, unitid))
        unitindex = self.cur.fetchone()[0]
        self.cur.execute("""DELETE FROM units WHERE
            fileid=? AND unitid=?;""", (fileid, unitid))
        state = [self._cacheunitstats([unit], fileid, unitindex, totals_without_unit)]
        # remove the current errors
        self.cur.execute("""DELETE FROM uniterrors WHERE
            fileid=? AND unitindex=?;""", (fileid, unitindex))
        if os.path.exists(suggestion_filename(filename)):
            checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
        state.extend(self._cacheunitschecks([unit], fileid, configid, checker, unitindex))
        return state

    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        print "Preparing localization files"
        for po_filename in glob.glob(path.join('po', 'pootle', '*', 'pootle.po')):
            lang = path.split(path.split(po_filename)[0])[1]
            lang_dir = path.join('mo', lang, 'LC_MESSAGES')
            mo_filename = path.join(lang_dir, 'django.mo')

            try:
                store = factory.getobject(po_filename)
                gettext.c2py(store.getheaderplural()[1])
            except Exception, e:
                print "skipping %s, probably invalid header: %s" % (lang, e)

            try:
                if not path.exists(lang_dir):
                    os.makedirs(lang_dir)
                print "compiling %s language" % lang
                subprocess.Popen(['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename])
            except Exception, e:
                print "skipping %s, running msgfmt failed: %s" % (lang, e)

    def handlefile(self, filename):
        logging.info("About to process %s:" % filename)
        try:
            store = factory.getobject(filename)
            source_lang = self.source_lang or store.getsourcelanguage()
            target_lang = self.target_lang or store.gettargetlanguage()
            project_style = self.project_style or store.getprojectstyle()
            min_similarity = self.min_similarity
            max_candidates = self.max_candidates

            if not source_lang or not target_lang:
                logging.error("Missing source or target language. Can't use "
                              "%s" % filename)
                return
        except Exception:
            logging.exception("Error while processing %s" % filename)
            return

        translate_unit = current_app.tmdb.translate_unit
        try:
            for unit in store.units:
                if unit.istranslatable():
                    # We need an explicit unicode (not multistring), otherwise
                    # psycopg2 can't adapt it:
                    translate_unit(unicode(unit.source), source_lang,
                                   target_lang, project_style, min_similarity,
                                   max_candidates)
        except Exception:
            logging.exception("Error when translating unit")
            raise

    def get_units(self, filenames):
        """Gets the units to import and its total count."""
        units = []
        all_filenames = set()

        for filename in filenames:
            if not os.path.exists(filename):
                self.stdout.write("File %s doesn't exist. Skipping it." %
                                  filename)
                continue

            if os.path.isdir(filename):
                for dirpath, dirs, fnames in os.walk(filename):
                    if (os.path.basename(dirpath) in
                        ["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]):

                        continue

                    for f in fnames:
                        all_filenames.add(os.path.join(dirpath, f))
            else:
                all_filenames.add(filename)

        for filename in all_filenames:
            store = factory.getobject(filename)
            if not store.gettargetlanguage() and not self.target_language:
                raise CommandError("Unable to determine target language for "
                                   "'%s'. Try again specifying a fallback "
                                   "target language with --target-language" %
                                   filename)

            self.filename = filename
            units.extend([unit for unit in store.units if unit.istranslated()])

        return units, len(units)

    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        for lang in self._langs:
            lang = lang.rstrip()

            po_path = os.path.join('pootle', 'locale', lang)
            mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')

            if not os.path.exists(mo_path):
                os.makedirs(mo_path)

            for po, mo in (('pootle.po', 'django.mo'),
                                     ('pootle_js.po', 'djangojs.mo')):
                po_filename = os.path.join(po_path, po)
                mo_filename = os.path.join(mo_path, mo)

                try:
                    store = factory.getobject(po_filename)
                    gettext.c2py(store.getheaderplural()[1])
                except Exception, e:
                    print "WARNING: %s, has invalid plural header: %s" % (lang, e)

                try:
                    if not os.path.exists(mo_path):
                        os.makedirs(mo_path)
                    print "COMPILING: %s language" % lang
                    subprocess.Popen(['msgfmt', '--strict',
                                      '-o', mo_filename, po_filename])
                except Exception, e:
                    print "SKIPPING: %s, running msgfmt failed: %s" % (lang, e)