import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/**
 * Returns a list of pointers to a GZIP archive entries positions (including the end of file).
 *
 * @param in the stream from which to read the GZIP archive.
 * @param pl a progress logger.
 * @return a list of longs where the <em>i</em>-th long is the offset in the stream of the first byte of the <em>i</em>-th archive entry.
 */
public static LongBigArrayBigList index(final InputStream in, final ProgressLogger pl) throws IOException {
	final LongBigArrayBigList pointers = new LongBigArrayBigList();
	long current = 0;
	final GZIPArchiveReader gzar = new GZIPArchiveReader(in);
	GZIPArchive.ReadEntry re;
	for (;;) {
		re = gzar.skipEntry();
		if (re == null) break;
		pointers.add(current);
		current += re.compressedSkipLength;
		if (pl != null) pl.lightUpdate();
	}
	in.close();
	return pointers;
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
@Test
   public void reverseGetEntry() throws IOException {
final LongBigArrayBigList pos = GZIPIndexer.index(new FileInputStream(ARCHIVE_PATH));
GZIPArchive.ReadEntry re;
FastBufferedInputStream fis = new FastBufferedInputStream(new FileInputStream(ARCHIVE_PATH));
GZIPArchiveReader gzar = new GZIPArchiveReader(fis);
byte[] actualMagic = new byte[EXPECTED_MAGIC.length];
for (int i = (int) pos.size64() - 1; i >= 0; i--) {
    gzar.position(pos.getLong(i));
    re = gzar.getEntry();
    if (re == null) break;
    LazyInflater lin = re.lazyInflater;
    InputStream in = lin.get();
    in.read(actualMagic);
    assertArrayEquals(EXPECTED_MAGIC, actualMagic);
    for (int j = 0; j < (i + 1) * 512; j++) in.read();
    lin.consume();

}
fis.close();
   }
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/**
 * Initializes internal data structures for transitions.
 */
private void initTransitionsMaps() {
	if (trId2bpnMap == null) {
		trId2bpnMap = new Object2LongOpenHashMap<String>();
		trId2bpnMap.defaultReturnValue(-1L);
	}
	if (tr2InPlacesMap == null) {
		tr2InPlacesMap = new Long2ObjectOpenHashMap<>();
		tr2InPlacesMap.defaultReturnValue(null);
	}
	if (tr2OutPlacesMap == null) {
		tr2OutPlacesMap = new Long2ObjectOpenHashMap<>();
		tr2OutPlacesMap.defaultReturnValue(null);
	}
	if (tr2InAllArcsMap == null) {
		tr2InAllArcsMap = new Object2ObjectOpenHashMap<String, LongBigArrayBigList>();
		tr2InAllArcsMap.defaultReturnValue(null);
	}
	if (tr2OutAllArcsMap == null) {
		tr2OutAllArcsMap = new Object2ObjectOpenHashMap<String, LongBigArrayBigList>();
		tr2OutAllArcsMap.defaultReturnValue(null);
	}
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/**
 * @param bpnsb
 * @param trId
 */
private void buildConnectedPlaces2Transition(StringBuilder bpnsb, long trId,
		Long2ObjectOpenHashMap<LongBigArrayBigList> tr2PlacesMap) {
	LongBigArrayBigList pls;
	long plsSize;
	pls = tr2PlacesMap.get(trId);
	if (pls != null) {
		plsSize = pls.size64();
	} else { // no place in input or output list of this transition
		plsSize = 0L;
	}
	bpnsb.append(WS).append(HK).append(plsSize);

	if (plsSize > 0L) {
		for (long plId : pls) {
			bpnsb.append(WS).append(plId);
		}
	}
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/** Commodity method for loading a big list of binary longs with specified endianness into a
 * {@linkplain LongBigArrays long big array}.
 * 
 * @param filename the file containing the longs.
 * @param byteOrder the endianness of the longs.
 * @return a big list of longs containing the longs in <code>file</code>. */
public static LongBigArrayBigList loadLongBigList( final CharSequence filename, final ByteOrder byteOrder ) throws IOException {
	final long length = new File( filename.toString() ).length() / ( Long.SIZE / Byte.SIZE );
	@SuppressWarnings("resource")
	final ReadableByteChannel channel = new FileInputStream( filename.toString() ).getChannel();
	final ByteBuffer byteBuffer = ByteBuffer.allocateDirect( 64 * 1024 ).order( byteOrder );
	final LongBigArrayBigList list = new LongBigArrayBigList( length );

	while ( channel.read( byteBuffer ) > 0 ) {
		byteBuffer.flip();
		while ( byteBuffer.hasRemaining() )
			list.add( byteBuffer.getLong() );
		byteBuffer.clear();
	}

	channel.close();
	return list;

}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/**
 * Create a StrippedPartition with only one equivalence class with the definied number of elements. <br/>
 * Tuple ids start with 0 to numberOfElements-1
 *
 * @param numberTuples
 */
public StrippedPartition(long numberTuples) {
    this.strippedPartition = new ObjectBigArrayBigList<LongBigArrayBigList>();
    this.elementCount = numberTuples;
    // StrippedPartition only contains partition with more than one elements.
    if (numberTuples > 1) {
        LongBigArrayBigList newEqClass = new LongBigArrayBigList();
        for (int i = 0; i < numberTuples; i++) {
            newEqClass.add(i);
        }
        this.strippedPartition.add(newEqClass);
    }
    this.calculateError();
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
@Test
public void testRandom() {
	// Weird skips
	final LongBigArrayBigList l = new LongBigArrayBigList();
	final XoRoShiRo128PlusRandom random = new XoRoShiRo128PlusRandom(0);
	for(long i = 10000000, c = 0; i-- != 0;) {
		c += Long.numberOfTrailingZeros(random.nextLong());
		l.add(c);
	}
	assertEquals(l, new EliasFanoMonotoneLongBigList(l));
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
/**
 * Initialises the data structure for unsafe transitions.
 */
private void initUnsafeTransMaps() {
	if (tr2InUnsafeArcsMap == null) {
		tr2InUnsafeArcsMap = new Object2ObjectOpenHashMap<String, LongBigArrayBigList>();
		tr2InUnsafeArcsMap.defaultReturnValue(null);
	}
	if (tr2OutUnsafeArcsMap == null) {
		tr2OutUnsafeArcsMap = new Object2ObjectOpenHashMap<String, LongBigArrayBigList>();
		tr2OutUnsafeArcsMap.defaultReturnValue(null);
	}

}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
public ObjectBigArrayBigList<LongBigArrayBigList> getStrippedPartition() {
    return this.strippedPartition;
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
public void empty() {
    this.strippedPartition = new ObjectBigArrayBigList<LongBigArrayBigList>();
    this.elementCount = 0;
    this.error = 0.0;
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
@Override
public void execute() throws AlgorithmExecutionException {

    level0 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    level1 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    prefix_blocks = new Object2ObjectOpenHashMap<OpenBitSet, ObjectArrayList<OpenBitSet>>();

    // Get information about table from database
    ObjectArrayList<Object2ObjectOpenHashMap<Object, LongBigArrayBigList>> partitions = loadData();
    setColumnIdentifiers();
    numberAttributes = this.columnNames.size();

    // Initialize table used for stripped partition product
    tTable = new LongBigArrayBigList(numberTuples);
    for (long i = 0; i < numberTuples; i++) {
        tTable.add(-1);
    }
    // Initialize FDTree to store fds and filter them before returning them.
    dependencies = new FDTree(numberAttributes);

    // Initialize Level 0
    CombinationHelper chLevel0 = new CombinationHelper();
    OpenBitSet rhsCandidatesLevel0 = new OpenBitSet();
    rhsCandidatesLevel0.set(1, numberAttributes + 1);
    chLevel0.setRhsCandidates(rhsCandidatesLevel0);
    StrippedPartition spLevel0 = new StrippedPartition(numberTuples);
    chLevel0.setPartition(spLevel0);
    spLevel0 = null;
    level0.put(new OpenBitSet(), chLevel0);
    chLevel0 = null;


    // Initialize Level 1
    for (int i = 1; i <= numberAttributes; i++) {
        OpenBitSet combinationLevel1 = new OpenBitSet();
        combinationLevel1.set(i);

        CombinationHelper chLevel1 = new CombinationHelper();
        OpenBitSet rhsCandidatesLevel1 = new OpenBitSet();
        rhsCandidatesLevel1.set(1, numberAttributes + 1);
        chLevel1.setRhsCandidates(rhsCandidatesLevel0);

        StrippedPartition spLevel1 = new StrippedPartition(partitions.get(i - 1));
        chLevel1.setPartition(spLevel1);

        level1.put(combinationLevel1, chLevel1);
    }
    partitions = null;

    // while loop (main part of TANE)
    int l = 1;
    while (!level1.isEmpty() && l <= numberAttributes) {
        // compute dependencies for a level
        computeDependencies(l);

        // prune the search space
        prune();

        // compute the combinations for the next level
        generateNextLevel();

        l++;
    }
    dependencies.filterGeneralizations();
    addAllDependenciesToResultReceiver();
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
@Override
public void execute() throws AlgorithmExecutionException {

    level0 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    level1 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    prefix_blocks = new Object2ObjectOpenHashMap<OpenBitSet, ObjectArrayList<OpenBitSet>>();

    // Get information about table from database or csv file
    ObjectArrayList<Object2ObjectOpenHashMap<Object, LongBigArrayBigList>> partitions = loadData();
    setColumnIdentifiers();
    numberAttributes = this.columnNames.size();

    // Initialize table used for stripped partition product
    tTable = new LongBigArrayBigList(numberTuples);
    for (long i = 0; i < numberTuples; i++) {
        tTable.add(-1);
    }

    // Initialize Level 0
    CombinationHelper chLevel0 = new CombinationHelper();
    OpenBitSet rhsCandidatesLevel0 = new OpenBitSet();
    rhsCandidatesLevel0.set(1, numberAttributes + 1);
    chLevel0.setRhsCandidates(rhsCandidatesLevel0);
    StrippedPartition spLevel0 = new StrippedPartition(numberTuples);
    chLevel0.setPartition(spLevel0);
    spLevel0 = null;
    level0.put(new OpenBitSet(), chLevel0);
    chLevel0 = null;


    // Initialize Level 1
    for (int i = 1; i <= numberAttributes; i++) {
        OpenBitSet combinationLevel1 = new OpenBitSet();
        combinationLevel1.set(i);

        CombinationHelper chLevel1 = new CombinationHelper();
        OpenBitSet rhsCandidatesLevel1 = new OpenBitSet();
        rhsCandidatesLevel1.set(1, numberAttributes + 1);
        chLevel1.setRhsCandidates(rhsCandidatesLevel0);

        StrippedPartition spLevel1 = new StrippedPartition(partitions.get(i - 1));
        chLevel1.setPartition(spLevel1);

        level1.put(combinationLevel1, chLevel1);
    }
    partitions = null;

    // while loop (main part of TANE)
    int l = 1;
    while (!level1.isEmpty() && l <= numberAttributes) {
        // compute dependencies for a level
        computeDependencies();

        // prune the search space
        prune();

        // compute the combinations for the next level
        generateNextLevel();
        l++;
    }
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
@Override
public void execute() throws AlgorithmExecutionException {

    level0 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    level1 = new Object2ObjectOpenHashMap<OpenBitSet, CombinationHelper>();
    prefix_blocks = new Object2ObjectOpenHashMap<OpenBitSet, ObjectArrayList<OpenBitSet>>();

    // Get information about table from database
    ObjectArrayList<Object2ObjectOpenHashMap<Object, LongBigArrayBigList>> partitions = loadData();
    setColumnIdentifiers();
    numberAttributes = this.columnNames.size();

    // Initialize table used for stripped partition product
    tTable = new LongBigArrayBigList(numberTuples);
    for (long i = 0; i < numberTuples; i++) {
        tTable.add(-1);
    }
    // Initialize FDTree to store fds and filter them before returning them.
    dependencies = new FDTree(numberAttributes);

    // Initialize Level 0
    CombinationHelper chLevel0 = new CombinationHelper();
    OpenBitSet rhsCandidatesLevel0 = new OpenBitSet();
    rhsCandidatesLevel0.set(1, numberAttributes + 1);
    chLevel0.setRhsCandidates(rhsCandidatesLevel0);
    StrippedPartition spLevel0 = new StrippedPartition(numberTuples);
    chLevel0.setPartition(spLevel0);
    spLevel0 = null;
    level0.put(new OpenBitSet(), chLevel0);
    chLevel0 = null;


    // Initialize Level 1
    for (int i = 1; i <= numberAttributes; i++) {
        OpenBitSet combinationLevel1 = new OpenBitSet();
        combinationLevel1.set(i);

        CombinationHelper chLevel1 = new CombinationHelper();
        OpenBitSet rhsCandidatesLevel1 = new OpenBitSet();
        rhsCandidatesLevel1.set(1, numberAttributes + 1);
        chLevel1.setRhsCandidates(rhsCandidatesLevel0);

        StrippedPartition spLevel1 = new StrippedPartition(partitions.get(i - 1));
        chLevel1.setPartition(spLevel1);

        level1.put(combinationLevel1, chLevel1);
    }
    partitions = null;

    // while loop (main part of TANE)
    int l = 1;
    while (!level1.isEmpty() && l <= numberAttributes) {
        // compute dependencies for a level
        computeDependencies(l);

        // prune the search space
        prune();

        // compute the combinations for the next level
        generateNextLevel();

        l++;
    }
}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
public static void main(final String[] arg) throws NoSuchMethodException, IOException, JSAPException {

		final SimpleJSAP jsap = new SimpleJSAP(TwoStepsGOV3Function.class.getName(), "Builds a two-steps GOV3 function mapping a newline-separated list of strings to their ordinal position, or to specific values.",
				new Parameter[] {
			new FlaggedOption("encoding", ForNameStringParser.getParser(Charset.class), "UTF-8", JSAP.NOT_REQUIRED, 'e', "encoding", "The string file encoding."),
			new FlaggedOption("tempDir", FileStringParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "temp-dir", "A directory for temporary files."),
			new Switch("iso", 'i', "iso", "Use ISO-8859-1 coding internally (i.e., just use the lower eight bits of each character)."),
			new Switch("utf32", JSAP.NO_SHORTFLAG, "utf-32", "Use UTF-32 internally (handles surrogate pairs)."),
			new Switch("zipped", 'z', "zipped", "The string list is compressed in gzip format."),
			new FlaggedOption("values", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'v', "values", "A binary file in DataInput format containing a long for each string (otherwise, the values will be the ordinal positions of the strings)."),
			new UnflaggedOption("function", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The filename for the serialised two-steps GOV3 function."),
			new UnflaggedOption("stringFile", JSAP.STRING_PARSER, "-", JSAP.NOT_REQUIRED, JSAP.NOT_GREEDY, "The name of a file containing a newline-separated list of strings, or - for standard input; in the first case, strings will not be loaded into core memory."),
		});

		final JSAPResult jsapResult = jsap.parse(arg);
		if (jsap.messagePrinted()) return;

		final String functionName = jsapResult.getString("function");
		final String stringFile = jsapResult.getString("stringFile");
		final Charset encoding = (Charset)jsapResult.getObject("encoding");
		final File tempDir = jsapResult.getFile("tempDir");
		final boolean zipped = jsapResult.getBoolean("zipped");
		final boolean iso = jsapResult.getBoolean("iso");
		final boolean utf32 = jsapResult.getBoolean("utf32");

		final Collection<MutableString> collection;
		if ("-".equals(stringFile)) {
			final ProgressLogger pl = new ProgressLogger(LOGGER);
			pl.displayLocalSpeed = true;
			pl.displayFreeMemory = true;
			pl.start("Loading strings...");
			collection = new LineIterator(new FastBufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(System.in) : System.in, encoding)), pl).allLines();
			pl.done();
		}
		else collection = new FileLinesCollection(stringFile, encoding.toString(), zipped);
		final TransformationStrategy<CharSequence> transformationStrategy = iso
				? TransformationStrategies.rawIso()
				: utf32
					? TransformationStrategies.rawUtf32()
					: TransformationStrategies.rawUtf16();

		BinIO.storeObject(new TwoStepsGOV3Function<CharSequence>(collection, transformationStrategy, LongBigArrayBigList.wrap(BinIO.loadLongsBig(jsapResult.getString("values"))), tempDir, null), functionName);
		LOGGER.info("Completed.");
	}
 

import it.unimi.dsi.fastutil.longs.LongBigArrayBigList; //导入依赖的package包/类
public static void main(final String[] arg) throws NoSuchMethodException, IOException, JSAPException {

		final SimpleJSAP jsap = new SimpleJSAP(TwoStepsMWHCFunction.class.getName(), "Builds a two-steps MWHC function mapping a newline-separated list of strings to their ordinal position, or to specific values.",
				new Parameter[] {
			new FlaggedOption("encoding", ForNameStringParser.getParser(Charset.class), "UTF-8", JSAP.NOT_REQUIRED, 'e', "encoding", "The string file encoding."),
			new FlaggedOption("tempDir", FileStringParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "temp-dir", "A directory for temporary files."),
			new Switch("iso", 'i', "iso", "Use ISO-8859-1 coding internally (i.e., just use the lower eight bits of each character)."),
			new Switch("utf32", JSAP.NO_SHORTFLAG, "utf-32", "Use UTF-32 internally (handles surrogate pairs)."),
			new Switch("zipped", 'z', "zipped", "The string list is compressed in gzip format."),
			new FlaggedOption("values", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'v', "values", "A binary file in DataInput format containing a long for each string (otherwise, the values will be the ordinal positions of the strings)."),
			new UnflaggedOption("function", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The filename for the serialised MWHC function."),
			new UnflaggedOption("stringFile", JSAP.STRING_PARSER, "-", JSAP.NOT_REQUIRED, JSAP.NOT_GREEDY, "The name of a file containing a newline-separated list of strings, or - for standard input; in the first case, strings will not be loaded into core memory."),
		});

		JSAPResult jsapResult = jsap.parse(arg);
		if (jsap.messagePrinted()) return;

		LOGGER.warn("This class is deprecated: please use TwoStepsGOV3Function");

		final String functionName = jsapResult.getString("function");
		final String stringFile = jsapResult.getString("stringFile");
		final Charset encoding = (Charset)jsapResult.getObject("encoding");
		final File tempDir = jsapResult.getFile("tempDir");
		final boolean zipped = jsapResult.getBoolean("zipped");
		final boolean iso = jsapResult.getBoolean("iso");
		final boolean utf32 = jsapResult.getBoolean("utf32");

		final Collection<MutableString> collection;
		if ("-".equals(stringFile)) {
			final ProgressLogger pl = new ProgressLogger(LOGGER);
			pl.displayLocalSpeed = true;
			pl.displayFreeMemory = true;
			pl.start("Loading strings...");
			collection = new LineIterator(new FastBufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(System.in) : System.in, encoding)), pl).allLines();
			pl.done();
		}
		else collection = new FileLinesCollection(stringFile, encoding.toString(), zipped);
		final TransformationStrategy<CharSequence> transformationStrategy = iso
				? TransformationStrategies.rawIso()
				: utf32
					? TransformationStrategies.rawUtf32()
					: TransformationStrategies.rawUtf16();

		BinIO.storeObject(new TwoStepsMWHCFunction<CharSequence>(collection, transformationStrategy, LongBigArrayBigList.wrap(BinIO.loadLongsBig(jsapResult.getString("values"))), tempDir, null), functionName);
		LOGGER.info("Completed.");
	}