import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Take a snapshot using the specified handler.
 * On failure the snapshot temporary working directory is removed.
 * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
 *       snapshot request if the table is busy with another snapshot/restore operation.
 * @param snapshot the snapshot description
 * @param handler the snapshot handler
 */
private synchronized void snapshotTable(SnapshotDescription snapshot,
    final TakeSnapshotHandler handler) throws HBaseSnapshotException {
  try {
    handler.prepare();
    this.executorService.submit(handler);
    this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
  } catch (Exception e) {
    // cleanup the working directory by trying to delete it from the fs.
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    try {
      if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
            ClientSnapshotDescriptionUtils.toString(snapshot));
      }
    } catch (IOException e1) {
      LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
          ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // fail the snapshot
    throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Migrates all snapshots, user tables and system tables that require migration.
 * First migrates snapshots.
 * Then migrates each user table in order,
 * then attempts ROOT (should be gone)
 * Migrates hbase:meta last to indicate migration is complete.
 */
private static void migrateFsTableDescriptors(FileSystem fs, Path rootDir) throws IOException {
  // First migrate snapshots - will migrate any snapshot dir that contains a table info file
  Path snapshotsDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  if (fs.exists(snapshotsDir)) {
    LOG.info("Migrating snapshots");
    FileStatus[] snapshots = fs.listStatus(snapshotsDir,
        new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
    for (FileStatus snapshot : snapshots) {
      migrateTable(fs, snapshot.getPath());
    }
  }
  
  LOG.info("Migrating user tables");
  List<Path> userTableDirs = FSUtils.getTableDirs(fs, rootDir);
  for (Path userTableDir : userTableDirs) {
    migrateTable(fs, userTableDir);
  }
  
  LOG.info("Migrating system tables");
  // migrate meta last because that's what we check to see if migration is complete
  migrateTableIfExists(fs, rootDir, TableName.META_TABLE_NAME);
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Test(timeout = 300000)
public void testDeleteSnapshot() throws Exception {

  String snapshotName = "completed";
  SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();

  DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();
  try {
    master.getMasterRpcServices().deleteSnapshot(null, request);
    fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
  } catch (ServiceException e) {
    LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
  }

  // write one snapshot to the fs
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);

  // then delete the existing snapshot,which shouldn't cause an exception to be thrown
  master.getMasterRpcServices().deleteSnapshot(null, request);
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Take a snapshot using the specified handler.
 * On failure the snapshot temporary working directory is removed.
 * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
 *       snapshot request if the table is busy with another snapshot/restore operation.
 * @param snapshot the snapshot description
 * @param handler the snapshot handler
 */
private synchronized void snapshotTable(SnapshotDescription snapshot,
    final TakeSnapshotHandler handler) throws HBaseSnapshotException {
  try {
    handler.prepare();
    this.executorService.submit(handler);
    this.snapshotHandlers.put(snapshot.getTable(), handler);
  } catch (Exception e) {
    // cleanup the working directory by trying to delete it from the fs.
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    try {
      if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
            SnapshotDescriptionUtils.toString(snapshot));
      }
    } catch (IOException e1) {
      LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
          SnapshotDescriptionUtils.toString(snapshot));
    }
    // fail the snapshot
    throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Test
public void testDeleteSnapshot() throws Exception {

  String snapshotName = "completed";
  SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();

  try {
    master.deleteSnapshot(new HSnapshotDescription(snapshot));
    fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
  } catch (IOException e) {
    LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
  }

  // write one snapshot to the fs
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);

  // then delete the existing snapshot,which shouldn't cause an exception to be thrown
  master.deleteSnapshot(new HSnapshotDescription(snapshot));
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Check that we don't get an exception if there is no recovered edits directory to copy
 * @throws Exception on failure
 */
@Test
public void testNoEditsDir() throws Exception {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
  ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
  FileSystem fs = UTIL.getTestFileSystem();
  Path root = UTIL.getDataTestDir();
  String regionName = "regionA";
  Path regionDir = new Path(root, regionName);
  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
  try {
    // doesn't really matter where the region's snapshot directory is, but this is pretty close
    Path snapshotRegionDir = new Path(workingDir, regionName);
    fs.mkdirs(snapshotRegionDir);
    Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionDir);
    assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits));

    CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
        snapshotRegionDir);
    task.call();
  } finally {
    // cleanup the working directory
    FSUtils.delete(fs, regionDir, true);
    FSUtils.delete(fs, workingDir, true);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Test(timeout = 300000)
public void testDeleteSnapshot() throws Exception {

  String snapshotName = "completed";
  SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();

  DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();
  try {
    master.deleteSnapshot(null, request);
    fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
  } catch (ServiceException e) {
    LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
  }

  // write one snapshot to the fs
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);

  // then delete the existing snapshot,which shouldn't cause an exception to be thrown
  master.deleteSnapshot(null, request);
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Test
public void testSnapshotTempDirReload() throws IOException {
  long period = Long.MAX_VALUE;
  // This doesn't refresh cache until we invoke it explicitly
  Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
      SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
  SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());

  // Add a new snapshot
  Path snapshot1 = new Path(snapshotDir, "snapshot1");
  Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
  fs.createNewFile(file1);
  assertTrue(cache.contains(file1.getName()));

  // Add another snapshot
  Path snapshot2 = new Path(snapshotDir, "snapshot2");
  Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
  fs.createNewFile(file2);
  assertTrue(cache.contains(file2.getName()));
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Check that we don't get an exception if there is no recovered edits directory to copy
 * @throws Exception on failure
 */
@Test
public void testNoEditsDir() throws Exception {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
  ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
  FileSystem fs = UTIL.getTestFileSystem();
  Path root = UTIL.getDataTestDir();
  String regionName = "regionA";
  Path regionDir = new Path(root, regionName);
  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
  try {
    // doesn't really matter where the region's snapshot directory is, but this is pretty close
    Path snapshotRegionDir = new Path(workingDir, regionName);
    fs.mkdirs(snapshotRegionDir);
    Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
    assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits));

    CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
        snapshotRegionDir);
    task.call();
  } finally {
    // cleanup the working directory
    FSUtils.delete(fs, regionDir, true);
    FSUtils.delete(fs, workingDir, true);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Get table descriptor
 * @param tableName is the table backed up
 * @return {@link TableDescriptor} saved in backup image of the table
 */
TableDescriptor getTableDesc(TableName tableName) throws IOException {
  Path tableInfoPath = this.getTableInfoPath(tableName);
  SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
  TableDescriptor tableDescriptor = manifest.getTableDescriptor();
  if (!tableDescriptor.getTableName().equals(tableName)) {
    LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
            + tableInfoPath.toString());
    LOG.error("tableDescriptor.getNameAsString() = "
            + tableDescriptor.getTableName().getNameAsString());
    throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
        + " under tableInfoPath: " + tableInfoPath.toString());
  }
  return tableDescriptor;
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> HRegionInfo.convert(r.getRegionInfo()))
      .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Action before cloning from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 * @throws InterruptedException
 */
private void preCloneSnapshot(final MasterProcedureEnv env)
    throws IOException, InterruptedException {
  if (!getTableName().isSystemTable()) {
    // Check and update namespace quota
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();

    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);

    ProcedureSyncWait.getMasterQuotaManager(env)
      .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
  }

  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preCreateTableAction(tableDescriptor, null, getUser());
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Take a snapshot using the specified handler.
 * On failure the snapshot temporary working directory is removed.
 * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
 *       snapshot request if the table is busy with another snapshot/restore operation.
 * @param snapshot the snapshot description
 * @param handler the snapshot handler
 */
private synchronized void snapshotTable(SnapshotDescription snapshot,
    final TakeSnapshotHandler handler) throws HBaseSnapshotException {
  try {
    handler.prepare();
    this.executorService.submit(handler);
    this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
  } catch (Exception e) {
    // cleanup the working directory by trying to delete it from the fs.
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    try {
      if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
            ClientSnapshotDescriptionUtils.toString(snapshot));
      }
    } catch (IOException e1) {
      LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
          ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // fail the snapshot
    throw new SnapshotCreationException("Could not build snapshot handler", e,
      ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedRegionManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  builder.corruptOneRegionManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress(null);
  } catch (CorruptedSnapshotException cse) {
    LOG.info("Expected exception " + cse);
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedDataManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  // consolidate to generate a data.manifest file
  builder.consolidate();
  builder.corruptDataManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress(null);
  } catch (CorruptedSnapshotException cse) {
    LOG.info("Expected exception " + cse);
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) {
    // list it, if user is the owner of snapshot
  } else {
    requirePermission("listSnapshot", Action.ADMIN);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Override
public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
    throws IOException {
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) {
    requirePermission("restoreSnapshot", hTableDescriptor.getTableName(), null, null,
      Permission.Action.ADMIN);
  } else {
    requirePermission("restore", Action.ADMIN);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, getActiveUser())) {
    // Snapshot owner is allowed to delete the snapshot
    // TODO: We are not logging this for audit
  } else {
    requirePermission("deleteSnapshot", Action.ADMIN);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
public void migrateSnapshots() throws IOException {
  //migrate snapshot dir
  Path oldSnapshotDir = new Path(rootDir, HConstants.OLD_SNAPSHOT_DIR_NAME);
  Path newSnapshotDir = new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
  if (fs.exists(oldSnapshotDir)) {
    boolean foundOldSnapshotDir = false;
    // Logic to verify old snapshot dir culled from SnapshotManager
    // ignore all the snapshots in progress
    FileStatus[] snapshots = fs.listStatus(oldSnapshotDir,
      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
    // loop through all the completed snapshots
    for (FileStatus snapshot : snapshots) {
      Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
      // if the snapshot is bad
      if (fs.exists(info)) {
        foundOldSnapshotDir = true;
        break;
      }
    }
    if(foundOldSnapshotDir) {
      LOG.info("Migrating snapshot dir");
      if (!fs.rename(oldSnapshotDir, newSnapshotDir)) {
        throw new IOException("Failed to move old snapshot dir "+
            oldSnapshotDir+" to new "+newSnapshotDir);
      }
    }
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; //导入依赖的package包/类
/**
 * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
 * filesystem
 * @param fs {@link FileSystem} where the snapshots are stored
 * @param rootDir hbase root directory
 * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
 * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
 * @param refreshThreadName name of the cache refresh thread
 * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
 */
public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
    long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
  this.fs = fs;
  this.fileInspector = inspectSnapshotFiles;
  this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  // periodically refresh the file cache to make sure we aren't superfluously saving files.
  this.refreshTimer = new Timer(refreshThreadName, true);
  this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
    cacheRefreshPeriod);
}