import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
        ee.currentTime(), mvcc), edit, true);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc)
throws IOException {
  String familyStr = Bytes.toString(family);
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes,
      ee.currentTime(), columnBytes));
    wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,999, mvcc),
        edit, true);
  }
  wal.sync();
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Utility method used to set the correct scopes on each log key. Doesn't set a scope on keys
 * from compaction WAL edits and if the scope is local.
 * @param htd Descriptor used to find the scope to use
 * @param logKey Key that may get scoped according to its edits
 * @param logEdit Edits used to lookup the scopes
 */
public static void scopeWALEdits(HTableDescriptor htd, WALKey logKey,
                                 WALEdit logEdit) {
  NavigableMap<byte[], Integer> scopes =
      new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
  byte[] family;
  for (Cell cell : logEdit.getCells()) {
    family = cell.getFamily();
    // This is expected and the KV should not be replicated
    if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue;
    // Unexpected, has a tendency to happen in unit tests
    assert htd.getFamily(family) != null;

    int scope = htd.getFamily(family).getScope();
    if (scope != REPLICATION_SCOPE_LOCAL &&
        !scopes.containsKey(family)) {
      scopes.put(family, scope);
    }
  }
  if (!scopes.isEmpty()) {
    logKey.setScopes(scopes);
  }
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Override
public void map(WALKey key, WALEdit value,
  Context context)
throws IOException {
  try {
    // skip all other tables
    if (Bytes.equals(table, key.getTablename().getName())) {
      for (Cell cell : value.getCells()) {
        KeyValue kv = KeyValueUtil.ensureKeyValueTypeForMR(cell);
        if (WALEdit.isMetaEditFamily(kv.getFamily())) continue;
        context.write(new ImmutableBytesWritable(kv.getRow()), kv);
      }
    }
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH_EXCEPTION",
    justification="Will never be null")
@Override
public long append(final HTableDescriptor htd, final HRegionInfo hri, final WALKey key,
    final WALEdit edits, final boolean inMemstore) throws IOException {
  if (this.closed) throw new IOException("Cannot append; log is closed");
  // Make a trace scope for the append.  It is closed on other side of the ring buffer by the
  // single consuming thread.  Don't have to worry about it.
  TraceScope scope = Trace.startSpan("FSHLog.append");

  // This is crazy how much it takes to make an edit.  Do we need all this stuff!!!!????  We need
  // all this to make a key and then below to append the edit, we need to carry htd, info,
  // etc. all over the ring buffer.
  FSWALEntry entry = null;
  long sequence = this.disruptor.getRingBuffer().next();
  try {
    RingBufferTruck truck = this.disruptor.getRingBuffer().get(sequence);
    // TODO: reuse FSWALEntry as we do SyncFuture rather create per append.
    entry = new FSWALEntry(sequence, key, edits, htd, hri, inMemstore);
    truck.loadPayload(entry, scope.detach());
  } finally {
    this.disruptor.getRingBuffer().publish(sequence);
  }
  return sequence;
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
FSWALEntry(final long sequence, final WALKey key, final WALEdit edit,
    final HTableDescriptor htd, final HRegionInfo hri, final boolean inMemstore) {
  super(key, edit);
  this.inMemstore = inMemstore;
  this.htd = htd;
  this.hri = hri;
  this.sequence = sequence;
  if (inMemstore) {
    // construct familyNames here to reduce the work of log sinker.
    ArrayList<Cell> cells = this.getEdit().getCells();
    if (CollectionUtils.isEmpty(cells)) {
      this.familyNames = Collections.<byte[]> emptySet();
    } else {
      Set<byte[]> familySet = Sets.newTreeSet(Bytes.BYTES_COMPARATOR);
      for (Cell cell : cells) {
        if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
          familySet.add(CellUtil.cloneFamily(cell));
        }
      }
      this.familyNames = Collections.unmodifiableSet(familySet);
    }
  } else {
    this.familyNames = Collections.<byte[]> emptySet();
  }
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Here is where a WAL edit gets its sequenceid.
 * @return The sequenceid we stamped on this edit.
 * @throws IOException
 */
long stampRegionSequenceId() throws IOException {
  long regionSequenceId = WALKey.NO_SEQUENCE_ID;
  MultiVersionConcurrencyControl mvcc = getKey().getMvcc();
  MultiVersionConcurrencyControl.WriteEntry we = null;

  if (mvcc != null) {
    we = mvcc.begin();
    regionSequenceId = we.getWriteNumber();
  }

  if (!this.getEdit().isReplay() && inMemstore) {
    for (Cell c:getEdit().getCells()) {
      CellUtil.setSequenceId(c, regionSequenceId);
    }
  }

  // This has to stay in this order
  WALKey key = getKey();
  key.setLogSeqNum(regionSequenceId);
  key.setWriteEntry(we);
  return regionSequenceId;
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
private static long writeMarker(final WAL wal, final HTableDescriptor htd, final HRegionInfo hri,
    final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync)
throws IOException {
  // TODO: Pass in current time to use?
  WALKey key =
    new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc);
  // Add it to the log but the false specifies that we don't need to add it to the memstore
  long trx = MultiVersionConcurrencyControl.NONE;
  try {
    trx = wal.append(htd, hri, key, edit, false);
    if (sync) wal.sync(trx);
  } finally {
    // If you get hung here, is it a real WAL or a mocked WAL? If the latter, you need to
    // trip the latch that is inside in getWriteEntry up in your mock. See down in the append
    // called from onEvent in FSHLog.
    MultiVersionConcurrencyControl.WriteEntry we = key.getWriteEntry();
    if (mvcc != null && we != null) mvcc.complete(we);
  }
  return trx;
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Method to safely get the next sequence number.
 *
 * @return Next sequence number unassociated with any actual edit.
 * @throws IOException
 */
@VisibleForTesting protected long getNextSequenceId(final WAL wal) throws IOException {
  // TODO: For review. Putting an empty edit in to get a sequenceid out will
  // not work if the
  // WAL is banjaxed... if it has gotten an exception and the WAL has not yet
  // been rolled or
  // aborted. In this case, we'll just get stuck here. For now, until
  // HBASE-12751, just have
  // a timeout. May happen in tests after we tightened the semantic via
  // HBASE-14317.
  // Also, the getSequenceId blocks on a latch. There is no global list of
  // outstanding latches
  // so if an abort or stop, there is no way to call them in.
  WALKey key = this.appendEmptyEdit(wal);
  mvcc.complete(key.getWriteEntry());
  return key.getSequenceId(this.maxWaitForSeqId);
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Append a faked WALEdit in order to get a long sequence number and wal syncer will just ignore
 * the WALEdit append later.
 *
 * @param wal
 * @return Return the key used appending with no sync and no append.
 * @throws IOException
 */
private WALKey appendEmptyEdit(final WAL wal) throws IOException {
  // we use HLogKey here instead of WALKey directly to support legacy
  // coprocessors.
  @SuppressWarnings("deprecation") WALKey key =
      new HLogKey(getRegionInfo().getEncodedNameAsBytes(), getRegionInfo().getTable(),
          WALKey.NO_SEQUENCE_ID, 0, null, HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());

  // Call append but with an empty WALEdit. The returned sequence id will not
  // be associated
  // with any edit and we can be sure it went in after all outstanding
  // appends.
  try {
    wal.append(getTableDesc(), getRegionInfo(), key, WALEdit.EMPTY_WALEDIT, false);
  } catch (Throwable t) {
    // If exception, our mvcc won't get cleaned up by client, so do it here.
    getMVCC().complete(key.getWriteEntry());
  }
  return key;
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Constructor
 * @param impl the coprocessor instance
 * @param priority chaining priority
 */
public RegionEnvironment(final Coprocessor impl, final int priority,
    final int seq, final Configuration conf, final Region region,
    final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
  super(impl, priority, seq, conf);
  this.region = region;
  this.rsServices = services;
  this.sharedData = sharedData;
  // Pick which version of the WAL related events we'll call.
  // This way we avoid calling the new version on older RegionObservers so
  // we can maintain binary compatibility.
  // See notes in javadoc for RegionObserver
  useLegacyPre = useLegacyMethod(impl.getClass(), "preWALRestore", ObserverContext.class,
      HRegionInfo.class, WALKey.class, WALEdit.class);
  useLegacyPost = useLegacyMethod(impl.getClass(), "postWALRestore", ObserverContext.class,
      HRegionInfo.class, WALKey.class, WALEdit.class);
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * @param info
 * @param logKey
 * @param logEdit
 * @return true if default behavior should be bypassed, false otherwise
 * @throws IOException
 */
public boolean preWALRestore(final HRegionInfo info, final WALKey logKey,
    final WALEdit logEdit) throws IOException {
  return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
    @Override
    public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
        throws IOException {
      // Once we don't need to support the legacy call, replace RegionOperation with a version
      // that's ObserverContext<RegionEnvironment> and avoid this cast.
      final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment();
      if (env.useLegacyPre) {
        if (logKey instanceof HLogKey) {
          oserver.preWALRestore(ctx, info, (HLogKey)logKey, logEdit);
        } else {
          legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey.");
        }
      } else {
        oserver.preWALRestore(ctx, info, logKey, logEdit);
      }
    }
  });
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * @param info
 * @param logKey
 * @param logEdit
 * @throws IOException
 */
public void postWALRestore(final HRegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
    @Override
    public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
        throws IOException {
      // Once we don't need to support the legacy call, replace RegionOperation with a version
      // that's ObserverContext<RegionEnvironment> and avoid this cast.
      final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment();
      if (env.useLegacyPost) {
        if (logKey instanceof HLogKey) {
          oserver.postWALRestore(ctx, info, (HLogKey)logKey, logEdit);
        } else {
          legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey.");
        }
      } else {
        oserver.postWALRestore(ctx, info, logKey, logEdit);
      }
    }
  });
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Test
public void testSystemTableWALEntryFilter() {
  SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter();

  // meta
  WALKey key1 = new WALKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
    TableName.META_TABLE_NAME);
  Entry metaEntry = new Entry(key1, null);

  assertNull(filter.filter(metaEntry));

  // ns table
  WALKey key2 = new WALKey(new byte[] {}, TableName.NAMESPACE_TABLE_NAME);
  Entry nsEntry = new Entry(key2, null);
  assertNull(filter.filter(nsEntry));

  // user table

  WALKey key3 = new WALKey(new byte[] {}, TableName.valueOf("foo"));
  Entry userEntry = new Entry(key3, null);

  assertEquals(userEntry, filter.filter(userEntry));
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
/**
 * Utility method to setup a WAL mock.
 * Needs to do the bit where we close latch on the WALKey on append else test hangs.
 * @return
 * @throws IOException
 */
private WAL mockWAL() throws IOException {
  WAL wal = mock(WAL.class);
  Mockito.when(wal.append((HTableDescriptor)Mockito.any(), (HRegionInfo)Mockito.any(),
      (WALKey)Mockito.any(), (WALEdit)Mockito.any(), Mockito.anyBoolean())).
    thenAnswer(new Answer<Long>() {
      @Override
      public Long answer(InvocationOnMock invocation) throws Throwable {
        WALKey key = invocation.getArgumentAt(2, WALKey.class);
        MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
        key.setWriteEntry(we);
        return 1L;
      }

  });
  return wal;
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
protected void addEdits(WAL log,
                        HRegionInfo hri,
                        HTableDescriptor htd,
                        int times,
                        MultiVersionConcurrencyControl mvcc)
    throws IOException {
  final byte[] row = Bytes.toBytes("row");
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKey key = new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(),
        WALKey.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc);
    log.append(htd, hri, key, cols, true);
  }
  log.sync();
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Test
public void shouldBulkLoadSingleFamilyHLog() throws IOException {
  when(log.append(any(HTableDescriptor.class), any(HRegionInfo.class),
          any(WALKey.class), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
          any(boolean.class))).thenAnswer(new Answer() {
    public Object answer(InvocationOnMock invocation) {
      WALKey walKey = invocation.getArgumentAt(2, WALKey.class);
      MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
      if (mvcc != null) {
        MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
        walKey.setWriteEntry(we);
      }
      return 01L;
    };
  });
  testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
  verify(log).sync(anyLong());
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Test
public void shouldBulkLoadManyFamilyHLog() throws IOException {
  when(log.append(any(HTableDescriptor.class), any(HRegionInfo.class),
          any(WALKey.class), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
          any(boolean.class))).thenAnswer(new Answer() {
            public Object answer(InvocationOnMock invocation) {
              WALKey walKey = invocation.getArgumentAt(2, WALKey.class);
              MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
              if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
              }
              return 01L;
            };
          });
  testRegionWithFamilies(family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2),
          false, null);
  verify(log).sync(anyLong());
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Test
public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
  when(log.append(any(HTableDescriptor.class), any(HRegionInfo.class),
          any(WALKey.class), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
          any(boolean.class))).thenAnswer(new Answer() {
    public Object answer(InvocationOnMock invocation) {
      WALKey walKey = invocation.getArgumentAt(2, WALKey.class);
      MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
      if (mvcc != null) {
        MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
        walKey.setWriteEntry(we);
      }
      return 01L;
    };
  });
  TableName tableName = TableName.valueOf("test", "test");
  testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2)
      .bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
  verify(log).sync(anyLong());
}
 

import org.apache.hadoop.hbase.wal.WALKey; //导入依赖的package包/类
@Override
public void map(WALKey key, WALEdit value,
  Context context)
throws IOException {
  try {
    // skip all other tables
    if (Bytes.equals(table, key.getTablename().getName())) {
      for (Cell cell : value.getCells()) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (WALEdit.isMetaEditFamily(kv.getFamily())) continue;
        context.write(new ImmutableBytesWritable(kv.getRow()), kv);
      }
    }
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}