• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

Java CommitCtx类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Java中org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx的典型用法代码示例。如果您正苦于以下问题:Java CommitCtx类的具体用法?Java CommitCtx怎么用?Java CommitCtx使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



CommitCtx类属于org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx包,在下文中一共展示了CommitCtx类的11个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。

示例1: testEviction

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
public void testEviction() throws IOException, InterruptedException {
  NfsConfiguration conf = new NfsConfiguration();

  // Only two entries will be in the cache
  conf.setInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 2);

  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new ShellBasedIdMapping(new NfsConfiguration()));
  OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new ShellBasedIdMapping(new NfsConfiguration()));
  OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new ShellBasedIdMapping(new NfsConfiguration()));
  OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new ShellBasedIdMapping(new NfsConfiguration()));
  OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new ShellBasedIdMapping(new NfsConfiguration()));

  OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);

  boolean ret = cache.put(new FileHandle(1), context1);
  assertTrue(ret);
  Thread.sleep(1000);
  ret = cache.put(new FileHandle(2), context2);
  assertTrue(ret);
  ret = cache.put(new FileHandle(3), context3);
  assertFalse(ret);
  assertTrue(cache.size() == 2);

  // Wait for the oldest stream to be evict-able, insert again
  Thread.sleep(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
  assertTrue(cache.size() == 2);

  ret = cache.put(new FileHandle(3), context3);
  assertTrue(ret);
  assertTrue(cache.size() == 2);
  assertTrue(cache.get(new FileHandle(1)) == null);

  // Test inactive entry is evicted immediately
  context3.setActiveStatusForTest(false);
  ret = cache.put(new FileHandle(4), context4);
  assertTrue(ret);

  // Now the cache has context2 and context4
  // Test eviction failure if all entries have pending work.
  context2.getPendingWritesForTest().put(new OffsetRange(0, 100),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  context4.getPendingCommitsForTest().put(new Long(100),
      new CommitCtx(0, null, 0, attr));
  Thread.sleep(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
  ret = cache.put(new FileHandle(5), context5);
  assertFalse(ret);
}
 
开发者ID:naver,项目名称:hadoop,代码行数:59,代码来源:TestOpenFileCtxCache.java


示例2: testCheckCommit

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommit() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  NfsConfiguration conf = new NfsConfiguration();
  conf.setBoolean(NfsConfigKeys.LARGE_FILE_UPLOAD, false);
  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new ShellBasedIdMapping(conf), false, conf);

  COMMIT_STATUS ret;

  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_CTX);

  ctx.getPendingWritesForTest().put(new OffsetRange(5, 10),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE);

  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  ctx.setNextOffsetForTest(10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  
  status = ctx.checkCommitInternal(10, ch, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);

  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  Assert.assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  long key = commits.firstKey();
  Assert.assertTrue(key == 11);

  // Test request with zero commit offset
  commits.remove(new Long(11));
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  key = commits.firstKey();
  Assert.assertTrue(key == 9);

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(5, 10));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}
 
开发者ID:naver,项目名称:hadoop,代码行数:67,代码来源:TestWrites.java


示例3: testCheckCommitLargeFileUpload

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with
// large file upload option.
public void testCheckCommitLargeFileUpload() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  NfsConfiguration conf = new NfsConfiguration();
  conf.setBoolean(NfsConfigKeys.LARGE_FILE_UPLOAD, true);
  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new ShellBasedIdMapping(conf), false, conf);

  COMMIT_STATUS ret;

  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_CTX);

  ctx.getPendingWritesForTest().put(new OffsetRange(10, 15),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE);

  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 8);
  ctx.setNextOffsetForTest(10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  
  // Test commit sequential writes
  status = ctx.checkCommitInternal(10, ch, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);

  // Test commit non-sequential writes
  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  Assert.assertTrue(commits.size() == 1);
  ret = ctx.checkCommit(dfsClient, 16, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_SPECIAL_SUCCESS);
  Assert.assertTrue(commits.size() == 1);
  
  // Test request with zero commit offset
  commits.remove(new Long(10));
  // There is one pending write [10,15]
  ret = ctx.checkCommitInternal(0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);
  
  ret = ctx.checkCommitInternal(9, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);
  Assert.assertTrue(commits.size() == 2);

  // Empty pending writes. nextOffset=10, flushed pos=8
  ctx.getPendingWritesForTest().remove(new OffsetRange(10, 15));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);
  
  // Empty pending writes
  ctx.setNextOffsetForTest((long) 8); // flushed pos = 8
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  
}
 
开发者ID:naver,项目名称:hadoop,代码行数:73,代码来源:TestWrites.java


示例4: testCheckCommitFromRead

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommitFromRead() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);
  NfsConfiguration config = new NfsConfiguration();

  config.setBoolean(NfsConfigKeys.LARGE_FILE_UPLOAD, false);
  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new ShellBasedIdMapping(config), false, config);

  FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
  COMMIT_STATUS ret;
  WriteManager wm = new WriteManager(new ShellBasedIdMapping(config), config, false);
  assertTrue(wm.addOpenFileStream(h, ctx));
  
  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals( COMMIT_STATUS.COMMIT_INACTIVE_CTX, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
  
  ctx.getPendingWritesForTest().put(new OffsetRange(10, 15),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE, ret);
  assertEquals(Nfs3Status.NFS3ERR_IO, wm.commitBeforeRead(dfsClient, h, 0));
  
  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  ctx.setNextOffsetForTest((long)10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, ch, 1, attr, false);
  assertEquals(COMMIT_STATUS.COMMIT_DO_SYNC, status);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 5));
 
  status = ctx.checkCommitInternal(10, ch, 1, attr, true);
  assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 10));

  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size()); // commit triggered by read doesn't wait
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 11));

  // Test request with zero commit offset
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size());
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 0));

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(10, 15));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
}
 
开发者ID:naver,项目名称:hadoop,代码行数:72,代码来源:TestWrites.java


示例5: testCheckCommitFromReadLargeFileUpload

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with large file upload option
public void testCheckCommitFromReadLargeFileUpload() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);
  NfsConfiguration config = new NfsConfiguration();

  config.setBoolean(NfsConfigKeys.LARGE_FILE_UPLOAD, true);
  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new ShellBasedIdMapping(config), false, config);

  FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
  COMMIT_STATUS ret;
  WriteManager wm = new WriteManager(new ShellBasedIdMapping(config), config, false);
  assertTrue(wm.addOpenFileStream(h, ctx));
  
  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals( COMMIT_STATUS.COMMIT_INACTIVE_CTX, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
  
  ctx.getPendingWritesForTest().put(new OffsetRange(10, 15),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE, ret);
  assertEquals(Nfs3Status.NFS3ERR_IO, wm.commitBeforeRead(dfsClient, h, 0));
  
  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 6);
  ctx.setNextOffsetForTest((long)10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, ch, 1, attr, false);
  assertEquals(COMMIT_STATUS.COMMIT_DO_SYNC, status);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 5));
 
  // Test request with sequential writes
  status = ctx.checkCommitInternal(9, ch, 1, attr, true);
  assertTrue(status == COMMIT_STATUS.COMMIT_SPECIAL_WAIT);
  ret = ctx.checkCommit(dfsClient, 9, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_SPECIAL_WAIT, ret);
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 9));

  // Test request with non-sequential writes
  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 16, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_SPECIAL_SUCCESS, ret);
  assertEquals(0, commits.size()); // commit triggered by read doesn't wait
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 16));

  // Test request with zero commit offset
  // There is one pending write [10,15]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_SPECIAL_WAIT, ret);
  assertEquals(0, commits.size());
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 0));

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(10, 15));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_SPECIAL_WAIT, ret);
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 0));
}
 
开发者ID:naver,项目名称:hadoop,代码行数:72,代码来源:TestWrites.java


示例6: testEviction

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
public void testEviction() throws IOException, InterruptedException {
  Configuration conf = new Configuration();

  // Only two entries will be in the cache
  conf.setInt(Nfs3Constant.MAX_OPEN_FILES, 2);

  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx context1 =
      new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
          new IdUserGroup());
  OpenFileCtx context2 =
      new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
          new IdUserGroup());
  OpenFileCtx context3 =
      new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
          new IdUserGroup());
  OpenFileCtx context4 =
      new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
          new IdUserGroup());
  OpenFileCtx context5 =
      new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
          new IdUserGroup());

  OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);

  boolean ret = cache.put(new FileHandle(1), context1);
  assertTrue(ret);
  Thread.sleep(1000);
  ret = cache.put(new FileHandle(2), context2);
  assertTrue(ret);
  ret = cache.put(new FileHandle(3), context3);
  assertFalse(ret);
  assertTrue(cache.size() == 2);

  // Wait for the oldest stream to be evict-able, insert again
  Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
  assertTrue(cache.size() == 2);

  ret = cache.put(new FileHandle(3), context3);
  assertTrue(ret);
  assertTrue(cache.size() == 2);
  assertTrue(cache.get(new FileHandle(1)) == null);

  // Test inactive entry is evicted immediately
  context3.setActiveStatusForTest(false);
  ret = cache.put(new FileHandle(4), context4);
  assertTrue(ret);

  // Now the cache has context2 and context4
  // Test eviction failure if all entries have pending work.
  context2.getPendingWritesForTest().put(new OffsetRange(0, 100),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  context4.getPendingCommitsForTest()
      .put(new Long(100), new CommitCtx(0, null, 0, attr));
  Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
  ret = cache.put(new FileHandle(5), context5);
  assertFalse(ret);
}
 
开发者ID:hopshadoop,项目名称:hops,代码行数:64,代码来源:TestOpenFileCtxCache.java


示例7: testCheckCommit

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommit() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new IdUserGroup());

  COMMIT_STATUS ret;

  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_CTX);

  ctx.getPendingWritesForTest().put(new OffsetRange(5, 10),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE);

  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  
  status = ctx.checkCommitInternal(10, ch, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);

  ConcurrentNavigableMap<Long, CommitCtx> commits =
      ctx.getPendingCommitsForTest();
  Assert.assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  long key = commits.firstKey();
  Assert.assertTrue(key == 11);

  // Test request with zero commit offset
  commits.remove(new Long(11));
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  key = commits.firstKey();
  Assert.assertTrue(key == 9);

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(5, 10));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}
 
开发者ID:hopshadoop,项目名称:hops,代码行数:64,代码来源:TestWrites.java


示例8: testCheckCommitFromRead

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommitFromRead() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new IdUserGroup());

  FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
  COMMIT_STATUS ret;
  WriteManager wm = new WriteManager(new IdUserGroup(), new Configuration());
  assertTrue(wm.addOpenFileStream(h, ctx));
  
  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_INACTIVE_CTX, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
  
  ctx.getPendingWritesForTest().put(new OffsetRange(5, 10),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE, ret);
  assertEquals(Nfs3Status.NFS3ERR_IO, wm.commitBeforeRead(dfsClient, h, 0));
  
  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, ch, 1, attr, false);
  assertEquals(COMMIT_STATUS.COMMIT_DO_SYNC, status);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 5));

  status = ctx.checkCommitInternal(10, ch, 1, attr, true);
  assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 10));

  ConcurrentNavigableMap<Long, CommitCtx> commits =
      ctx.getPendingCommitsForTest();
  assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size()); // commit triggered by read doesn't wait
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX,
      wm.commitBeforeRead(dfsClient, h, 11));

  // Test request with zero commit offset
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size());
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX,
      wm.commitBeforeRead(dfsClient, h, 0));

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(5, 10));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
}
 
开发者ID:hopshadoop,项目名称:hops,代码行数:71,代码来源:TestWrites.java


示例9: testEviction

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
public void testEviction() throws IOException, InterruptedException {
  Configuration conf = new Configuration();

  // Only two entries will be in the cache
  conf.setInt(Nfs3Constant.MAX_OPEN_FILES, 2);

  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new IdUserGroup());
  OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new IdUserGroup());
  OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new IdUserGroup());
  OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new IdUserGroup());
  OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath",
      dfsClient, new IdUserGroup());

  OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);

  boolean ret = cache.put(new FileHandle(1), context1);
  assertTrue(ret);
  Thread.sleep(1000);
  ret = cache.put(new FileHandle(2), context2);
  assertTrue(ret);
  ret = cache.put(new FileHandle(3), context3);
  assertFalse(ret);
  assertTrue(cache.size() == 2);

  // Wait for the oldest stream to be evict-able, insert again
  Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
  assertTrue(cache.size() == 2);

  ret = cache.put(new FileHandle(3), context3);
  assertTrue(ret);
  assertTrue(cache.size() == 2);
  assertTrue(cache.get(new FileHandle(1)) == null);

  // Test inactive entry is evicted immediately
  context3.setActiveStatusForTest(false);
  ret = cache.put(new FileHandle(4), context4);
  assertTrue(ret);

  // Now the cache has context2 and context4
  // Test eviction failure if all entries have pending work.
  context2.getPendingWritesForTest().put(new OffsetRange(0, 100),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  context4.getPendingCommitsForTest().put(new Long(100),
      new CommitCtx(0, null, 0, attr));
  Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
  ret = cache.put(new FileHandle(5), context5);
  assertFalse(ret);
}
 
开发者ID:Seagate,项目名称:hadoop-on-lustre2,代码行数:59,代码来源:TestOpenFileCtxCache.java


示例10: testCheckCommit

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommit() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new IdUserGroup());

  COMMIT_STATUS ret;

  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_CTX);

  ctx.getPendingWritesForTest().put(new OffsetRange(5, 10),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE);

  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  
  status = ctx.checkCommitInternal(10, ch, 1, attr, false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);

  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  Assert.assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  long key = commits.firstKey();
  Assert.assertTrue(key == 11);

  // Test request with zero commit offset
  commits.remove(new Long(11));
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  key = commits.firstKey();
  Assert.assertTrue(key == 9);

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(5, 10));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}
 
开发者ID:Seagate,项目名称:hadoop-on-lustre2,代码行数:64,代码来源:TestWrites.java


示例11: testCheckCommitFromRead

import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; //导入依赖的package包/类
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
// includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX,
// COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC.
public void testCheckCommitFromRead() throws IOException {
  DFSClient dfsClient = Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr = new Nfs3FileAttributes();
  HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long) 0);

  OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
      new IdUserGroup());

  FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
  COMMIT_STATUS ret;
  WriteManager wm = new WriteManager(new IdUserGroup(), new Configuration());
  assertTrue(wm.addOpenFileStream(h, ctx));
  
  // Test inactive open file context
  ctx.setActiveStatusForTest(false);
  Channel ch = Mockito.mock(Channel.class);
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals( COMMIT_STATUS.COMMIT_INACTIVE_CTX, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
  
  ctx.getPendingWritesForTest().put(new OffsetRange(5, 10),
      new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE, ret);
  assertEquals(Nfs3Status.NFS3ERR_IO, wm.commitBeforeRead(dfsClient, h, 0));
  
  // Test request with non zero commit offset
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long) 10);
  COMMIT_STATUS status = ctx.checkCommitInternal(5, ch, 1, attr, false);
  assertEquals(COMMIT_STATUS.COMMIT_DO_SYNC, status);
  // Do_SYNC state will be updated to FINISHED after data sync
  ret = ctx.checkCommit(dfsClient, 5, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 5));
 
  status = ctx.checkCommitInternal(10, ch, 1, attr, true);
  assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret = ctx.checkCommit(dfsClient, 10, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 10));

  ConcurrentNavigableMap<Long, CommitCtx> commits = ctx
      .getPendingCommitsForTest();
  assertTrue(commits.size() == 0);
  ret = ctx.checkCommit(dfsClient, 11, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size()); // commit triggered by read doesn't wait
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 11));

  // Test request with zero commit offset
  // There is one pending write [5,10]
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_WAIT, ret);
  assertEquals(0, commits.size());
  assertEquals(Nfs3Status.NFS3ERR_JUKEBOX, wm.commitBeforeRead(dfsClient, h, 0));

  // Empty pending writes
  ctx.getPendingWritesForTest().remove(new OffsetRange(5, 10));
  ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, true);
  assertEquals(COMMIT_STATUS.COMMIT_FINISHED, ret);
  assertEquals(Nfs3Status.NFS3_OK, wm.commitBeforeRead(dfsClient, h, 0));
}
 
开发者ID:Seagate,项目名称:hadoop-on-lustre2,代码行数:69,代码来源:TestWrites.java



注:本文中的org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Java MatrixEntry类代码示例发布时间:2022-05-22
下一篇:
Java RopMethod类代码示例发布时间:2022-05-22
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap