下面列出了怎么用org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem的API类实例代码及写法，或者点击链接到github查看源代码。
private MockFileSystem createFileSystemForServiceName(final String service)
throws IOException {
MockFileSystem mockFs = new MockFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn(service);
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
int unique = 0;
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(new Text(service));
// use unique value so when we restore from token storage, we can
// tell if it's really the same token
token.setKind(new Text("token" + unique++));
return token;
}
});
return mockFs;
}
@Test
public void testSingleTokenFetch() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/[email protected]");
String renewer = Master.getMasterPrincipal(conf);
Credentials credentials = new Credentials();
final MockFileSystem fs = new MockFileSystem();
final MockFileSystem mockFs = (MockFileSystem) fs.getRawFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn("host:0");
when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0"));
Path mockPath = mock(Path.class);
when(mockPath.getFileSystem(conf)).thenReturn(mockFs);
Path[] paths = new Path[]{ mockPath, mockPath };
when(mockFs.addDelegationTokens("me", credentials)).thenReturn(null);
TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf);
verify(mockFs, times(1)).addDelegationTokens(renewer, credentials);
}
@SuppressWarnings("deprecation")
@Test
public void testGetTokensForNamenodes() throws IOException,
URISyntaxException {
Path TEST_ROOT_DIR =
new Path(System.getProperty("test.build.data", "test/build/data"));
// ick, but need fq path minus file:/
String binaryTokenFile =
FileSystem.getLocal(conf)
.makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri()
.getPath();
MockFileSystem fs1 = createFileSystemForServiceName("service1");
Credentials creds = new Credentials();
Token<?> token1 = fs1.getDelegationToken(renewer);
creds.addToken(token1.getService(), token1);
// wait to set, else the obtain tokens call above will fail with FNF
conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
String fs_addr = fs1.getCanonicalServiceName();
Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
assertNotNull("Token for nn is null", nnt);
}
@Test
public void testFsWithChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem fs3 = createFileSystemForServiceName(null);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs1, fs2, fs3);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false); // has no tokens of own, only child tokens
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, true);
verifyTokenFetch(fs3, false);
assertEquals(2, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
@Test
public void testFsWithDuplicateChildren() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs1");
MockFileSystem fs = createFileSystemForServiceName(service);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs, true);
assertEquals(1, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service));
}
@Test
public void testFsWithDuplicateChildrenTokenExists() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs1");
Token<?> token = mock(Token.class);
credentials.addToken(service, token);
MockFileSystem fs = createFileSystemForServiceName(service);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs, false);
assertEquals(1, credentials.numberOfTokens());
assertSame(token, credentials.getToken(service));
}
@Test
public void testFsWithChildTokensOneExists() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Token<?> token = mock(Token.class);
credentials.addToken(service2, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem fs3 = createFileSystemForServiceName(null);
MockFileSystem multiFs = createFileSystemForServiceName(null, fs1, fs2, fs3);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, false); // we had added its token to credentials
verifyTokenFetch(fs3, false);
assertEquals(2, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service1));
assertSame(token, credentials.getToken(service2));
}
@Test
public void testFsWithMyOwnAndChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Text myService = new Text("multiTokenFs");
Token<?> token = mock(Token.class);
credentials.addToken(service2, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, true); // its own token and also of its children
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, false); // we had added its token to credentials
assertEquals(3, credentials.numberOfTokens());
assertNotNull(credentials.getToken(myService));
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
@Test
public void testFsWithMyOwnExistsAndChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Text myService = new Text("multiTokenFs");
Token<?> token = mock(Token.class);
credentials.addToken(myService, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false); // we had added its token to credentials
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, true);
assertEquals(3, credentials.numberOfTokens());
assertSame(token, credentials.getToken(myService));
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
public static MockFileSystem createFileSystemForServiceName(
final Text service, final FileSystem... children) throws IOException {
final MockFileSystem fs = new MockFileSystem();
final MockFileSystem mockFs = fs.getRawFileSystem();
if (service != null) {
when(mockFs.getCanonicalServiceName()).thenReturn(service.toString());
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(service);
return token;
}
});
}
when(mockFs.getChildFileSystems()).thenReturn(children);
return fs;
}
private MockFileSystem createFileSystemForServiceName(final String service)
throws IOException {
MockFileSystem mockFs = new MockFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn(service);
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
int unique = 0;
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(new Text(service));
// use unique value so when we restore from token storage, we can
// tell if it's really the same token
token.setKind(new Text("token" + unique++));
return token;
}
});
return mockFs;
}
@Test
public void testSingleTokenFetch() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/[email protected]");
String renewer = Master.getMasterPrincipal(conf);
Credentials credentials = new Credentials();
final MockFileSystem fs = new MockFileSystem();
final MockFileSystem mockFs = (MockFileSystem) fs.getRawFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn("host:0");
when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0"));
Path mockPath = mock(Path.class);
when(mockPath.getFileSystem(conf)).thenReturn(mockFs);
Path[] paths = new Path[]{ mockPath, mockPath };
when(mockFs.addDelegationTokens("me", credentials)).thenReturn(null);
TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf);
verify(mockFs, times(1)).addDelegationTokens(renewer, credentials);
}
@SuppressWarnings("deprecation")
@Test
public void testGetTokensForNamenodes() throws IOException,
URISyntaxException {
Path TEST_ROOT_DIR =
new Path(System.getProperty("test.build.data", "test/build/data"));
// ick, but need fq path minus file:/
String binaryTokenFile =
FileSystem.getLocal(conf)
.makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri()
.getPath();
MockFileSystem fs1 = createFileSystemForServiceName("service1");
Credentials creds = new Credentials();
Token<?> token1 = fs1.getDelegationToken(renewer);
creds.addToken(token1.getService(), token1);
// wait to set, else the obtain tokens call above will fail with FNF
conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
String fs_addr = fs1.getCanonicalServiceName();
Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
assertNotNull("Token for nn is null", nnt);
}
@Test
public void testFsWithChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem fs3 = createFileSystemForServiceName(null);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs1, fs2, fs3);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false); // has no tokens of own, only child tokens
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, true);
verifyTokenFetch(fs3, false);
assertEquals(2, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
@Test
public void testFsWithDuplicateChildren() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs1");
MockFileSystem fs = createFileSystemForServiceName(service);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs, true);
assertEquals(1, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service));
}
@Test
public void testFsWithDuplicateChildrenTokenExists() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs1");
Token<?> token = mock(Token.class);
credentials.addToken(service, token);
MockFileSystem fs = createFileSystemForServiceName(service);
MockFileSystem multiFs =
createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs, false);
assertEquals(1, credentials.numberOfTokens());
assertSame(token, credentials.getToken(service));
}
@Test
public void testFsWithChildTokensOneExists() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Token<?> token = mock(Token.class);
credentials.addToken(service2, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem fs3 = createFileSystemForServiceName(null);
MockFileSystem multiFs = createFileSystemForServiceName(null, fs1, fs2, fs3);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false);
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, false); // we had added its token to credentials
verifyTokenFetch(fs3, false);
assertEquals(2, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service1));
assertSame(token, credentials.getToken(service2));
}
@Test
public void testFsWithMyOwnAndChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Text myService = new Text("multiTokenFs");
Token<?> token = mock(Token.class);
credentials.addToken(service2, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, true); // its own token and also of its children
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, false); // we had added its token to credentials
assertEquals(3, credentials.numberOfTokens());
assertNotNull(credentials.getToken(myService));
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
@Test
public void testFsWithMyOwnExistsAndChildTokens() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Text myService = new Text("multiTokenFs");
Token<?> token = mock(Token.class);
credentials.addToken(myService, token);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
multiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(multiFs, false); // we had added its token to credentials
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, true);
assertEquals(3, credentials.numberOfTokens());
assertSame(token, credentials.getToken(myService));
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
}
public static MockFileSystem createFileSystemForServiceName(
final Text service, final FileSystem... children) throws IOException {
final MockFileSystem fs = new MockFileSystem();
final MockFileSystem mockFs = fs.getRawFileSystem();
if (service != null) {
when(mockFs.getCanonicalServiceName()).thenReturn(service.toString());
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(service);
return token;
}
});
}
when(mockFs.getChildFileSystems()).thenReturn(children);
return fs;
}
private MockFileSystem createFileSystemForServiceName(final String service)
throws IOException {
MockFileSystem mockFs = new MockFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn(service);
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
int unique = 0;
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(new Text(service));
// use unique value so when we restore from token storage, we can
// tell if it's really the same token
token.setKind(new Text("token" + unique++));
return token;
}
});
return mockFs;
}
private MockFileSystem createFileSystemForServiceName(final String service)
throws IOException {
MockFileSystem mockFs = new MockFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn(service);
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
new Answer<Token<?>>() {
int unique = 0;
@Override
public Token<?> answer(InvocationOnMock invocation) throws Throwable {
Token<?> token = new Token<TokenIdentifier>();
token.setService(new Text(service));
// use unique value so when we restore from token storage, we can
// tell if it's really the same token
token.setKind(new Text("token" + unique++));
return token;
}
});
return mockFs;
}
@Test
public void testFsWithNoToken() throws Exception {
MockFileSystem fs = createFileSystemForServiceName(null);
Credentials credentials = new Credentials();
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, false);
assertEquals(0, credentials.numberOfTokens());
}
@Test
public void testFsWithToken() throws Exception {
Text service = new Text("singleTokenFs");
MockFileSystem fs = createFileSystemForServiceName(service);
Credentials credentials = new Credentials();
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, true);
assertEquals(1, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service));
}
@Test
public void testFsWithTokenExists() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs");
MockFileSystem fs = createFileSystemForServiceName(service);
Token<?> token = mock(Token.class);
credentials.addToken(service, token);
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, false);
assertEquals(1, credentials.numberOfTokens());
assertSame(token, credentials.getToken(service));
}
@Test
public void testFsWithNestedDuplicatesChildren() throws Exception {
Credentials credentials = new Credentials();
Text service1 = new Text("singleTokenFs1");
Text service2 = new Text("singleTokenFs2");
Text service4 = new Text("singleTokenFs4");
Text multiService = new Text("multiTokenFs");
Token<?> token2 = mock(Token.class);
credentials.addToken(service2, token2);
MockFileSystem fs1 = createFileSystemForServiceName(service1);
MockFileSystem fs1B = createFileSystemForServiceName(service1);
MockFileSystem fs2 = createFileSystemForServiceName(service2);
MockFileSystem fs3 = createFileSystemForServiceName(null);
MockFileSystem fs4 = createFileSystemForServiceName(service4);
// now let's get dirty! ensure dup tokens aren't fetched even when
// repeated and dupped in a nested fs. fs4 is a real test of the drill
// down: multi-filter-multi-filter-filter-fs4.
MockFileSystem multiFs = createFileSystemForServiceName(multiService,
fs1, fs1B, fs2, fs2, new FilterFileSystem(fs3),
new FilterFileSystem(new FilterFileSystem(fs4)));
MockFileSystem superMultiFs = createFileSystemForServiceName(null,
fs1, fs1B, fs1, new FilterFileSystem(fs3), new FilterFileSystem(multiFs));
superMultiFs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(superMultiFs, false); // does not have its own token
verifyTokenFetch(multiFs, true); // has its own token
verifyTokenFetch(fs1, true);
verifyTokenFetch(fs2, false); // we had added its token to credentials
verifyTokenFetch(fs3, false); // has no tokens
verifyTokenFetch(fs4, true);
assertEquals(4, credentials.numberOfTokens()); //fs1+fs2+fs4+multifs (fs3=0)
assertNotNull(credentials.getToken(service1));
assertNotNull(credentials.getToken(service2));
assertSame(token2, credentials.getToken(service2));
assertNotNull(credentials.getToken(multiService));
assertNotNull(credentials.getToken(service4));
}
private void verifyTokenFetch(MockFileSystem fs, boolean expected) throws IOException {
verify(fs.getRawFileSystem(), atLeast(1)).getCanonicalServiceName();
if (expected) {
verify(fs.getRawFileSystem()).getDelegationToken(renewer);
} else {
verify(fs.getRawFileSystem(), never()).getDelegationToken(any(String.class));
}
verify(fs.getRawFileSystem(), atLeast(1)).getChildFileSystems();
}
@Test
public void testFsWithNoToken() throws Exception {
MockFileSystem fs = createFileSystemForServiceName(null);
Credentials credentials = new Credentials();
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, false);
assertEquals(0, credentials.numberOfTokens());
}
@Test
public void testFsWithToken() throws Exception {
Text service = new Text("singleTokenFs");
MockFileSystem fs = createFileSystemForServiceName(service);
Credentials credentials = new Credentials();
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, true);
assertEquals(1, credentials.numberOfTokens());
assertNotNull(credentials.getToken(service));
}
@Test
public void testFsWithTokenExists() throws Exception {
Credentials credentials = new Credentials();
Text service = new Text("singleTokenFs");
MockFileSystem fs = createFileSystemForServiceName(service);
Token<?> token = mock(Token.class);
credentials.addToken(service, token);
fs.addDelegationTokens(renewer, credentials);
verifyTokenFetch(fs, false);
assertEquals(1, credentials.numberOfTokens());
assertSame(token, credentials.getToken(service));
}