下面列出了org.apache.hadoop.fs.ParentNotDirectoryException#org.apache.hadoop.fs.Options 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/**
* Rename file or directory.
* @see ClientProtocol#rename2(String, String, Options.Rename...)
*/
public void rename(String src, String dst, Options.Rename... options)
throws IOException {
checkOpen();
TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
try {
namenode.rename2(src, dst, options);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
DSQuotaExceededException.class,
FileAlreadyExistsException.class,
FileNotFoundException.class,
ParentNotDirectoryException.class,
SafeModeException.class,
NSQuotaExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
scope.close();
}
}
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
throws IOException {
checkNNStartup();
if(stateChangeLog.isDebugEnabled()) {
stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
}
if (!checkPathLength(dst)) {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
boolean success = false;
try {
namesystem.renameTo(src, dst, cacheEntry != null, options);
success = true;
} finally {
RetryCache.setState(cacheEntry, success);
}
metrics.incrFilesRenamed();
}
/**
* Rename file or directory.
* @see ClientProtocol#rename2(String, String, Options.Rename...)
*/
public void rename(String src, String dst, Options.Rename... options)
throws IOException {
checkOpen();
TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
try {
namenode.rename2(src, dst, options);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
DSQuotaExceededException.class,
FileAlreadyExistsException.class,
FileNotFoundException.class,
ParentNotDirectoryException.class,
SafeModeException.class,
NSQuotaExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
scope.close();
}
}
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
throws IOException {
checkNNStartup();
if(stateChangeLog.isDebugEnabled()) {
stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
}
if (!checkPathLength(dst)) {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
boolean success = false;
try {
namesystem.renameTo(src, dst, cacheEntry != null, options);
success = true;
} finally {
RetryCache.setState(cacheEntry, success);
}
metrics.incrFilesRenamed();
}
private DataOutputStream getOutputStream(FileSystemWALPointer pointer) throws IOException
{
Preconditions.checkArgument(outputStream == null, "output stream is not null");
if (pointer.offset > 0 && (fileSystemWAL.fileContext.getDefaultFileSystem() instanceof LocalFs ||
fileSystemWAL.fileContext.getDefaultFileSystem() instanceof RawLocalFs)) {
//On local file system the stream is always closed and never flushed so we open it again in append mode if the
//offset > 0. This block is entered only when appending to wal while writing on local fs.
return fileSystemWAL.fileContext.create(new Path(fileSystemWAL.tempPartFiles.get(pointer.partNum)),
EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND), Options.CreateOpts.CreateParent.createParent());
}
String partFile = fileSystemWAL.getPartFilePath(pointer.partNum);
String tmpFilePath = createTmpFilePath(partFile);
fileSystemWAL.tempPartFiles.put(pointer.partNum, tmpFilePath);
Preconditions.checkArgument(pointer.offset == 0, "offset > 0");
LOG.debug("open {} => {}", pointer.partNum, tmpFilePath);
outputStream = fileSystemWAL.fileContext.create(new Path(tmpFilePath),
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());
return outputStream;
}
private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException
{
FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
File file = new File("target/IOUtilsTest/testCopyPartial/input");
createDataFile(file, dataSize);
FileContext fileContext = FileContext.getFileContext();
DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));
Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
DataOutputStream outputStream = fileContext.create(output, EnumSet
.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());
if (offset == 0) {
IOUtils.copyPartial(inputStream, size, outputStream);
} else {
IOUtils.copyPartial(inputStream, offset, size, outputStream);
}
outputStream.close();
Assert.assertTrue("output exists", fileContext.util().exists(output));
Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
// FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
}
/** @throws Exception If failed. */
@Test
public void testCreateBase() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
Path file = new Path(dir, "someFile");
assertPathDoesNotExist(fs, file);
FsPermission fsPerm = new FsPermission((short)644);
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(fsPerm));
// Try to write something in file.
os.write("abc".getBytes());
os.close();
// Check file status.
FileStatus fileStatus = fs.getFileStatus(file);
assertFalse(fileStatus.isDirectory());
assertEquals(file, fileStatus.getPath());
assertEquals(fsPerm, fileStatus.getPermission());
}
/** @throws Exception If failed. */
@Test
public void testDeleteFailsIfNonRecursive() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
fs.delete(someDir2, false);
return null;
}
}, PathIsNotEmptyDirectoryException.class, null);
assertPathExists(fs, someDir2);
assertPathExists(fs, someDir3);
}
/**
* A wrapper around {@link FileSystem#rename(Path, Path)} which throws {@link IOException} if
* {@link FileSystem#rename(Path, Path)} returns False.
*/
public static void renamePath(FileSystem fs, Path oldName, Path newName, boolean overwrite) throws IOException {
//In default implementation of rename with rewrite option in FileSystem, if the parent dir of dst does not exist, it will throw exception,
//Which will fail some of our job unintentionally. So we only call that method when fs is an instance of DistributedFileSystem to avoid inconsistency problem
if(fs instanceof DistributedFileSystem) {
Options.Rename renameOptions = (overwrite) ? Options.Rename.OVERWRITE : Options.Rename.NONE;
((DistributedFileSystem) fs).rename(oldName, newName, renameOptions);
} else {
if (!fs.exists(oldName)) {
throw new FileNotFoundException(String.format("Failed to rename %s to %s: src not found", oldName, newName));
}
if (fs.exists(newName)) {
if (overwrite) {
HadoopUtils.moveToTrash(fs, newName);
} else {
throw new FileAlreadyExistsException(String.format("Failed to rename %s to %s: dst already exists", oldName, newName));
}
}
if (!fs.rename(oldName, newName)) {
throw new IOException(String.format("Failed to rename %s to %s", oldName, newName));
}
}
}
/** @throws Exception If failed. */
@Test
public void testDeleteRecursivelyFromRoot() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");
FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
Path root = new Path(fsHome, "/");
assertFalse(fs.delete(root, true));
assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));
assertPathDoesNotExist(fs, someDir3);
assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
assertPathExists(fs, root);
}
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckDefaultPermission() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
fs.setPermission(file, null);
assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
}
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckNonRecursiveness() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
Path tmpDir = new Path(fsHome, "/tmp");
FsPermission perm = new FsPermission((short)123);
fs.setPermission(tmpDir, perm);
assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
}
/** @throws Exception If failed. */
@SuppressWarnings("OctalInteger")
@Test
public void testSetPermission() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
for (short i = 0; i <= 0777; i += 7) {
FsPermission perm = new FsPermission(i);
fs.setPermission(file, perm);
assertEquals(perm, fs.getFileStatus(file).getPermission());
}
}
/** @throws Exception If failed. */
@Test
public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "myFile");
FsPermission perm = new FsPermission((short)123);
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
fs.setPermission(file, perm);
os.close();
assertEquals(perm, fs.getFileStatus(file).getPermission());
}
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersPathIsNull() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
fs.setOwner(null, "aUser", "aGroup");
return null;
}
}, NullPointerException.class, "Ouch! Argument cannot be null: p");
}
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersUserIsNull() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
fs.setOwner(file, null, "aGroup");
return null;
}
}, NullPointerException.class, "Ouch! Argument cannot be null: username");
}
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
fs.setOwner(file, "aUser", null);
return null;
}
}, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
}
/** @throws Exception If failed. */
@Test
public void testSetOwner() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());
fs.setOwner(file, "aUser", "aGroup");
assertEquals("aUser", fs.getFileStatus(file).getOwner());
assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
/** @throws Exception If failed. */
@Test
public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "myFile");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
fs.setOwner(file, "aUser", "aGroup");
os.close();
assertEquals("aUser", fs.getFileStatus(file).getOwner());
assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckNonRecursiveness() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "/tmp/my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
Path tmpDir = new Path(fsHome, "/tmp");
fs.setOwner(file, "fUser", "fGroup");
fs.setOwner(tmpDir, "dUser", "dGroup");
assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());
assertEquals("fUser", fs.getFileStatus(file).getOwner());
assertEquals("fGroup", fs.getFileStatus(file).getGroup());
}
/** @throws Exception If failed. */
@Test
public void testOpenIfPathIsAlreadyOpened() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "someFile");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
FSDataInputStream is1 = fs.open(file);
FSDataInputStream is2 = fs.open(file);
is1.close();
is2.close();
}
/** @throws Exception If failed. */
@Test
public void testOpen() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path file = new Path(fsHome, "someFile");
int cnt = 2 * 1024;
try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()))) {
for (long i = 0; i < cnt; i++)
out.writeLong(i);
}
assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());
try (FSDataInputStream in = fs.open(file, 1024)) {
for (long i = 0; i < cnt; i++)
assertEquals(i, in.readLong());
}
}
/** @throws Exception If failed. */
@Test
public void testAppendIfPathPointsToDirectory() throws Exception {
final Path fsHome = new Path(primaryFsUri);
final Path dir = new Path(fsHome, "/tmp");
Path file = new Path(dir, "my");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
@Override public Object call() throws Exception {
return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
Options.CreateOpts.perms(FsPermission.getDefault()));
}
}, IOException.class, null);
}
/** @throws Exception If failed. */
@Test
public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "someFile");
FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
Options.CreateOpts.perms(FsPermission.getDefault()));
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
return fs.create(file, EnumSet.of(CreateFlag.APPEND),
Options.CreateOpts.perms(FsPermission.getDefault()));
}
}, IOException.class, null);
appendOs.close();
}
/** @throws Exception If failed. */
@Test
public void testRenameCheckParametersDstPathIsNull() throws Exception {
Path fsHome = new Path(primaryFsUri);
final Path file = new Path(fsHome, "someFile");
fs.create(file, EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault())).close();
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
fs.rename(file, null);
return null;
}
}, NullPointerException.class, "Ouch! Argument cannot be null: f");
}
/** @throws Exception If failed. */
@Test
public void testRenameDirectory() throws Exception {
Path fsHome = new Path(primaryFsUri);
Path dir = new Path(fsHome, "/tmp/");
Path newDir = new Path(fsHome, "/tmpNew/");
FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
Options.CreateOpts.perms(FsPermission.getDefault()));
os.close();
fs.rename(dir, newDir);
assertPathDoesNotExist(fs, dir);
assertPathExists(fs, newDir);
}
@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
String operatorIdStr = String.valueOf(operatorId);
Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
String window = Long.toHexString(windowId);
boolean stateSaved = false;
FSDataOutputStream stream = null;
try {
stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
Options.CreateOpts.CreateParent.createParent());
store(stream, object);
stateSaved = true;
} catch (Throwable t) {
logger.debug("while saving {} {}", operatorId, window, t);
stateSaved = false;
throw Throwables.propagate(t);
} finally {
try {
if (stream != null) {
stream.close();
}
} catch (IOException ie) {
stateSaved = false;
throw new RuntimeException(ie);
} finally {
if (stateSaved) {
logger.debug("Saving {}: {}", operatorId, window);
fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
Options.Rename.OVERWRITE);
}
}
}
}
protected void moveToFinalPath(Path tempPath, Path finalPath) throws IOException {
if (fs instanceof DistributedFileSystem) {
((DistributedFileSystem) fs).rename(tempPath, finalPath, Options.Rename.OVERWRITE);
} else {
if (!fs.rename(tempPath, finalPath)) {
throw new IOException(String.format("Failed to commit %s (from %s)",
finalPath.toUri(), tempPath));
}
}
LOGGER.info("Committed {} to {}", tempPath, finalPath.toUri());
}
@Test
public void testCreateInternal_shouldCreateParent() throws Exception {
Configuration config = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
GoogleHadoopFS ghfs = new GoogleHadoopFS(initUri, config);
Path filePath =
new Path(initUri.resolve("/testCreateInternal_shouldCreateParent/dir/file").toString());
try (FSDataOutputStream stream =
ghfs.createInternal(
filePath,
EnumSet.of(CreateFlag.CREATE),
/* absolutePermission= */ null,
/* bufferSize= */ 128,
/* replication= */ (short) 1,
/* blockSize= */ 32,
() -> {},
new Options.ChecksumOpt(),
/* createParent= */ true)) {
stream.write(1);
assertThat(stream.size()).isEqualTo(1);
}
FileStatus parentStatus = ghfs.getFileStatus(filePath.getParent());
assertThat(parentStatus.getModificationTime()).isGreaterThan(0L);
}
public LogWriter(final Configuration conf, final Path remoteAppLogFile,
UserGroupInformation userUgi) throws IOException {
try {
this.fsDataOStream =
userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() {
@Override
public FSDataOutputStream run() throws Exception {
fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
fc.setUMask(APP_LOG_FILE_UMASK);
return fc.create(
remoteAppLogFile,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
new Options.CreateOpts[] {});
}
});
} catch (InterruptedException e) {
throw new IOException(e);
}
// Keys are not sorted: null arg
// 256KB minBlockSize : Expected log size for each container too
this.writer =
new TFile.Writer(this.fsDataOStream, 256 * 1024, conf.get(
YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE,
YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf);
//Write the version string
writeVersion();
}