下面列出了怎么用org.apache.hadoop.util.Shell.ShellCommandExecutor的API类实例代码及写法,或者点击链接到github查看源代码。
public static boolean isAvailable() {
if (Shell.WINDOWS) {
ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
new String[] { Shell.WINUTILS, "help" });
try {
shellExecutor.execute();
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
} finally {
String output = shellExecutor.getOutput();
if (output != null &&
output.contains("Prints to stdout a list of processes in the task")) {
return true;
}
}
}
return false;
}
public void run() {
try {
Vector<String> args = new Vector<String>();
if (isSetsidAvailable()) {
args.add("setsid");
}
args.add("bash");
args.add("-c");
args.add(" echo $$ > " + pidFile + "; sh " + shellScript + " " + N
+ ";");
shexec = new ShellCommandExecutor(args.toArray(new String[0]));
shexec.execute();
} catch (ExitCodeException ee) {
LOG.info("Shell Command exit with a non-zero exit code. This is"
+ " expected as we are killing the subprocesses of the"
+ " task intentionally. " + ee);
} catch (IOException ioe) {
LOG.info("Error executing shell command " + ioe);
} finally {
LOG.info("Exit code: " + shexec.getExitCode());
}
}
/**
* Is perl supported on this machine ?
* @return true if perl is available and is working as expected
*/
public static boolean hasPerlSupport() {
boolean hasPerl = false;
ShellCommandExecutor shexec = new ShellCommandExecutor(
new String[] { "perl", "-e", "print 42" });
try {
shexec.execute();
if (shexec.getOutput().equals("42")) {
hasPerl = true;
}
else {
LOG.warn("Perl is installed, but isn't behaving as expected.");
}
} catch (Exception e) {
LOG.warn("Could not run perl: " + e);
}
return hasPerl;
}
/**
* Change the permissions on a file / directory, recursively, if
* needed.
* @param filename name of the file whose permissions are to change
* @param perm permission string
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
*/
public static int chmod(String filename, String perm, boolean recursive)
throws IOException {
String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
String[] args = new String[cmd.length + 1];
System.arraycopy(cmd, 0, args, 0, cmd.length);
args[cmd.length] = new File(filename).getPath();
ShellCommandExecutor shExec = new ShellCommandExecutor(args);
try {
shExec.execute();
}catch(IOException e) {
if(LOG.isDebugEnabled()) {
LOG.debug("Error while changing permission : " + filename
+" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
public static boolean isAvailable() {
if (Shell.WINDOWS) {
ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
new String[] { Shell.WINUTILS, "help" });
try {
shellExecutor.execute();
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
} finally {
String output = shellExecutor.getOutput();
if (output != null &&
output.contains("Prints to stdout a list of processes in the task")) {
return true;
}
}
}
return false;
}
public void run() {
try {
Vector<String> args = new Vector<String>();
if (isSetsidAvailable()) {
args.add("setsid");
}
args.add("bash");
args.add("-c");
args.add(" echo $$ > " + pidFile + "; sh " + shellScript + " " + N
+ ";");
shexec = new ShellCommandExecutor(args.toArray(new String[0]));
shexec.execute();
} catch (ExitCodeException ee) {
LOG.info("Shell Command exit with a non-zero exit code. This is"
+ " expected as we are killing the subprocesses of the"
+ " task intentionally. " + ee);
} catch (IOException ioe) {
LOG.info("Error executing shell command " + ioe);
} finally {
LOG.info("Exit code: " + shexec.getExitCode());
}
}
@Override
public void init() throws IOException {
// Send command to executor which will just start up,
// verify configuration/permissions and exit
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
"--checksetup"));
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
if (LOG.isDebugEnabled()) {
LOG.debug("checkLinuxExecutorSetup: " + Arrays.toString(commandArray));
}
try {
shExec.execute();
} catch (ExitCodeException e) {
int exitCode = shExec.getExitCode();
LOG.warn("Exit code from container executor initialization is : "
+ exitCode, e);
logOutput(shExec.getOutput());
throw new IOException("Linux container executor not configured properly"
+ " (error=" + exitCode + ")", e);
}
resourcesHandler.init(this);
}
public void mountCgroups(List<String> cgroupKVs, String hierarchy)
throws IOException {
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe, "--mount-cgroups", hierarchy));
command.addAll(cgroupKVs);
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
if (LOG.isDebugEnabled()) {
LOG.debug("mountCgroups: " + Arrays.toString(commandArray));
}
try {
shExec.execute();
} catch (IOException e) {
int ret_code = shExec.getExitCode();
LOG.warn("Exception in LinuxContainerExecutor mountCgroups ", e);
logOutput(shExec.getOutput());
throw new IOException("Problem mounting cgroups " + cgroupKVs +
"; exit code = " + ret_code + " and output: " + shExec.getOutput(), e);
}
}
/**
* Is perl supported on this machine ?
* @return true if perl is available and is working as expected
*/
public static boolean hasPerlSupport() {
boolean hasPerl = false;
ShellCommandExecutor shexec = new ShellCommandExecutor(
new String[] { "perl", "-e", "print 42" });
try {
shexec.execute();
if (shexec.getOutput().equals("42")) {
hasPerl = true;
}
else {
LOG.warn("Perl is installed, but isn't behaving as expected.");
}
} catch (Exception e) {
LOG.warn("Could not run perl: " + e);
}
return hasPerl;
}
/**
* Change the permissions on a file / directory, recursively, if
* needed.
* @param filename name of the file whose permissions are to change
* @param perm permission string
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
*/
public static int chmod(String filename, String perm, boolean recursive)
throws IOException {
String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
String[] args = new String[cmd.length + 1];
System.arraycopy(cmd, 0, args, 0, cmd.length);
args[cmd.length] = new File(filename).getPath();
ShellCommandExecutor shExec = new ShellCommandExecutor(args);
try {
shExec.execute();
}catch(IOException e) {
if(LOG.isDebugEnabled()) {
LOG.debug("Error while changing permission : " + filename
+" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
/**
* Change the permissions on a file / directory, recursively, if
* needed.
* @param filename name of the file whose permissions are to change
* @param perm permission string
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException exception on chmod
*/
public static int chmod(String filename, String perm, boolean recursive)
throws IOException {
String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
String[] args = new String[cmd.length + 1];
System.arraycopy(cmd, 0, args, 0, cmd.length);
args[cmd.length] = new File(filename).getPath();
ShellCommandExecutor shExec = new ShellCommandExecutor(args);
try {
shExec.execute();
}catch(IOException e) {
if(LOG.isDebugEnabled()) {
LOG.debug("Error while changing permission : {} Exception: {}", filename, StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
/**
* Helper method that runs a LinuxTaskController command
*
* @param taskCommand
* @param user
* @param cmdArgs
* @param env
* @throws IOException
*/
private void runCommand(TaskCommands taskCommand, String user,
List<String> cmdArgs, File workDir, Map<String, String> env)
throws IOException {
ShellCommandExecutor shExec =
buildTaskControllerExecutor(taskCommand, user, cmdArgs, workDir, env);
try {
shExec.execute();
} catch (Exception e) {
LOG.warn("Exit code from " + taskCommand.toString() + " is : "
+ shExec.getExitCode());
LOG.warn("Exception thrown by " + taskCommand.toString() + " : "
+ StringUtils.stringifyException(e));
LOG.info("Output from LinuxTaskController's " + taskCommand.toString()
+ " follows:");
logOutput(shExec.getOutput());
throw new IOException(e);
}
if (LOG.isDebugEnabled()) {
LOG.info("Output from LinuxTaskController's " + taskCommand.toString()
+ " follows:");
logOutput(shExec.getOutput());
}
}
/**
* Convenience method used to sending appropriate Kill signal to the task
* VM
* @param context
* @param command
* @throws IOException
*/
private void finishTask(TaskControllerContext context,
TaskCommands command) throws IOException{
if(context.task == null) {
LOG.info("Context task null not killing the JVM");
return;
}
ShellCommandExecutor shExec = buildTaskControllerExecutor(
command, context.env.conf.getUser(),
buildKillTaskCommandArgs(context), context.env.workDir,
context.env.env);
try {
shExec.execute();
} catch (Exception e) {
LOG.warn("Output from task-contoller is : " + shExec.getOutput());
throw new IOException(e);
}
}
@Override
void terminateTask(TaskControllerContext context) {
ShellCommandExecutor shexec = context.shExec;
if (shexec != null) {
Process process = shexec.getProcess();
if (Shell.WINDOWS) {
// Currently we don't use setsid on WINDOWS.
//So kill the process alone.
if (process != null) {
process.destroy();
}
}
else { // In addition to the task JVM, kill its subprocesses also.
String pid = context.pid;
if (pid != null) {
if(ProcessTree.isSetsidAvailable) {
ProcessTree.terminateProcessGroup(pid);
}else {
ProcessTree.terminateProcess(pid);
}
}
}
}
}
@Override
void killTask(TaskControllerContext context) {
ShellCommandExecutor shexec = context.shExec;
if (shexec != null) {
if (Shell.WINDOWS) {
//We don't do send kill process signal in case of windows as
//already we have done a process.destroy() in termintateTaskJVM()
return;
}
String pid = context.pid;
if (pid != null) {
if(ProcessTree.isSetsidAvailable) {
ProcessTree.killProcessGroup(pid);
}else {
ProcessTree.killProcess(pid);
}
}
}
}
public void testShellCommandTimeout() throws Throwable {
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File shellFile = new File(rootDir, "timeout.sh");
String timeoutCommand = "sleep 4; echo \"hello\"";
PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
writer.println(timeoutCommand);
writer.close();
shellFile.setExecutable(true);
Shell.ShellCommandExecutor shexc
= new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
null, null, 100);
try {
shexc.execute();
} catch (Exception e) {
//When timing out exception is thrown.
}
shellFile.delete();
assertTrue("Script didnt not timeout" , shexc.isTimedOut());
}
public void run() {
try {
Vector<String> args = new Vector<String>();
if(ProcessTree.isSetsidAvailable) {
args.add("setsid");
}
args.add("bash");
args.add("-c");
args.add(" echo $$ > " + pidFile + "; sh " +
shellScript + " " + N + ";") ;
shexec = new ShellCommandExecutor(args.toArray(new String[0]));
shexec.execute();
} catch (ExitCodeException ee) {
LOG.info("Shell Command exit with a non-zero exit code. This is" +
" expected as we are killing the subprocesses of the" +
" task intentionally. " + ee);
} catch (IOException ioe) {
LOG.info("Error executing shell command " + ioe);
} finally {
LOG.info("Exit code: " + shexec.getExitCode());
}
}
public List<TaskTrackerLoadInfo> releaseTrackers(int numTrackers)
throws IOException {
List<TaskTrackerLoadInfo> releasedTrackers =
new ArrayList<TaskTrackerLoadInfo>();
TaskTrackerLoadInfoIterator iterator = new WastedTimeTTLIIterator();
iterator.setTrackers(trackers);
while (releasedTrackers.size() < numTrackers && iterator.hasNext()) {
TaskTrackerLoadInfo tracker = iterator.next();
String host = tracker.getTaskTrackerHost();
if (trackers.contains(host)) {
continue;
}
ShellCommandExecutor removeHostCommand = new ShellCommandExecutor(
new String[]{"ssh", hostName,
"cd " + hadoopHome + " && " + "bin/hadoop " +
TTMover.class.getCanonicalName() + " -remove " + host});
try {
removeHostCommand.execute();
releasedTrackers.add(tracker);
} catch (IOException ex) {
DynamicCloudsDaemon.LOG.error("Error removing tracker " +
tracker.getTaskTrackerName(), ex);
}
}
return releasedTrackers;
}
public List<TaskTrackerLoadInfo> addTrackers(List<TaskTrackerLoadInfo> trackers) {
List<TaskTrackerLoadInfo> trackersAdded = new ArrayList<TaskTrackerLoadInfo>();
for (TaskTrackerLoadInfo tracker : trackers) {
String host = tracker.getTaskTrackerHost();
ShellCommandExecutor addHostCommand =
new ShellCommandExecutor(
new String[]{"ssh", hostName,
"cd " + hadoopHome + " && " + "bin/hadoop " +
TTMover.class.getCanonicalName() +
" -add " + host});
try {
addHostCommand.execute();
trackersAdded.add(tracker);
} catch (IOException ex) {
DynamicCloudsDaemon.LOG.error("Error adding tracker " + tracker.getTaskTrackerName(), ex);
}
}
return trackersAdded;
}
/**
* Change the permissions on a file / directory, recursively, if
* needed.
* @param filename name of the file whose permissions are to change
* @param perm permission string
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
* @throws InterruptedException
*/
public static int chmod(String filename, String perm, boolean recursive)
throws IOException, InterruptedException {
StringBuffer cmdBuf = new StringBuffer();
cmdBuf.append("chmod ");
if (recursive) {
cmdBuf.append("-R ");
}
cmdBuf.append(perm).append(" ");
cmdBuf.append(filename);
String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
try {
shExec.execute();
}catch(IOException e) {
if(Log.isDebugEnabled()) {
Log.debug("Error while changing permission : " + filename
+" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
/**
* Sends kill signal to process, forcefully terminating the process.
*
* @param pid process id
*/
public static void killProcess(String pid) {
//If process tree is not alive then return immediately.
if(!ProcessTree.isAlive(pid)) {
return;
}
String[] args = { "kill", "-9", pid };
ShellCommandExecutor shexec = new ShellCommandExecutor(args);
try {
shexec.execute();
} catch (IOException e) {
LOG.warn("Error sending SIGKILL to process "+ pid + " ."+
StringUtils.stringifyException(e));
} finally {
LOG.info("Killing process " + pid + " with SIGKILL. Exit code "
+ shexec.getExitCode());
}
}
/**
* Sends kill signal to all process belonging to same process group,
* forcefully terminating the process group.
*
* @param pgrpId process group id
*/
public static void killProcessGroup(String pgrpId) {
//If process tree is not alive then return immediately.
if(!ProcessTree.isProcessGroupAlive(pgrpId)) {
return;
}
String[] args = { "kill", "-9", "-"+pgrpId };
ShellCommandExecutor shexec = new ShellCommandExecutor(args);
try {
shexec.execute();
} catch (IOException e) {
LOG.warn("Error sending SIGKILL to process group "+ pgrpId + " ."+
StringUtils.stringifyException(e));
} finally {
LOG.info("Killing process group" + pgrpId + " with SIGKILL. Exit code "
+ shexec.getExitCode());
}
}
String getAllProcessInfoFromShell() {
ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
try {
shellExecutor.execute();
return shellExecutor.getOutput();
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
}
return null;
}
String getSystemInfoInfoFromShell() {
ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
new String[] { Shell.WINUTILS, "systeminfo" });
try {
shellExecutor.execute();
return shellExecutor.getOutput();
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
}
return null;
}
/**
* Returns true if the process with the specified pid is alive.
*
* @param pid String pid
* @return boolean true if the process is alive
*/
@VisibleForTesting
public static boolean containerIsAlive(String pid) throws IOException {
try {
new ShellCommandExecutor(Shell.getCheckProcessIsAliveCommand(pid))
.execute();
// successful execution means process is alive
return true;
}
catch (Shell.ExitCodeException e) {
// failure (non-zero exit code) means process is not alive
return false;
}
}
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath,
String containerIdStr, String user, Path pidFile, Resource resource,
File wordDir, Map<String, String> environment)
throws IOException {
String[] command = getRunCommand(wrapperScriptPath,
containerIdStr, user, pidFile, this.getConf(), resource);
LOG.info("launchContainer: " + Arrays.toString(command));
return new ShellCommandExecutor(
command,
wordDir,
environment);
}
/**
* Returns true if the process with the specified pid is alive.
*
* @param pid String pid
* @return boolean true if the process is alive
*/
@VisibleForTesting
public static boolean containerIsAlive(String pid) throws IOException {
try {
new ShellCommandExecutor(Shell.getCheckProcessIsAliveCommand(pid))
.execute();
// successful execution means process is alive
return true;
}
catch (ExitCodeException e) {
// failure (non-zero exit code) means process is not alive
return false;
}
}
private static void unTarUsingTar(File inFile, File untarDir,
boolean gzipped) throws IOException {
StringBuffer untarCommand = new StringBuffer();
if (gzipped) {
untarCommand.append(" gzip -dc '");
untarCommand.append(FileUtil.makeShellPath(inFile));
untarCommand.append("' | (");
}
untarCommand.append("cd '");
untarCommand.append(FileUtil.makeShellPath(untarDir));
untarCommand.append("' ; ");
untarCommand.append("tar -xf ");
if (gzipped) {
untarCommand.append(" -)");
} else {
untarCommand.append(FileUtil.makeShellPath(inFile));
}
String[] shellCmd = { "bash", "-c", untarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + inFile +
". Tar process exited with exit code " + exitcode);
}
}
public NodeHealthMonitorExecutor(String[] args) {
ArrayList<String> execScript = new ArrayList<String>();
execScript.add(nodeHealthScript);
if (args != null) {
execScript.addAll(Arrays.asList(args));
}
shexec = new ShellCommandExecutor(execScript
.toArray(new String[execScript.size()]), null, null, scriptTimeout);
}
String getAllProcessInfoFromShell() {
ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
try {
shellExecutor.execute();
return shellExecutor.getOutput();
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
}
return null;
}