001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.util;
019
020 import java.io.BufferedReader;
021 import java.io.File;
022 import java.io.IOException;
023 import java.io.InputStreamReader;
024 import java.util.Arrays;
025 import java.util.Map;
026 import java.util.Timer;
027 import java.util.TimerTask;
028 import java.util.concurrent.atomic.AtomicBoolean;
029
030 import org.apache.commons.logging.Log;
031 import org.apache.commons.logging.LogFactory;
032 import org.apache.hadoop.classification.InterfaceAudience;
033 import org.apache.hadoop.classification.InterfaceStability;
034
035 /**
036 * A base class for running a Unix command.
037 *
038 * <code>Shell</code> can be used to run unix commands like <code>du</code> or
039 * <code>df</code>. It also offers facilities to gate commands by
040 * time-intervals.
041 */
042 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
043 @InterfaceStability.Unstable
044 abstract public class Shell {
045
046 public static final Log LOG = LogFactory.getLog(Shell.class);
047
048 private static boolean IS_JAVA7_OR_ABOVE =
049 System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
050
051 public static boolean isJava7OrAbove() {
052 return IS_JAVA7_OR_ABOVE;
053 }
054
055 /** a Unix command to get the current user's name */
056 public final static String USER_NAME_COMMAND = "whoami";
057
058 /** Windows CreateProcess synchronization object */
059 public static final Object WindowsProcessLaunchLock = new Object();
060
061 /** a Unix command to get the current user's groups list */
062 public static String[] getGroupsCommand() {
063 return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
064 : new String[]{"bash", "-c", "groups"};
065 }
066
067 /** a Unix command to get a given user's groups list */
068 public static String[] getGroupsForUserCommand(final String user) {
069 //'groups username' command return is non-consistent across different unixes
070 return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
071 : new String [] {"bash", "-c", "id -Gn " + user};
072 }
073
074 /** a Unix command to get a given netgroup's user list */
075 public static String[] getUsersForNetgroupCommand(final String netgroup) {
076 //'groups username' command return is non-consistent across different unixes
077 return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
078 : new String [] {"bash", "-c", "getent netgroup " + netgroup};
079 }
080
081 /** Return a command to get permission information. */
082 public static String[] getGetPermissionCommand() {
083 return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
084 : new String[] { "/bin/ls", "-ld" };
085 }
086
087 /** Return a command to set permission */
088 public static String[] getSetPermissionCommand(String perm, boolean recursive) {
089 if (recursive) {
090 return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
091 : new String[] { "chmod", "-R", perm };
092 } else {
093 return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
094 : new String[] { "chmod", perm };
095 }
096 }
097
098 /**
099 * Return a command to set permission for specific file.
100 *
101 * @param perm String permission to set
102 * @param recursive boolean true to apply to all sub-directories recursively
103 * @param file String file to set
104 * @return String[] containing command and arguments
105 */
106 public static String[] getSetPermissionCommand(String perm, boolean recursive,
107 String file) {
108 String[] baseCmd = getSetPermissionCommand(perm, recursive);
109 String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
110 cmdWithFile[cmdWithFile.length - 1] = file;
111 return cmdWithFile;
112 }
113
114 /** Return a command to set owner */
115 public static String[] getSetOwnerCommand(String owner) {
116 return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
117 : new String[] { "chown", owner };
118 }
119
120 /** Return a command to create symbolic links */
121 public static String[] getSymlinkCommand(String target, String link) {
122 return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
123 : new String[] { "ln", "-s", target, link };
124 }
125
126 /** Return a command to read the target of the a symbolic link*/
127 public static String[] getReadlinkCommand(String link) {
128 return WINDOWS ? new String[] { WINUTILS, "readlink", link }
129 : new String[] { "readlink", link };
130 }
131
132 /** Return a command for determining if process with specified pid is alive. */
133 public static String[] getCheckProcessIsAliveCommand(String pid) {
134 return Shell.WINDOWS ?
135 new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
136 new String[] { "kill", "-0", isSetsidAvailable ? "-" + pid : pid };
137 }
138
139 /** Return a command to send a signal to a given pid */
140 public static String[] getSignalKillCommand(int code, String pid) {
141 return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid } :
142 new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
143 }
144
145 /** Return a regular expression string that match environment variables */
146 public static String getEnvironmentVariableRegex() {
147 return (WINDOWS) ? "%([A-Za-z_][A-Za-z0-9_]*?)%" :
148 "\\$([A-Za-z_][A-Za-z0-9_]*)";
149 }
150
151 /**
152 * Returns a File referencing a script with the given basename, inside the
153 * given parent directory. The file extension is inferred by platform: ".cmd"
154 * on Windows, or ".sh" otherwise.
155 *
156 * @param parent File parent directory
157 * @param basename String script file basename
158 * @return File referencing the script in the directory
159 */
160 public static File appendScriptExtension(File parent, String basename) {
161 return new File(parent, appendScriptExtension(basename));
162 }
163
164 /**
165 * Returns a script file name with the given basename. The file extension is
166 * inferred by platform: ".cmd" on Windows, or ".sh" otherwise.
167 *
168 * @param basename String script file basename
169 * @return String script file name
170 */
171 public static String appendScriptExtension(String basename) {
172 return basename + (WINDOWS ? ".cmd" : ".sh");
173 }
174
175 /**
176 * Returns a command to run the given script. The script interpreter is
177 * inferred by platform: cmd on Windows or bash otherwise.
178 *
179 * @param script File script to run
180 * @return String[] command to run the script
181 */
182 public static String[] getRunScriptCommand(File script) {
183 String absolutePath = script.getAbsolutePath();
184 return WINDOWS ? new String[] { "cmd", "/c", absolutePath } :
185 new String[] { "/bin/bash", absolutePath };
186 }
187
188 /** a Unix command to set permission */
189 public static final String SET_PERMISSION_COMMAND = "chmod";
190 /** a Unix command to set owner */
191 public static final String SET_OWNER_COMMAND = "chown";
192
193 /** a Unix command to set the change user's groups list */
194 public static final String SET_GROUP_COMMAND = "chgrp";
195 /** a Unix command to create a link */
196 public static final String LINK_COMMAND = "ln";
197 /** a Unix command to get a link target */
198 public static final String READ_LINK_COMMAND = "readlink";
199
200 /**Time after which the executing script would be timedout*/
201 protected long timeOutInterval = 0L;
202 /** If or not script timed out*/
203 private AtomicBoolean timedOut;
204
205
206 /** Centralized logic to discover and validate the sanity of the Hadoop
207 * home directory. Returns either NULL or a directory that exists and
208 * was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV
209 * variable. This does a lot of work so it should only be called
210 * privately for initialization once per process.
211 **/
212 private static String checkHadoopHome() {
213
214 // first check the Dflag hadoop.home.dir with JVM scope
215 String home = System.getProperty("hadoop.home.dir");
216
217 // fall back to the system/user-global env variable
218 if (home == null) {
219 home = System.getenv("HADOOP_HOME");
220 }
221
222 try {
223 // couldn't find either setting for hadoop's home directory
224 if (home == null) {
225 throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
226 }
227
228 if (home.startsWith("\"") && home.endsWith("\"")) {
229 home = home.substring(1, home.length()-1);
230 }
231
232 // check that the home setting is actually a directory that exists
233 File homedir = new File(home);
234 if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
235 throw new IOException("Hadoop home directory " + homedir
236 + " does not exist, is not a directory, or is not an absolute path.");
237 }
238
239 home = homedir.getCanonicalPath();
240
241 } catch (IOException ioe) {
242 if (LOG.isDebugEnabled()) {
243 LOG.debug("Failed to detect a valid hadoop home directory", ioe);
244 }
245 home = null;
246 }
247
248 return home;
249 }
250 private static String HADOOP_HOME_DIR = checkHadoopHome();
251
252 // Public getter, throws an exception if HADOOP_HOME failed validation
253 // checks and is being referenced downstream.
254 public static final String getHadoopHome() throws IOException {
255 if (HADOOP_HOME_DIR == null) {
256 throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
257 }
258
259 return HADOOP_HOME_DIR;
260 }
261
262 /** fully qualify the path to a binary that should be in a known hadoop
263 * bin location. This is primarily useful for disambiguating call-outs
264 * to executable sub-components of Hadoop to avoid clashes with other
265 * executables that may be in the path. Caveat: this call doesn't
266 * just format the path to the bin directory. It also checks for file
267 * existence of the composed path. The output of this call should be
268 * cached by callers.
269 * */
270 public static final String getQualifiedBinPath(String executable)
271 throws IOException {
272 // construct hadoop bin path to the specified executable
273 String fullExeName = HADOOP_HOME_DIR + File.separator + "bin"
274 + File.separator + executable;
275
276 File exeFile = new File(fullExeName);
277 if (!exeFile.exists()) {
278 throw new IOException("Could not locate executable " + fullExeName
279 + " in the Hadoop binaries.");
280 }
281
282 return exeFile.getCanonicalPath();
283 }
284
285 /** Set to true on Windows platforms */
286 public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
287 = System.getProperty("os.name").startsWith("Windows");
288
289 public static final boolean LINUX
290 = System.getProperty("os.name").startsWith("Linux");
291
292 /** a Windows utility to emulate Unix commands */
293 public static final String WINUTILS = getWinUtilsPath();
294
295 public static final String getWinUtilsPath() {
296 String winUtilsPath = null;
297
298 try {
299 if (WINDOWS) {
300 winUtilsPath = getQualifiedBinPath("winutils.exe");
301 }
302 } catch (IOException ioe) {
303 LOG.error("Failed to locate the winutils binary in the hadoop binary path",
304 ioe);
305 }
306
307 return winUtilsPath;
308 }
309
310 public static final boolean isSetsidAvailable = isSetsidSupported();
311 private static boolean isSetsidSupported() {
312 if (Shell.WINDOWS) {
313 return false;
314 }
315 ShellCommandExecutor shexec = null;
316 boolean setsidSupported = true;
317 try {
318 String[] args = {"setsid", "bash", "-c", "echo $$"};
319 shexec = new ShellCommandExecutor(args);
320 shexec.execute();
321 } catch (IOException ioe) {
322 LOG.debug("setsid is not available on this machine. So not using it.");
323 setsidSupported = false;
324 } finally { // handle the exit code
325 if (LOG.isDebugEnabled()) {
326 LOG.debug("setsid exited with exit code "
327 + (shexec != null ? shexec.getExitCode() : "(null executor)"));
328 }
329 }
330 return setsidSupported;
331 }
332
333 /** Token separator regex used to parse Shell tool outputs */
334 public static final String TOKEN_SEPARATOR_REGEX
335 = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
336
337 private long interval; // refresh interval in msec
338 private long lastTime; // last time the command was performed
339 private Map<String, String> environment; // env for the command execution
340 private File dir;
341 private Process process; // sub process used to execute the command
342 private int exitCode;
343
344 /**If or not script finished executing*/
345 private volatile AtomicBoolean completed;
346
347 public Shell() {
348 this(0L);
349 }
350
351 /**
352 * @param interval the minimum duration to wait before re-executing the
353 * command.
354 */
355 public Shell( long interval ) {
356 this.interval = interval;
357 this.lastTime = (interval<0) ? 0 : -interval;
358 }
359
360 /** set the environment for the command
361 * @param env Mapping of environment variables
362 */
363 protected void setEnvironment(Map<String, String> env) {
364 this.environment = env;
365 }
366
367 /** set the working directory
368 * @param dir The directory where the command would be executed
369 */
370 protected void setWorkingDirectory(File dir) {
371 this.dir = dir;
372 }
373
374 /** check to see if a command needs to be executed and execute if needed */
375 protected void run() throws IOException {
376 if (lastTime + interval > Time.now())
377 return;
378 exitCode = 0; // reset for next run
379 runCommand();
380 }
381
382 /** Run a command */
383 private void runCommand() throws IOException {
384 ProcessBuilder builder = new ProcessBuilder(getExecString());
385 Timer timeOutTimer = null;
386 ShellTimeoutTimerTask timeoutTimerTask = null;
387 timedOut = new AtomicBoolean(false);
388 completed = new AtomicBoolean(false);
389
390 if (environment != null) {
391 builder.environment().putAll(this.environment);
392 }
393 if (dir != null) {
394 builder.directory(this.dir);
395 }
396
397 if (Shell.WINDOWS) {
398 synchronized (WindowsProcessLaunchLock) {
399 // To workaround the race condition issue with child processes
400 // inheriting unintended handles during process launch that can
401 // lead to hangs on reading output and error streams, we
402 // serialize process creation. More info available at:
403 // http://support.microsoft.com/kb/315939
404 process = builder.start();
405 }
406 } else {
407 process = builder.start();
408 }
409
410 if (timeOutInterval > 0) {
411 timeOutTimer = new Timer("Shell command timeout");
412 timeoutTimerTask = new ShellTimeoutTimerTask(
413 this);
414 //One time scheduling.
415 timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
416 }
417 final BufferedReader errReader =
418 new BufferedReader(new InputStreamReader(process
419 .getErrorStream()));
420 BufferedReader inReader =
421 new BufferedReader(new InputStreamReader(process
422 .getInputStream()));
423 final StringBuffer errMsg = new StringBuffer();
424
425 // read error and input streams as this would free up the buffers
426 // free the error stream buffer
427 Thread errThread = new Thread() {
428 @Override
429 public void run() {
430 try {
431 String line = errReader.readLine();
432 while((line != null) && !isInterrupted()) {
433 errMsg.append(line);
434 errMsg.append(System.getProperty("line.separator"));
435 line = errReader.readLine();
436 }
437 } catch(IOException ioe) {
438 LOG.warn("Error reading the error stream", ioe);
439 }
440 }
441 };
442 try {
443 errThread.start();
444 } catch (IllegalStateException ise) { }
445 try {
446 parseExecResult(inReader); // parse the output
447 // clear the input stream buffer
448 String line = inReader.readLine();
449 while(line != null) {
450 line = inReader.readLine();
451 }
452 // wait for the process to finish and check the exit code
453 exitCode = process.waitFor();
454 try {
455 // make sure that the error thread exits
456 errThread.join();
457 } catch (InterruptedException ie) {
458 LOG.warn("Interrupted while reading the error stream", ie);
459 }
460 completed.set(true);
461 //the timeout thread handling
462 //taken care in finally block
463 if (exitCode != 0) {
464 throw new ExitCodeException(exitCode, errMsg.toString());
465 }
466 } catch (InterruptedException ie) {
467 throw new IOException(ie.toString());
468 } finally {
469 if (timeOutTimer != null) {
470 timeOutTimer.cancel();
471 }
472 // close the input stream
473 try {
474 inReader.close();
475 } catch (IOException ioe) {
476 LOG.warn("Error while closing the input stream", ioe);
477 }
478 try {
479 if (!completed.get()) {
480 errThread.interrupt();
481 errThread.join();
482 }
483 } catch (InterruptedException ie) {
484 LOG.warn("Interrupted while joining errThread");
485 }
486 try {
487 errReader.close();
488 } catch (IOException ioe) {
489 LOG.warn("Error while closing the error stream", ioe);
490 }
491 process.destroy();
492 lastTime = Time.now();
493 }
494 }
495
496 /** return an array containing the command name & its parameters */
497 protected abstract String[] getExecString();
498
499 /** Parse the execution result */
500 protected abstract void parseExecResult(BufferedReader lines)
501 throws IOException;
502
503 /** get the current sub-process executing the given command
504 * @return process executing the command
505 */
506 public Process getProcess() {
507 return process;
508 }
509
510 /** get the exit code
511 * @return the exit code of the process
512 */
513 public int getExitCode() {
514 return exitCode;
515 }
516
517 /**
518 * This is an IOException with exit code added.
519 */
520 public static class ExitCodeException extends IOException {
521 int exitCode;
522
523 public ExitCodeException(int exitCode, String message) {
524 super(message);
525 this.exitCode = exitCode;
526 }
527
528 public int getExitCode() {
529 return exitCode;
530 }
531 }
532
533 /**
534 * A simple shell command executor.
535 *
536 * <code>ShellCommandExecutor</code>should be used in cases where the output
537 * of the command needs no explicit parsing and where the command, working
538 * directory and the environment remains unchanged. The output of the command
539 * is stored as-is and is expected to be small.
540 */
541 public static class ShellCommandExecutor extends Shell {
542
543 private String[] command;
544 private StringBuffer output;
545
546
547 public ShellCommandExecutor(String[] execString) {
548 this(execString, null);
549 }
550
551 public ShellCommandExecutor(String[] execString, File dir) {
552 this(execString, dir, null);
553 }
554
555 public ShellCommandExecutor(String[] execString, File dir,
556 Map<String, String> env) {
557 this(execString, dir, env , 0L);
558 }
559
560 /**
561 * Create a new instance of the ShellCommandExecutor to execute a command.
562 *
563 * @param execString The command to execute with arguments
564 * @param dir If not-null, specifies the directory which should be set
565 * as the current working directory for the command.
566 * If null, the current working directory is not modified.
567 * @param env If not-null, environment of the command will include the
568 * key-value pairs specified in the map. If null, the current
569 * environment is not modified.
570 * @param timeout Specifies the time in milliseconds, after which the
571 * command will be killed and the status marked as timedout.
572 * If 0, the command will not be timed out.
573 */
574 public ShellCommandExecutor(String[] execString, File dir,
575 Map<String, String> env, long timeout) {
576 command = execString.clone();
577 if (dir != null) {
578 setWorkingDirectory(dir);
579 }
580 if (env != null) {
581 setEnvironment(env);
582 }
583 timeOutInterval = timeout;
584 }
585
586
587 /** Execute the shell command. */
588 public void execute() throws IOException {
589 this.run();
590 }
591
592 @Override
593 public String[] getExecString() {
594 return command;
595 }
596
597 @Override
598 protected void parseExecResult(BufferedReader lines) throws IOException {
599 output = new StringBuffer();
600 char[] buf = new char[512];
601 int nRead;
602 while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
603 output.append(buf, 0, nRead);
604 }
605 }
606
607 /** Get the output of the shell command.*/
608 public String getOutput() {
609 return (output == null) ? "" : output.toString();
610 }
611
612 /**
613 * Returns the commands of this instance.
614 * Arguments with spaces in are presented with quotes round; other
615 * arguments are presented raw
616 *
617 * @return a string representation of the object.
618 */
619 @Override
620 public String toString() {
621 StringBuilder builder = new StringBuilder();
622 String[] args = getExecString();
623 for (String s : args) {
624 if (s.indexOf(' ') >= 0) {
625 builder.append('"').append(s).append('"');
626 } else {
627 builder.append(s);
628 }
629 builder.append(' ');
630 }
631 return builder.toString();
632 }
633 }
634
635 /**
636 * To check if the passed script to shell command executor timed out or
637 * not.
638 *
639 * @return if the script timed out.
640 */
641 public boolean isTimedOut() {
642 return timedOut.get();
643 }
644
645 /**
646 * Set if the command has timed out.
647 *
648 */
649 private void setTimedOut() {
650 this.timedOut.set(true);
651 }
652
653 /**
654 * Static method to execute a shell command.
655 * Covers most of the simple cases without requiring the user to implement
656 * the <code>Shell</code> interface.
657 * @param cmd shell command to execute.
658 * @return the output of the executed command.
659 */
660 public static String execCommand(String ... cmd) throws IOException {
661 return execCommand(null, cmd, 0L);
662 }
663
664 /**
665 * Static method to execute a shell command.
666 * Covers most of the simple cases without requiring the user to implement
667 * the <code>Shell</code> interface.
668 * @param env the map of environment key=value
669 * @param cmd shell command to execute.
670 * @param timeout time in milliseconds after which script should be marked timeout
671 * @return the output of the executed command.o
672 */
673
674 public static String execCommand(Map<String, String> env, String[] cmd,
675 long timeout) throws IOException {
676 ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env,
677 timeout);
678 exec.execute();
679 return exec.getOutput();
680 }
681
682 /**
683 * Static method to execute a shell command.
684 * Covers most of the simple cases without requiring the user to implement
685 * the <code>Shell</code> interface.
686 * @param env the map of environment key=value
687 * @param cmd shell command to execute.
688 * @return the output of the executed command.
689 */
690 public static String execCommand(Map<String,String> env, String ... cmd)
691 throws IOException {
692 return execCommand(env, cmd, 0L);
693 }
694
695 /**
696 * Timer which is used to timeout scripts spawned off by shell.
697 */
698 private static class ShellTimeoutTimerTask extends TimerTask {
699
700 private Shell shell;
701
702 public ShellTimeoutTimerTask(Shell shell) {
703 this.shell = shell;
704 }
705
706 @Override
707 public void run() {
708 Process p = shell.getProcess();
709 try {
710 p.exitValue();
711 } catch (Exception e) {
712 //Process has not terminated.
713 //So check if it has completed
714 //if not just destroy it.
715 if (p != null && !shell.completed.get()) {
716 shell.setTimedOut();
717 p.destroy();
718 }
719 }
720 }
721 }
722 }