001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.util;
019
020 import java.io.BufferedReader;
021 import java.io.File;
022 import java.io.IOException;
023 import java.io.InputStreamReader;
024 import java.util.Arrays;
025 import java.util.Map;
026 import java.util.Timer;
027 import java.util.TimerTask;
028 import java.util.concurrent.atomic.AtomicBoolean;
029
030 import org.apache.commons.logging.Log;
031 import org.apache.commons.logging.LogFactory;
032 import org.apache.hadoop.classification.InterfaceAudience;
033 import org.apache.hadoop.classification.InterfaceStability;
034
035 /**
036 * A base class for running a Unix command.
037 *
038 * <code>Shell</code> can be used to run unix commands like <code>du</code> or
039 * <code>df</code>. It also offers facilities to gate commands by
040 * time-intervals.
041 */
042 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
043 @InterfaceStability.Unstable
044 abstract public class Shell {
045
046 public static final Log LOG = LogFactory.getLog(Shell.class);
047
048 private static boolean IS_JAVA7_OR_ABOVE =
049 System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
050
051 public static boolean isJava7OrAbove() {
052 return IS_JAVA7_OR_ABOVE;
053 }
054
055 /** a Unix command to get the current user's name */
056 public final static String USER_NAME_COMMAND = "whoami";
057
058 /** Windows CreateProcess synchronization object */
059 public static final Object WindowsProcessLaunchLock = new Object();
060
061 /** a Unix command to get the current user's groups list */
062 public static String[] getGroupsCommand() {
063 return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
064 : new String[]{"bash", "-c", "groups"};
065 }
066
067 /** a Unix command to get a given user's groups list */
068 public static String[] getGroupsForUserCommand(final String user) {
069 //'groups username' command return is non-consistent across different unixes
070 return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
071 : new String [] {"bash", "-c", "id -Gn " + user};
072 }
073
074 /** a Unix command to get a given netgroup's user list */
075 public static String[] getUsersForNetgroupCommand(final String netgroup) {
076 //'groups username' command return is non-consistent across different unixes
077 return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
078 : new String [] {"bash", "-c", "getent netgroup " + netgroup};
079 }
080
081 /** Return a command to get permission information. */
082 public static String[] getGetPermissionCommand() {
083 return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
084 : new String[] { "/bin/ls", "-ld" };
085 }
086
087 /** Return a command to set permission */
088 public static String[] getSetPermissionCommand(String perm, boolean recursive) {
089 if (recursive) {
090 return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
091 : new String[] { "chmod", "-R", perm };
092 } else {
093 return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
094 : new String[] { "chmod", perm };
095 }
096 }
097
098 /**
099 * Return a command to set permission for specific file.
100 *
101 * @param perm String permission to set
102 * @param recursive boolean true to apply to all sub-directories recursively
103 * @param file String file to set
104 * @return String[] containing command and arguments
105 */
106 public static String[] getSetPermissionCommand(String perm, boolean recursive,
107 String file) {
108 String[] baseCmd = getSetPermissionCommand(perm, recursive);
109 String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
110 cmdWithFile[cmdWithFile.length - 1] = file;
111 return cmdWithFile;
112 }
113
114 /** Return a command to set owner */
115 public static String[] getSetOwnerCommand(String owner) {
116 return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
117 : new String[] { "chown", owner };
118 }
119
120 /** Return a command to create symbolic links */
121 public static String[] getSymlinkCommand(String target, String link) {
122 return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
123 : new String[] { "ln", "-s", target, link };
124 }
125
126 /** Return a command for determining if process with specified pid is alive. */
127 public static String[] getCheckProcessIsAliveCommand(String pid) {
128 return Shell.WINDOWS ?
129 new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
130 new String[] { "kill", "-0", isSetsidAvailable ? "-" + pid : pid };
131 }
132
133 /** Return a command to send a signal to a given pid */
134 public static String[] getSignalKillCommand(int code, String pid) {
135 return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid } :
136 new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
137 }
138
139 /** Return a regular expression string that match environment variables */
140 public static String getEnvironmentVariableRegex() {
141 return (WINDOWS) ? "%([A-Za-z_][A-Za-z0-9_]*?)%" :
142 "\\$([A-Za-z_][A-Za-z0-9_]*)";
143 }
144
145 /**
146 * Returns a File referencing a script with the given basename, inside the
147 * given parent directory. The file extension is inferred by platform: ".cmd"
148 * on Windows, or ".sh" otherwise.
149 *
150 * @param parent File parent directory
151 * @param basename String script file basename
152 * @return File referencing the script in the directory
153 */
154 public static File appendScriptExtension(File parent, String basename) {
155 return new File(parent, appendScriptExtension(basename));
156 }
157
158 /**
159 * Returns a script file name with the given basename. The file extension is
160 * inferred by platform: ".cmd" on Windows, or ".sh" otherwise.
161 *
162 * @param basename String script file basename
163 * @return String script file name
164 */
165 public static String appendScriptExtension(String basename) {
166 return basename + (WINDOWS ? ".cmd" : ".sh");
167 }
168
169 /**
170 * Returns a command to run the given script. The script interpreter is
171 * inferred by platform: cmd on Windows or bash otherwise.
172 *
173 * @param script File script to run
174 * @return String[] command to run the script
175 */
176 public static String[] getRunScriptCommand(File script) {
177 String absolutePath = script.getAbsolutePath();
178 return WINDOWS ? new String[] { "cmd", "/c", absolutePath } :
179 new String[] { "/bin/bash", absolutePath };
180 }
181
182 /** a Unix command to set permission */
183 public static final String SET_PERMISSION_COMMAND = "chmod";
184 /** a Unix command to set owner */
185 public static final String SET_OWNER_COMMAND = "chown";
186
187 /** a Unix command to set the change user's groups list */
188 public static final String SET_GROUP_COMMAND = "chgrp";
189 /** a Unix command to create a link */
190 public static final String LINK_COMMAND = "ln";
191 /** a Unix command to get a link target */
192 public static final String READ_LINK_COMMAND = "readlink";
193
194 /**Time after which the executing script would be timedout*/
195 protected long timeOutInterval = 0L;
196 /** If or not script timed out*/
197 private AtomicBoolean timedOut;
198
199
200 /** Centralized logic to discover and validate the sanity of the Hadoop
201 * home directory. Returns either NULL or a directory that exists and
202 * was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV
203 * variable. This does a lot of work so it should only be called
204 * privately for initialization once per process.
205 **/
206 private static String checkHadoopHome() {
207
208 // first check the Dflag hadoop.home.dir with JVM scope
209 String home = System.getProperty("hadoop.home.dir");
210
211 // fall back to the system/user-global env variable
212 if (home == null) {
213 home = System.getenv("HADOOP_HOME");
214 }
215
216 try {
217 // couldn't find either setting for hadoop's home directory
218 if (home == null) {
219 throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
220 }
221
222 if (home.startsWith("\"") && home.endsWith("\"")) {
223 home = home.substring(1, home.length()-1);
224 }
225
226 // check that the home setting is actually a directory that exists
227 File homedir = new File(home);
228 if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
229 throw new IOException("Hadoop home directory " + homedir
230 + " does not exist, is not a directory, or is not an absolute path.");
231 }
232
233 home = homedir.getCanonicalPath();
234
235 } catch (IOException ioe) {
236 if (LOG.isDebugEnabled()) {
237 LOG.debug("Failed to detect a valid hadoop home directory", ioe);
238 }
239 home = null;
240 }
241
242 return home;
243 }
244 private static String HADOOP_HOME_DIR = checkHadoopHome();
245
246 // Public getter, throws an exception if HADOOP_HOME failed validation
247 // checks and is being referenced downstream.
248 public static final String getHadoopHome() throws IOException {
249 if (HADOOP_HOME_DIR == null) {
250 throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
251 }
252
253 return HADOOP_HOME_DIR;
254 }
255
256 /** fully qualify the path to a binary that should be in a known hadoop
257 * bin location. This is primarily useful for disambiguating call-outs
258 * to executable sub-components of Hadoop to avoid clashes with other
259 * executables that may be in the path. Caveat: this call doesn't
260 * just format the path to the bin directory. It also checks for file
261 * existence of the composed path. The output of this call should be
262 * cached by callers.
263 * */
264 public static final String getQualifiedBinPath(String executable)
265 throws IOException {
266 // construct hadoop bin path to the specified executable
267 String fullExeName = HADOOP_HOME_DIR + File.separator + "bin"
268 + File.separator + executable;
269
270 File exeFile = new File(fullExeName);
271 if (!exeFile.exists()) {
272 throw new IOException("Could not locate executable " + fullExeName
273 + " in the Hadoop binaries.");
274 }
275
276 return exeFile.getCanonicalPath();
277 }
278
279 /** Set to true on Windows platforms */
280 public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
281 = System.getProperty("os.name").startsWith("Windows");
282
283 public static final boolean LINUX
284 = System.getProperty("os.name").startsWith("Linux");
285
286 /** a Windows utility to emulate Unix commands */
287 public static final String WINUTILS = getWinUtilsPath();
288
289 public static final String getWinUtilsPath() {
290 String winUtilsPath = null;
291
292 try {
293 if (WINDOWS) {
294 winUtilsPath = getQualifiedBinPath("winutils.exe");
295 }
296 } catch (IOException ioe) {
297 LOG.error("Failed to locate the winutils binary in the hadoop binary path",
298 ioe);
299 }
300
301 return winUtilsPath;
302 }
303
304 public static final boolean isSetsidAvailable = isSetsidSupported();
305 private static boolean isSetsidSupported() {
306 if (Shell.WINDOWS) {
307 return false;
308 }
309 ShellCommandExecutor shexec = null;
310 boolean setsidSupported = true;
311 try {
312 String[] args = {"setsid", "bash", "-c", "echo $$"};
313 shexec = new ShellCommandExecutor(args);
314 shexec.execute();
315 } catch (IOException ioe) {
316 LOG.debug("setsid is not available on this machine. So not using it.");
317 setsidSupported = false;
318 } finally { // handle the exit code
319 if (LOG.isDebugEnabled()) {
320 LOG.debug("setsid exited with exit code "
321 + (shexec != null ? shexec.getExitCode() : "(null executor)"));
322 }
323 }
324 return setsidSupported;
325 }
326
327 /** Token separator regex used to parse Shell tool outputs */
328 public static final String TOKEN_SEPARATOR_REGEX
329 = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
330
331 private long interval; // refresh interval in msec
332 private long lastTime; // last time the command was performed
333 private Map<String, String> environment; // env for the command execution
334 private File dir;
335 private Process process; // sub process used to execute the command
336 private int exitCode;
337
338 /**If or not script finished executing*/
339 private volatile AtomicBoolean completed;
340
341 public Shell() {
342 this(0L);
343 }
344
345 /**
346 * @param interval the minimum duration to wait before re-executing the
347 * command.
348 */
349 public Shell( long interval ) {
350 this.interval = interval;
351 this.lastTime = (interval<0) ? 0 : -interval;
352 }
353
354 /** set the environment for the command
355 * @param env Mapping of environment variables
356 */
357 protected void setEnvironment(Map<String, String> env) {
358 this.environment = env;
359 }
360
361 /** set the working directory
362 * @param dir The directory where the command would be executed
363 */
364 protected void setWorkingDirectory(File dir) {
365 this.dir = dir;
366 }
367
368 /** check to see if a command needs to be executed and execute if needed */
369 protected void run() throws IOException {
370 if (lastTime + interval > Time.now())
371 return;
372 exitCode = 0; // reset for next run
373 runCommand();
374 }
375
376 /** Run a command */
377 private void runCommand() throws IOException {
378 ProcessBuilder builder = new ProcessBuilder(getExecString());
379 Timer timeOutTimer = null;
380 ShellTimeoutTimerTask timeoutTimerTask = null;
381 timedOut = new AtomicBoolean(false);
382 completed = new AtomicBoolean(false);
383
384 if (environment != null) {
385 builder.environment().putAll(this.environment);
386 }
387 if (dir != null) {
388 builder.directory(this.dir);
389 }
390
391 if (Shell.WINDOWS) {
392 synchronized (WindowsProcessLaunchLock) {
393 // To workaround the race condition issue with child processes
394 // inheriting unintended handles during process launch that can
395 // lead to hangs on reading output and error streams, we
396 // serialize process creation. More info available at:
397 // http://support.microsoft.com/kb/315939
398 process = builder.start();
399 }
400 } else {
401 process = builder.start();
402 }
403
404 if (timeOutInterval > 0) {
405 timeOutTimer = new Timer("Shell command timeout");
406 timeoutTimerTask = new ShellTimeoutTimerTask(
407 this);
408 //One time scheduling.
409 timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
410 }
411 final BufferedReader errReader =
412 new BufferedReader(new InputStreamReader(process
413 .getErrorStream()));
414 BufferedReader inReader =
415 new BufferedReader(new InputStreamReader(process
416 .getInputStream()));
417 final StringBuffer errMsg = new StringBuffer();
418
419 // read error and input streams as this would free up the buffers
420 // free the error stream buffer
421 Thread errThread = new Thread() {
422 @Override
423 public void run() {
424 try {
425 String line = errReader.readLine();
426 while((line != null) && !isInterrupted()) {
427 errMsg.append(line);
428 errMsg.append(System.getProperty("line.separator"));
429 line = errReader.readLine();
430 }
431 } catch(IOException ioe) {
432 LOG.warn("Error reading the error stream", ioe);
433 }
434 }
435 };
436 try {
437 errThread.start();
438 } catch (IllegalStateException ise) { }
439 try {
440 parseExecResult(inReader); // parse the output
441 // clear the input stream buffer
442 String line = inReader.readLine();
443 while(line != null) {
444 line = inReader.readLine();
445 }
446 // wait for the process to finish and check the exit code
447 exitCode = process.waitFor();
448 try {
449 // make sure that the error thread exits
450 errThread.join();
451 } catch (InterruptedException ie) {
452 LOG.warn("Interrupted while reading the error stream", ie);
453 }
454 completed.set(true);
455 //the timeout thread handling
456 //taken care in finally block
457 if (exitCode != 0) {
458 throw new ExitCodeException(exitCode, errMsg.toString());
459 }
460 } catch (InterruptedException ie) {
461 throw new IOException(ie.toString());
462 } finally {
463 if (timeOutTimer != null) {
464 timeOutTimer.cancel();
465 }
466 // close the input stream
467 try {
468 inReader.close();
469 } catch (IOException ioe) {
470 LOG.warn("Error while closing the input stream", ioe);
471 }
472 if (!completed.get()) {
473 errThread.interrupt();
474 }
475 try {
476 errReader.close();
477 } catch (IOException ioe) {
478 LOG.warn("Error while closing the error stream", ioe);
479 }
480 process.destroy();
481 lastTime = Time.now();
482 }
483 }
484
485 /** return an array containing the command name & its parameters */
486 protected abstract String[] getExecString();
487
488 /** Parse the execution result */
489 protected abstract void parseExecResult(BufferedReader lines)
490 throws IOException;
491
492 /** get the current sub-process executing the given command
493 * @return process executing the command
494 */
495 public Process getProcess() {
496 return process;
497 }
498
499 /** get the exit code
500 * @return the exit code of the process
501 */
502 public int getExitCode() {
503 return exitCode;
504 }
505
506 /**
507 * This is an IOException with exit code added.
508 */
509 public static class ExitCodeException extends IOException {
510 int exitCode;
511
512 public ExitCodeException(int exitCode, String message) {
513 super(message);
514 this.exitCode = exitCode;
515 }
516
517 public int getExitCode() {
518 return exitCode;
519 }
520 }
521
522 /**
523 * A simple shell command executor.
524 *
525 * <code>ShellCommandExecutor</code>should be used in cases where the output
526 * of the command needs no explicit parsing and where the command, working
527 * directory and the environment remains unchanged. The output of the command
528 * is stored as-is and is expected to be small.
529 */
530 public static class ShellCommandExecutor extends Shell {
531
532 private String[] command;
533 private StringBuffer output;
534
535
536 public ShellCommandExecutor(String[] execString) {
537 this(execString, null);
538 }
539
540 public ShellCommandExecutor(String[] execString, File dir) {
541 this(execString, dir, null);
542 }
543
544 public ShellCommandExecutor(String[] execString, File dir,
545 Map<String, String> env) {
546 this(execString, dir, env , 0L);
547 }
548
549 /**
550 * Create a new instance of the ShellCommandExecutor to execute a command.
551 *
552 * @param execString The command to execute with arguments
553 * @param dir If not-null, specifies the directory which should be set
554 * as the current working directory for the command.
555 * If null, the current working directory is not modified.
556 * @param env If not-null, environment of the command will include the
557 * key-value pairs specified in the map. If null, the current
558 * environment is not modified.
559 * @param timeout Specifies the time in milliseconds, after which the
560 * command will be killed and the status marked as timedout.
561 * If 0, the command will not be timed out.
562 */
563 public ShellCommandExecutor(String[] execString, File dir,
564 Map<String, String> env, long timeout) {
565 command = execString.clone();
566 if (dir != null) {
567 setWorkingDirectory(dir);
568 }
569 if (env != null) {
570 setEnvironment(env);
571 }
572 timeOutInterval = timeout;
573 }
574
575
576 /** Execute the shell command. */
577 public void execute() throws IOException {
578 this.run();
579 }
580
581 @Override
582 public String[] getExecString() {
583 return command;
584 }
585
586 @Override
587 protected void parseExecResult(BufferedReader lines) throws IOException {
588 output = new StringBuffer();
589 char[] buf = new char[512];
590 int nRead;
591 while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
592 output.append(buf, 0, nRead);
593 }
594 }
595
596 /** Get the output of the shell command.*/
597 public String getOutput() {
598 return (output == null) ? "" : output.toString();
599 }
600
601 /**
602 * Returns the commands of this instance.
603 * Arguments with spaces in are presented with quotes round; other
604 * arguments are presented raw
605 *
606 * @return a string representation of the object.
607 */
608 @Override
609 public String toString() {
610 StringBuilder builder = new StringBuilder();
611 String[] args = getExecString();
612 for (String s : args) {
613 if (s.indexOf(' ') >= 0) {
614 builder.append('"').append(s).append('"');
615 } else {
616 builder.append(s);
617 }
618 builder.append(' ');
619 }
620 return builder.toString();
621 }
622 }
623
624 /**
625 * To check if the passed script to shell command executor timed out or
626 * not.
627 *
628 * @return if the script timed out.
629 */
630 public boolean isTimedOut() {
631 return timedOut.get();
632 }
633
634 /**
635 * Set if the command has timed out.
636 *
637 */
638 private void setTimedOut() {
639 this.timedOut.set(true);
640 }
641
642 /**
643 * Static method to execute a shell command.
644 * Covers most of the simple cases without requiring the user to implement
645 * the <code>Shell</code> interface.
646 * @param cmd shell command to execute.
647 * @return the output of the executed command.
648 */
649 public static String execCommand(String ... cmd) throws IOException {
650 return execCommand(null, cmd, 0L);
651 }
652
653 /**
654 * Static method to execute a shell command.
655 * Covers most of the simple cases without requiring the user to implement
656 * the <code>Shell</code> interface.
657 * @param env the map of environment key=value
658 * @param cmd shell command to execute.
659 * @param timeout time in milliseconds after which script should be marked timeout
660 * @return the output of the executed command.o
661 */
662
663 public static String execCommand(Map<String, String> env, String[] cmd,
664 long timeout) throws IOException {
665 ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env,
666 timeout);
667 exec.execute();
668 return exec.getOutput();
669 }
670
671 /**
672 * Static method to execute a shell command.
673 * Covers most of the simple cases without requiring the user to implement
674 * the <code>Shell</code> interface.
675 * @param env the map of environment key=value
676 * @param cmd shell command to execute.
677 * @return the output of the executed command.
678 */
679 public static String execCommand(Map<String,String> env, String ... cmd)
680 throws IOException {
681 return execCommand(env, cmd, 0L);
682 }
683
684 /**
685 * Timer which is used to timeout scripts spawned off by shell.
686 */
687 private static class ShellTimeoutTimerTask extends TimerTask {
688
689 private Shell shell;
690
691 public ShellTimeoutTimerTask(Shell shell) {
692 this.shell = shell;
693 }
694
695 @Override
696 public void run() {
697 Process p = shell.getProcess();
698 try {
699 p.exitValue();
700 } catch (Exception e) {
701 //Process has not terminated.
702 //So check if it has completed
703 //if not just destroy it.
704 if (p != null && !shell.completed.get()) {
705 shell.setTimedOut();
706 p.destroy();
707 }
708 }
709 }
710 }
711 }