001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.util;
019
020 import java.io.BufferedReader;
021 import java.io.File;
022 import java.io.IOException;
023 import java.io.InputStreamReader;
024 import java.util.Map;
025 import java.util.Timer;
026 import java.util.TimerTask;
027 import java.util.concurrent.atomic.AtomicBoolean;
028
029 import org.apache.commons.logging.Log;
030 import org.apache.commons.logging.LogFactory;
031 import org.apache.hadoop.classification.InterfaceAudience;
032 import org.apache.hadoop.classification.InterfaceStability;
033 import org.apache.hadoop.conf.Configuration;
034
035 /**
036 * A base class for running a Unix command.
037 *
038 * <code>Shell</code> can be used to run unix commands like <code>du</code> or
039 * <code>df</code>. It also offers facilities to gate commands by
040 * time-intervals.
041 */
042 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
043 @InterfaceStability.Unstable
044 abstract public class Shell {
045
046 public static final Log LOG = LogFactory.getLog(Shell.class);
047
048 /** a Unix command to get the current user's name */
049 public final static String USER_NAME_COMMAND = "whoami";
050 /** a Unix command to get the current user's groups list */
051 public static String[] getGroupsCommand() {
052 return new String[]{"bash", "-c", "groups"};
053 }
054 /** a Unix command to get a given user's groups list */
055 public static String[] getGroupsForUserCommand(final String user) {
056 //'groups username' command return is non-consistent across different unixes
057 return new String [] {"bash", "-c", "id -Gn " + user};
058 }
059 /** a Unix command to get a given netgroup's user list */
060 public static String[] getUsersForNetgroupCommand(final String netgroup) {
061 //'groups username' command return is non-consistent across different unixes
062 return new String [] {"bash", "-c", "getent netgroup " + netgroup};
063 }
064 /** a Unix command to set permission */
065 public static final String SET_PERMISSION_COMMAND = "chmod";
066 /** a Unix command to set owner */
067 public static final String SET_OWNER_COMMAND = "chown";
068 public static final String SET_GROUP_COMMAND = "chgrp";
069 /** a Unix command to create a link */
070 public static final String LINK_COMMAND = "ln";
071 /** a Unix command to get a link target */
072 public static final String READ_LINK_COMMAND = "readlink";
073 /** Return a Unix command to get permission information. */
074 public static String[] getGET_PERMISSION_COMMAND() {
075 //force /bin/ls, except on windows.
076 return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"};
077 }
078
079 /**Time after which the executing script would be timedout*/
080 protected long timeOutInterval = 0L;
081 /** If or not script timed out*/
082 private AtomicBoolean timedOut;
083
084 /** a Unix command to get ulimit of a process. */
085 public static final String ULIMIT_COMMAND = "ulimit";
086
087 /**
088 * Get the Unix command for setting the maximum virtual memory available
089 * to a given child process. This is only relevant when we are forking a
090 * process from within the Mapper or the Reducer implementations.
091 * Also see Hadoop Pipes and Hadoop Streaming.
092 *
093 * It also checks to ensure that we are running on a *nix platform else
094 * (e.g. in Cygwin/Windows) it returns <code>null</code>.
095 * @param memoryLimit virtual memory limit
096 * @return a <code>String[]</code> with the ulimit command arguments or
097 * <code>null</code> if we are running on a non *nix platform or
098 * if the limit is unspecified.
099 */
100 public static String[] getUlimitMemoryCommand(int memoryLimit) {
101 // ulimit isn't supported on Windows
102 if (WINDOWS) {
103 return null;
104 }
105
106 return new String[] {ULIMIT_COMMAND, "-v", String.valueOf(memoryLimit)};
107 }
108
109 /**
110 * Get the Unix command for setting the maximum virtual memory available
111 * to a given child process. This is only relevant when we are forking a
112 * process from within the Mapper or the Reducer implementations.
113 * see also Hadoop Pipes and Streaming.
114 *
115 * It also checks to ensure that we are running on a *nix platform else
116 * (e.g. in Cygwin/Windows) it returns <code>null</code>.
117 * @param conf configuration
118 * @return a <code>String[]</code> with the ulimit command arguments or
119 * <code>null</code> if we are running on a non *nix platform or
120 * if the limit is unspecified.
121 * @deprecated Use {@link #getUlimitMemoryCommand(int)}
122 */
123 @Deprecated
124 public static String[] getUlimitMemoryCommand(Configuration conf) {
125 // ulimit isn't supported on Windows
126 if (WINDOWS) {
127 return null;
128 }
129
130 // get the memory limit from the configuration
131 String ulimit = conf.get("mapred.child.ulimit");
132 if (ulimit == null) {
133 return null;
134 }
135
136 // Parse it to ensure it is legal/sane
137 int memoryLimit = Integer.valueOf(ulimit);
138
139 return getUlimitMemoryCommand(memoryLimit);
140 }
141
142 /** Set to true on Windows platforms */
143 public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
144 = System.getProperty("os.name").startsWith("Windows");
145
146 private long interval; // refresh interval in msec
147 private long lastTime; // last time the command was performed
148 private Map<String, String> environment; // env for the command execution
149 private File dir;
150 private Process process; // sub process used to execute the command
151 private int exitCode;
152
153 /**If or not script finished executing*/
154 private volatile AtomicBoolean completed;
155
156 public Shell() {
157 this(0L);
158 }
159
160 /**
161 * @param interval the minimum duration to wait before re-executing the
162 * command.
163 */
164 public Shell( long interval ) {
165 this.interval = interval;
166 this.lastTime = (interval<0) ? 0 : -interval;
167 }
168
169 /** set the environment for the command
170 * @param env Mapping of environment variables
171 */
172 protected void setEnvironment(Map<String, String> env) {
173 this.environment = env;
174 }
175
176 /** set the working directory
177 * @param dir The directory where the command would be executed
178 */
179 protected void setWorkingDirectory(File dir) {
180 this.dir = dir;
181 }
182
183 /** check to see if a command needs to be executed and execute if needed */
184 protected void run() throws IOException {
185 if (lastTime + interval > System.currentTimeMillis())
186 return;
187 exitCode = 0; // reset for next run
188 runCommand();
189 }
190
191 /** Run a command */
192 private void runCommand() throws IOException {
193 ProcessBuilder builder = new ProcessBuilder(getExecString());
194 Timer timeOutTimer = null;
195 ShellTimeoutTimerTask timeoutTimerTask = null;
196 timedOut = new AtomicBoolean(false);
197 completed = new AtomicBoolean(false);
198
199 if (environment != null) {
200 builder.environment().putAll(this.environment);
201 }
202 if (dir != null) {
203 builder.directory(this.dir);
204 }
205
206 process = builder.start();
207 if (timeOutInterval > 0) {
208 timeOutTimer = new Timer("Shell command timeout");
209 timeoutTimerTask = new ShellTimeoutTimerTask(
210 this);
211 //One time scheduling.
212 timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
213 }
214 final BufferedReader errReader =
215 new BufferedReader(new InputStreamReader(process
216 .getErrorStream()));
217 BufferedReader inReader =
218 new BufferedReader(new InputStreamReader(process
219 .getInputStream()));
220 final StringBuffer errMsg = new StringBuffer();
221
222 // read error and input streams as this would free up the buffers
223 // free the error stream buffer
224 Thread errThread = new Thread() {
225 @Override
226 public void run() {
227 try {
228 String line = errReader.readLine();
229 while((line != null) && !isInterrupted()) {
230 errMsg.append(line);
231 errMsg.append(System.getProperty("line.separator"));
232 line = errReader.readLine();
233 }
234 } catch(IOException ioe) {
235 LOG.warn("Error reading the error stream", ioe);
236 }
237 }
238 };
239 try {
240 errThread.start();
241 } catch (IllegalStateException ise) { }
242 try {
243 parseExecResult(inReader); // parse the output
244 // clear the input stream buffer
245 String line = inReader.readLine();
246 while(line != null) {
247 line = inReader.readLine();
248 }
249 // wait for the process to finish and check the exit code
250 exitCode = process.waitFor();
251 try {
252 // make sure that the error thread exits
253 errThread.join();
254 } catch (InterruptedException ie) {
255 LOG.warn("Interrupted while reading the error stream", ie);
256 }
257 completed.set(true);
258 //the timeout thread handling
259 //taken care in finally block
260 if (exitCode != 0) {
261 throw new ExitCodeException(exitCode, errMsg.toString());
262 }
263 } catch (InterruptedException ie) {
264 throw new IOException(ie.toString());
265 } finally {
266 if (timeOutTimer != null) {
267 timeOutTimer.cancel();
268 }
269 // close the input stream
270 try {
271 inReader.close();
272 } catch (IOException ioe) {
273 LOG.warn("Error while closing the input stream", ioe);
274 }
275 if (!completed.get()) {
276 errThread.interrupt();
277 }
278 try {
279 errReader.close();
280 } catch (IOException ioe) {
281 LOG.warn("Error while closing the error stream", ioe);
282 }
283 process.destroy();
284 lastTime = System.currentTimeMillis();
285 }
286 }
287
288 /** return an array containing the command name & its parameters */
289 protected abstract String[] getExecString();
290
291 /** Parse the execution result */
292 protected abstract void parseExecResult(BufferedReader lines)
293 throws IOException;
294
295 /** get the current sub-process executing the given command
296 * @return process executing the command
297 */
298 public Process getProcess() {
299 return process;
300 }
301
302 /** get the exit code
303 * @return the exit code of the process
304 */
305 public int getExitCode() {
306 return exitCode;
307 }
308
309 /**
310 * This is an IOException with exit code added.
311 */
312 public static class ExitCodeException extends IOException {
313 int exitCode;
314
315 public ExitCodeException(int exitCode, String message) {
316 super(message);
317 this.exitCode = exitCode;
318 }
319
320 public int getExitCode() {
321 return exitCode;
322 }
323 }
324
325 /**
326 * A simple shell command executor.
327 *
328 * <code>ShellCommandExecutor</code>should be used in cases where the output
329 * of the command needs no explicit parsing and where the command, working
330 * directory and the environment remains unchanged. The output of the command
331 * is stored as-is and is expected to be small.
332 */
333 public static class ShellCommandExecutor extends Shell {
334
335 private String[] command;
336 private StringBuffer output;
337
338
339 public ShellCommandExecutor(String[] execString) {
340 this(execString, null);
341 }
342
343 public ShellCommandExecutor(String[] execString, File dir) {
344 this(execString, dir, null);
345 }
346
347 public ShellCommandExecutor(String[] execString, File dir,
348 Map<String, String> env) {
349 this(execString, dir, env , 0L);
350 }
351
352 /**
353 * Create a new instance of the ShellCommandExecutor to execute a command.
354 *
355 * @param execString The command to execute with arguments
356 * @param dir If not-null, specifies the directory which should be set
357 * as the current working directory for the command.
358 * If null, the current working directory is not modified.
359 * @param env If not-null, environment of the command will include the
360 * key-value pairs specified in the map. If null, the current
361 * environment is not modified.
362 * @param timeout Specifies the time in milliseconds, after which the
363 * command will be killed and the status marked as timedout.
364 * If 0, the command will not be timed out.
365 */
366 public ShellCommandExecutor(String[] execString, File dir,
367 Map<String, String> env, long timeout) {
368 command = execString.clone();
369 if (dir != null) {
370 setWorkingDirectory(dir);
371 }
372 if (env != null) {
373 setEnvironment(env);
374 }
375 timeOutInterval = timeout;
376 }
377
378
379 /** Execute the shell command. */
380 public void execute() throws IOException {
381 this.run();
382 }
383
384 public String[] getExecString() {
385 return command;
386 }
387
388 protected void parseExecResult(BufferedReader lines) throws IOException {
389 output = new StringBuffer();
390 char[] buf = new char[512];
391 int nRead;
392 while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
393 output.append(buf, 0, nRead);
394 }
395 }
396
397 /** Get the output of the shell command.*/
398 public String getOutput() {
399 return (output == null) ? "" : output.toString();
400 }
401
402 /**
403 * Returns the commands of this instance.
404 * Arguments with spaces in are presented with quotes round; other
405 * arguments are presented raw
406 *
407 * @return a string representation of the object.
408 */
409 public String toString() {
410 StringBuilder builder = new StringBuilder();
411 String[] args = getExecString();
412 for (String s : args) {
413 if (s.indexOf(' ') >= 0) {
414 builder.append('"').append(s).append('"');
415 } else {
416 builder.append(s);
417 }
418 builder.append(' ');
419 }
420 return builder.toString();
421 }
422 }
423
424 /**
425 * To check if the passed script to shell command executor timed out or
426 * not.
427 *
428 * @return if the script timed out.
429 */
430 public boolean isTimedOut() {
431 return timedOut.get();
432 }
433
434 /**
435 * Set if the command has timed out.
436 *
437 */
438 private void setTimedOut() {
439 this.timedOut.set(true);
440 }
441
442 /**
443 * Static method to execute a shell command.
444 * Covers most of the simple cases without requiring the user to implement
445 * the <code>Shell</code> interface.
446 * @param cmd shell command to execute.
447 * @return the output of the executed command.
448 */
449 public static String execCommand(String ... cmd) throws IOException {
450 return execCommand(null, cmd, 0L);
451 }
452
453 /**
454 * Static method to execute a shell command.
455 * Covers most of the simple cases without requiring the user to implement
456 * the <code>Shell</code> interface.
457 * @param env the map of environment key=value
458 * @param cmd shell command to execute.
459 * @param timeout time in milliseconds after which script should be marked timeout
460 * @return the output of the executed command.o
461 */
462
463 public static String execCommand(Map<String, String> env, String[] cmd,
464 long timeout) throws IOException {
465 ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env,
466 timeout);
467 exec.execute();
468 return exec.getOutput();
469 }
470
471 /**
472 * Static method to execute a shell command.
473 * Covers most of the simple cases without requiring the user to implement
474 * the <code>Shell</code> interface.
475 * @param env the map of environment key=value
476 * @param cmd shell command to execute.
477 * @return the output of the executed command.
478 */
479 public static String execCommand(Map<String,String> env, String ... cmd)
480 throws IOException {
481 return execCommand(env, cmd, 0L);
482 }
483
484 /**
485 * Timer which is used to timeout scripts spawned off by shell.
486 */
487 private static class ShellTimeoutTimerTask extends TimerTask {
488
489 private Shell shell;
490
491 public ShellTimeoutTimerTask(Shell shell) {
492 this.shell = shell;
493 }
494
495 @Override
496 public void run() {
497 Process p = shell.getProcess();
498 try {
499 p.exitValue();
500 } catch (Exception e) {
501 //Process has not terminated.
502 //So check if it has completed
503 //if not just destroy it.
504 if (p != null && !shell.completed.get()) {
505 shell.setTimedOut();
506 p.destroy();
507 }
508 }
509 }
510 }
511 }