001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.*;
022 import java.util.ArrayList;
023 import java.util.Arrays;
024 import java.util.Enumeration;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.jar.Attributes;
028 import java.util.jar.JarOutputStream;
029 import java.util.jar.Manifest;
030 import java.util.zip.GZIPInputStream;
031 import java.util.zip.ZipEntry;
032 import java.util.zip.ZipFile;
033
034 import org.apache.commons.collections.map.CaseInsensitiveMap;
035 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
036 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
037 import org.apache.hadoop.classification.InterfaceAudience;
038 import org.apache.hadoop.classification.InterfaceStability;
039 import org.apache.hadoop.conf.Configuration;
040 import org.apache.hadoop.fs.permission.FsAction;
041 import org.apache.hadoop.fs.permission.FsPermission;
042 import org.apache.hadoop.io.IOUtils;
043 import org.apache.hadoop.io.nativeio.NativeIO;
044 import org.apache.hadoop.util.StringUtils;
045 import org.apache.hadoop.util.Shell;
046 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
047 import org.apache.commons.logging.Log;
048 import org.apache.commons.logging.LogFactory;
049
050 /**
051 * A collection of file-processing util methods
052 */
053 @InterfaceAudience.Public
054 @InterfaceStability.Evolving
055 public class FileUtil {
056
057 private static final Log LOG = LogFactory.getLog(FileUtil.class);
058
059 /* The error code is defined in winutils to indicate insufficient
060 * privilege to create symbolic links. This value need to keep in
061 * sync with the constant of the same name in:
062 * "src\winutils\common.h"
063 * */
064 public static final int SYMLINK_NO_PRIVILEGE = 2;
065
066 /**
067 * convert an array of FileStatus to an array of Path
068 *
069 * @param stats
070 * an array of FileStatus objects
071 * @return an array of paths corresponding to the input
072 */
073 public static Path[] stat2Paths(FileStatus[] stats) {
074 if (stats == null)
075 return null;
076 Path[] ret = new Path[stats.length];
077 for (int i = 0; i < stats.length; ++i) {
078 ret[i] = stats[i].getPath();
079 }
080 return ret;
081 }
082
083 /**
084 * convert an array of FileStatus to an array of Path.
085 * If stats if null, return path
086 * @param stats
087 * an array of FileStatus objects
088 * @param path
089 * default path to return in stats is null
090 * @return an array of paths corresponding to the input
091 */
092 public static Path[] stat2Paths(FileStatus[] stats, Path path) {
093 if (stats == null)
094 return new Path[]{path};
095 else
096 return stat2Paths(stats);
097 }
098
099 /**
100 * Delete a directory and all its contents. If
101 * we return false, the directory may be partially-deleted.
102 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
103 * to by the symlink is not deleted.
104 * (2) If dir is symlink to a directory, symlink is deleted. The directory
105 * pointed to by symlink is not deleted.
106 * (3) If dir is a normal file, it is deleted.
107 * (4) If dir is a normal directory, then dir and all its contents recursively
108 * are deleted.
109 */
110 public static boolean fullyDelete(final File dir) {
111 return fullyDelete(dir, false);
112 }
113
114 /**
115 * Delete a directory and all its contents. If
116 * we return false, the directory may be partially-deleted.
117 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
118 * to by the symlink is not deleted.
119 * (2) If dir is symlink to a directory, symlink is deleted. The directory
120 * pointed to by symlink is not deleted.
121 * (3) If dir is a normal file, it is deleted.
122 * (4) If dir is a normal directory, then dir and all its contents recursively
123 * are deleted.
124 * @param dir the file or directory to be deleted
125 * @param tryGrantPermissions true if permissions should be modified to delete a file.
126 * @return true on success false on failure.
127 */
128 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
129 if (tryGrantPermissions) {
130 // try to chmod +rwx the parent folder of the 'dir':
131 File parent = dir.getParentFile();
132 grantPermissions(parent);
133 }
134 if (deleteImpl(dir, false)) {
135 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
136 // (d) symlink to a directory
137 return true;
138 }
139 // handle nonempty directory deletion
140 if (!fullyDeleteContents(dir, tryGrantPermissions)) {
141 return false;
142 }
143 return deleteImpl(dir, true);
144 }
145
146 /*
147 * Pure-Java implementation of "chmod +rwx f".
148 */
149 private static void grantPermissions(final File f) {
150 FileUtil.setExecutable(f, true);
151 FileUtil.setReadable(f, true);
152 FileUtil.setWritable(f, true);
153 }
154
155 private static boolean deleteImpl(final File f, final boolean doLog) {
156 if (f == null) {
157 LOG.warn("null file argument.");
158 return false;
159 }
160 final boolean wasDeleted = f.delete();
161 if (wasDeleted) {
162 return true;
163 }
164 final boolean ex = f.exists();
165 if (doLog && ex) {
166 LOG.warn("Failed to delete file or dir ["
167 + f.getAbsolutePath() + "]: it still exists.");
168 }
169 return !ex;
170 }
171
172 /**
173 * Delete the contents of a directory, not the directory itself. If
174 * we return false, the directory may be partially-deleted.
175 * If dir is a symlink to a directory, all the contents of the actual
176 * directory pointed to by dir will be deleted.
177 */
178 public static boolean fullyDeleteContents(final File dir) {
179 return fullyDeleteContents(dir, false);
180 }
181
182 /**
183 * Delete the contents of a directory, not the directory itself. If
184 * we return false, the directory may be partially-deleted.
185 * If dir is a symlink to a directory, all the contents of the actual
186 * directory pointed to by dir will be deleted.
187 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
188 * and all the underlying directories before trying to delete their contents.
189 */
190 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
191 if (tryGrantPermissions) {
192 // to be able to list the dir and delete files from it
193 // we must grant the dir rwx permissions:
194 grantPermissions(dir);
195 }
196 boolean deletionSucceeded = true;
197 final File[] contents = dir.listFiles();
198 if (contents != null) {
199 for (int i = 0; i < contents.length; i++) {
200 if (contents[i].isFile()) {
201 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
202 deletionSucceeded = false;
203 continue; // continue deletion of other files/dirs under dir
204 }
205 } else {
206 // Either directory or symlink to another directory.
207 // Try deleting the directory as this might be a symlink
208 boolean b = false;
209 b = deleteImpl(contents[i], false);
210 if (b){
211 //this was indeed a symlink or an empty directory
212 continue;
213 }
214 // if not an empty directory or symlink let
215 // fullydelete handle it.
216 if (!fullyDelete(contents[i], tryGrantPermissions)) {
217 deletionSucceeded = false;
218 // continue deletion of other files/dirs under dir
219 }
220 }
221 }
222 }
223 return deletionSucceeded;
224 }
225
226 /**
227 * Recursively delete a directory.
228 *
229 * @param fs {@link FileSystem} on which the path is present
230 * @param dir directory to recursively delete
231 * @throws IOException
232 * @deprecated Use {@link FileSystem#delete(Path, boolean)}
233 */
234 @Deprecated
235 public static void fullyDelete(FileSystem fs, Path dir)
236 throws IOException {
237 fs.delete(dir, true);
238 }
239
240 //
241 // If the destination is a subdirectory of the source, then
242 // generate exception
243 //
244 private static void checkDependencies(FileSystem srcFS,
245 Path src,
246 FileSystem dstFS,
247 Path dst)
248 throws IOException {
249 if (srcFS == dstFS) {
250 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
251 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
252 if (dstq.startsWith(srcq)) {
253 if (srcq.length() == dstq.length()) {
254 throw new IOException("Cannot copy " + src + " to itself.");
255 } else {
256 throw new IOException("Cannot copy " + src + " to its subdirectory " +
257 dst);
258 }
259 }
260 }
261 }
262
263 /** Copy files between FileSystems. */
264 public static boolean copy(FileSystem srcFS, Path src,
265 FileSystem dstFS, Path dst,
266 boolean deleteSource,
267 Configuration conf) throws IOException {
268 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
269 }
270
271 public static boolean copy(FileSystem srcFS, Path[] srcs,
272 FileSystem dstFS, Path dst,
273 boolean deleteSource,
274 boolean overwrite, Configuration conf)
275 throws IOException {
276 boolean gotException = false;
277 boolean returnVal = true;
278 StringBuilder exceptions = new StringBuilder();
279
280 if (srcs.length == 1)
281 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
282
283 // Check if dest is directory
284 if (!dstFS.exists(dst)) {
285 throw new IOException("`" + dst +"': specified destination directory " +
286 "does not exist");
287 } else {
288 FileStatus sdst = dstFS.getFileStatus(dst);
289 if (!sdst.isDirectory())
290 throw new IOException("copying multiple files, but last argument `" +
291 dst + "' is not a directory");
292 }
293
294 for (Path src : srcs) {
295 try {
296 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
297 returnVal = false;
298 } catch (IOException e) {
299 gotException = true;
300 exceptions.append(e.getMessage());
301 exceptions.append("\n");
302 }
303 }
304 if (gotException) {
305 throw new IOException(exceptions.toString());
306 }
307 return returnVal;
308 }
309
310 /** Copy files between FileSystems. */
311 public static boolean copy(FileSystem srcFS, Path src,
312 FileSystem dstFS, Path dst,
313 boolean deleteSource,
314 boolean overwrite,
315 Configuration conf) throws IOException {
316 FileStatus fileStatus = srcFS.getFileStatus(src);
317 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
318 }
319
320 /** Copy files between FileSystems. */
321 private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
322 FileSystem dstFS, Path dst,
323 boolean deleteSource,
324 boolean overwrite,
325 Configuration conf) throws IOException {
326 Path src = srcStatus.getPath();
327 dst = checkDest(src.getName(), dstFS, dst, overwrite);
328 if (srcStatus.isDirectory()) {
329 checkDependencies(srcFS, src, dstFS, dst);
330 if (!dstFS.mkdirs(dst)) {
331 return false;
332 }
333 FileStatus contents[] = srcFS.listStatus(src);
334 for (int i = 0; i < contents.length; i++) {
335 copy(srcFS, contents[i], dstFS,
336 new Path(dst, contents[i].getPath().getName()),
337 deleteSource, overwrite, conf);
338 }
339 } else {
340 InputStream in=null;
341 OutputStream out = null;
342 try {
343 in = srcFS.open(src);
344 out = dstFS.create(dst, overwrite);
345 IOUtils.copyBytes(in, out, conf, true);
346 } catch (IOException e) {
347 IOUtils.closeStream(out);
348 IOUtils.closeStream(in);
349 throw e;
350 }
351 }
352 if (deleteSource) {
353 return srcFS.delete(src, true);
354 } else {
355 return true;
356 }
357
358 }
359
360 /** Copy all files in a directory to one output file (merge). */
361 public static boolean copyMerge(FileSystem srcFS, Path srcDir,
362 FileSystem dstFS, Path dstFile,
363 boolean deleteSource,
364 Configuration conf, String addString) throws IOException {
365 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
366
367 if (!srcFS.getFileStatus(srcDir).isDirectory())
368 return false;
369
370 OutputStream out = dstFS.create(dstFile);
371
372 try {
373 FileStatus contents[] = srcFS.listStatus(srcDir);
374 Arrays.sort(contents);
375 for (int i = 0; i < contents.length; i++) {
376 if (contents[i].isFile()) {
377 InputStream in = srcFS.open(contents[i].getPath());
378 try {
379 IOUtils.copyBytes(in, out, conf, false);
380 if (addString!=null)
381 out.write(addString.getBytes("UTF-8"));
382
383 } finally {
384 in.close();
385 }
386 }
387 }
388 } finally {
389 out.close();
390 }
391
392
393 if (deleteSource) {
394 return srcFS.delete(srcDir, true);
395 } else {
396 return true;
397 }
398 }
399
400 /** Copy local files to a FileSystem. */
401 public static boolean copy(File src,
402 FileSystem dstFS, Path dst,
403 boolean deleteSource,
404 Configuration conf) throws IOException {
405 dst = checkDest(src.getName(), dstFS, dst, false);
406
407 if (src.isDirectory()) {
408 if (!dstFS.mkdirs(dst)) {
409 return false;
410 }
411 File contents[] = listFiles(src);
412 for (int i = 0; i < contents.length; i++) {
413 copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
414 deleteSource, conf);
415 }
416 } else if (src.isFile()) {
417 InputStream in = null;
418 OutputStream out =null;
419 try {
420 in = new FileInputStream(src);
421 out = dstFS.create(dst);
422 IOUtils.copyBytes(in, out, conf);
423 } catch (IOException e) {
424 IOUtils.closeStream( out );
425 IOUtils.closeStream( in );
426 throw e;
427 }
428 } else {
429 throw new IOException(src.toString() +
430 ": No such file or directory");
431 }
432 if (deleteSource) {
433 return FileUtil.fullyDelete(src);
434 } else {
435 return true;
436 }
437 }
438
439 /** Copy FileSystem files to local files. */
440 public static boolean copy(FileSystem srcFS, Path src,
441 File dst, boolean deleteSource,
442 Configuration conf) throws IOException {
443 FileStatus filestatus = srcFS.getFileStatus(src);
444 return copy(srcFS, filestatus, dst, deleteSource, conf);
445 }
446
447 /** Copy FileSystem files to local files. */
448 private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
449 File dst, boolean deleteSource,
450 Configuration conf) throws IOException {
451 Path src = srcStatus.getPath();
452 if (srcStatus.isDirectory()) {
453 if (!dst.mkdirs()) {
454 return false;
455 }
456 FileStatus contents[] = srcFS.listStatus(src);
457 for (int i = 0; i < contents.length; i++) {
458 copy(srcFS, contents[i],
459 new File(dst, contents[i].getPath().getName()),
460 deleteSource, conf);
461 }
462 } else {
463 InputStream in = srcFS.open(src);
464 IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
465 }
466 if (deleteSource) {
467 return srcFS.delete(src, true);
468 } else {
469 return true;
470 }
471 }
472
473 private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
474 boolean overwrite) throws IOException {
475 if (dstFS.exists(dst)) {
476 FileStatus sdst = dstFS.getFileStatus(dst);
477 if (sdst.isDirectory()) {
478 if (null == srcName) {
479 throw new IOException("Target " + dst + " is a directory");
480 }
481 return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
482 } else if (!overwrite) {
483 throw new IOException("Target " + dst + " already exists");
484 }
485 }
486 return dst;
487 }
488
489 /**
490 * Convert a os-native filename to a path that works for the shell.
491 * @param filename The filename to convert
492 * @return The unix pathname
493 * @throws IOException on windows, there can be problems with the subprocess
494 */
495 public static String makeShellPath(String filename) throws IOException {
496 return filename;
497 }
498
499 /**
500 * Convert a os-native filename to a path that works for the shell.
501 * @param file The filename to convert
502 * @return The unix pathname
503 * @throws IOException on windows, there can be problems with the subprocess
504 */
505 public static String makeShellPath(File file) throws IOException {
506 return makeShellPath(file, false);
507 }
508
509 /**
510 * Convert a os-native filename to a path that works for the shell.
511 * @param file The filename to convert
512 * @param makeCanonicalPath
513 * Whether to make canonical path for the file passed
514 * @return The unix pathname
515 * @throws IOException on windows, there can be problems with the subprocess
516 */
517 public static String makeShellPath(File file, boolean makeCanonicalPath)
518 throws IOException {
519 if (makeCanonicalPath) {
520 return makeShellPath(file.getCanonicalPath());
521 } else {
522 return makeShellPath(file.toString());
523 }
524 }
525
526 /**
527 * Takes an input dir and returns the du on that local directory. Very basic
528 * implementation.
529 *
530 * @param dir
531 * The input dir to get the disk space of this local dir
532 * @return The total disk space of the input local directory
533 */
534 public static long getDU(File dir) {
535 long size = 0;
536 if (!dir.exists())
537 return 0;
538 if (!dir.isDirectory()) {
539 return dir.length();
540 } else {
541 File[] allFiles = dir.listFiles();
542 if(allFiles != null) {
543 for (int i = 0; i < allFiles.length; i++) {
544 boolean isSymLink;
545 try {
546 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
547 } catch(IOException ioe) {
548 isSymLink = true;
549 }
550 if(!isSymLink) {
551 size += getDU(allFiles[i]);
552 }
553 }
554 }
555 return size;
556 }
557 }
558
559 /**
560 * Given a File input it will unzip the file in a the unzip directory
561 * passed as the second parameter
562 * @param inFile The zip file as input
563 * @param unzipDir The unzip directory where to unzip the zip file.
564 * @throws IOException
565 */
566 public static void unZip(File inFile, File unzipDir) throws IOException {
567 Enumeration<? extends ZipEntry> entries;
568 ZipFile zipFile = new ZipFile(inFile);
569
570 try {
571 entries = zipFile.entries();
572 while (entries.hasMoreElements()) {
573 ZipEntry entry = entries.nextElement();
574 if (!entry.isDirectory()) {
575 InputStream in = zipFile.getInputStream(entry);
576 try {
577 File file = new File(unzipDir, entry.getName());
578 if (!file.getParentFile().mkdirs()) {
579 if (!file.getParentFile().isDirectory()) {
580 throw new IOException("Mkdirs failed to create " +
581 file.getParentFile().toString());
582 }
583 }
584 OutputStream out = new FileOutputStream(file);
585 try {
586 byte[] buffer = new byte[8192];
587 int i;
588 while ((i = in.read(buffer)) != -1) {
589 out.write(buffer, 0, i);
590 }
591 } finally {
592 out.close();
593 }
594 } finally {
595 in.close();
596 }
597 }
598 }
599 } finally {
600 zipFile.close();
601 }
602 }
603
604 /**
605 * Given a Tar File as input it will untar the file in a the untar directory
606 * passed as the second parameter
607 *
608 * This utility will untar ".tar" files and ".tar.gz","tgz" files.
609 *
610 * @param inFile The tar file as input.
611 * @param untarDir The untar directory where to untar the tar file.
612 * @throws IOException
613 */
614 public static void unTar(File inFile, File untarDir) throws IOException {
615 if (!untarDir.mkdirs()) {
616 if (!untarDir.isDirectory()) {
617 throw new IOException("Mkdirs failed to create " + untarDir);
618 }
619 }
620
621 boolean gzipped = inFile.toString().endsWith("gz");
622 if(Shell.WINDOWS) {
623 // Tar is not native to Windows. Use simple Java based implementation for
624 // tests and simple tar archives
625 unTarUsingJava(inFile, untarDir, gzipped);
626 }
627 else {
628 // spawn tar utility to untar archive for full fledged unix behavior such
629 // as resolving symlinks in tar archives
630 unTarUsingTar(inFile, untarDir, gzipped);
631 }
632 }
633
634 private static void unTarUsingTar(File inFile, File untarDir,
635 boolean gzipped) throws IOException {
636 StringBuffer untarCommand = new StringBuffer();
637 if (gzipped) {
638 untarCommand.append(" gzip -dc '");
639 untarCommand.append(FileUtil.makeShellPath(inFile));
640 untarCommand.append("' | (");
641 }
642 untarCommand.append("cd '");
643 untarCommand.append(FileUtil.makeShellPath(untarDir));
644 untarCommand.append("' ; ");
645 untarCommand.append("tar -xf ");
646
647 if (gzipped) {
648 untarCommand.append(" -)");
649 } else {
650 untarCommand.append(FileUtil.makeShellPath(inFile));
651 }
652 String[] shellCmd = { "bash", "-c", untarCommand.toString() };
653 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
654 shexec.execute();
655 int exitcode = shexec.getExitCode();
656 if (exitcode != 0) {
657 throw new IOException("Error untarring file " + inFile +
658 ". Tar process exited with exit code " + exitcode);
659 }
660 }
661
662 private static void unTarUsingJava(File inFile, File untarDir,
663 boolean gzipped) throws IOException {
664 InputStream inputStream = null;
665 TarArchiveInputStream tis = null;
666 try {
667 if (gzipped) {
668 inputStream = new BufferedInputStream(new GZIPInputStream(
669 new FileInputStream(inFile)));
670 } else {
671 inputStream = new BufferedInputStream(new FileInputStream(inFile));
672 }
673
674 tis = new TarArchiveInputStream(inputStream);
675
676 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
677 unpackEntries(tis, entry, untarDir);
678 entry = tis.getNextTarEntry();
679 }
680 } finally {
681 IOUtils.cleanup(LOG, tis, inputStream);
682 }
683 }
684
685 private static void unpackEntries(TarArchiveInputStream tis,
686 TarArchiveEntry entry, File outputDir) throws IOException {
687 if (entry.isDirectory()) {
688 File subDir = new File(outputDir, entry.getName());
689 if (!subDir.mkdir() && !subDir.isDirectory()) {
690 throw new IOException("Mkdirs failed to create tar internal dir "
691 + outputDir);
692 }
693
694 for (TarArchiveEntry e : entry.getDirectoryEntries()) {
695 unpackEntries(tis, e, subDir);
696 }
697
698 return;
699 }
700
701 File outputFile = new File(outputDir, entry.getName());
702 if (!outputDir.exists()) {
703 if (!outputDir.mkdirs()) {
704 throw new IOException("Mkdirs failed to create tar internal dir "
705 + outputDir);
706 }
707 }
708
709 int count;
710 byte data[] = new byte[2048];
711 BufferedOutputStream outputStream = new BufferedOutputStream(
712 new FileOutputStream(outputFile));
713
714 while ((count = tis.read(data)) != -1) {
715 outputStream.write(data, 0, count);
716 }
717
718 outputStream.flush();
719 outputStream.close();
720 }
721
722 /**
723 * Class for creating hardlinks.
724 * Supports Unix, WindXP.
725 * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
726 */
727 @Deprecated
728 public static class HardLink extends org.apache.hadoop.fs.HardLink {
729 // This is a stub to assist with coordinated change between
730 // COMMON and HDFS projects. It will be removed after the
731 // corresponding change is committed to HDFS.
732 }
733
734 /**
735 * Create a soft link between a src and destination
736 * only on a local disk. HDFS does not support this.
737 * On Windows, when symlink creation fails due to security
738 * setting, we will log a warning. The return code in this
739 * case is 2.
740 * @param target the target for symlink
741 * @param linkname the symlink
742 * @return value returned by the command
743 */
744 public static int symLink(String target, String linkname) throws IOException{
745 // Run the input paths through Java's File so that they are converted to the
746 // native OS form
747 File targetFile = new File(target);
748 File linkFile = new File(linkname);
749
750 // If not on Java7+, copy a file instead of creating a symlink since
751 // Java6 has close to no support for symlinks on Windows. Specifically
752 // File#length and File#renameTo do not work as expected.
753 // (see HADOOP-9061 for additional details)
754 // We still create symlinks for directories, since the scenario in this
755 // case is different. The directory content could change in which
756 // case the symlink loses its purpose (for example task attempt log folder
757 // is symlinked under userlogs and userlogs are generated afterwards).
758 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
759 try {
760 LOG.info("FileUtil#symlink: On Java6, copying file instead "
761 + linkname + " -> " + target);
762 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
763 } catch (IOException ex) {
764 LOG.warn("FileUtil#symlink failed to copy the file with error: "
765 + ex.getMessage());
766 // Exit with non-zero exit code
767 return 1;
768 }
769 return 0;
770 }
771
772 String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
773 linkFile.getPath());
774 ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
775 try {
776 shExec.execute();
777 } catch (Shell.ExitCodeException ec) {
778 int returnVal = ec.getExitCode();
779 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
780 LOG.warn("Fail to create symbolic links on Windows. "
781 + "The default security settings in Windows disallow non-elevated "
782 + "administrators and all non-administrators from creating symbolic links. "
783 + "This behavior can be changed in the Local Security Policy management console");
784 } else if (returnVal != 0) {
785 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
786 + returnVal + " with: " + ec.getMessage());
787 }
788 return returnVal;
789 } catch (IOException e) {
790 if (LOG.isDebugEnabled()) {
791 LOG.debug("Error while create symlink " + linkname + " to " + target
792 + "." + " Exception: " + StringUtils.stringifyException(e));
793 }
794 throw e;
795 }
796 return shExec.getExitCode();
797 }
798
799 /**
800 * Change the permissions on a filename.
801 * @param filename the name of the file to change
802 * @param perm the permission string
803 * @return the exit code from the command
804 * @throws IOException
805 * @throws InterruptedException
806 */
807 public static int chmod(String filename, String perm
808 ) throws IOException, InterruptedException {
809 return chmod(filename, perm, false);
810 }
811
812 /**
813 * Change the permissions on a file / directory, recursively, if
814 * needed.
815 * @param filename name of the file whose permissions are to change
816 * @param perm permission string
817 * @param recursive true, if permissions should be changed recursively
818 * @return the exit code from the command.
819 * @throws IOException
820 */
821 public static int chmod(String filename, String perm, boolean recursive)
822 throws IOException {
823 String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
824 String[] args = new String[cmd.length + 1];
825 System.arraycopy(cmd, 0, args, 0, cmd.length);
826 args[cmd.length] = new File(filename).getPath();
827 ShellCommandExecutor shExec = new ShellCommandExecutor(args);
828 try {
829 shExec.execute();
830 }catch(IOException e) {
831 if(LOG.isDebugEnabled()) {
832 LOG.debug("Error while changing permission : " + filename
833 +" Exception: " + StringUtils.stringifyException(e));
834 }
835 }
836 return shExec.getExitCode();
837 }
838
839 /**
840 * Set the ownership on a file / directory. User name and group name
841 * cannot both be null.
842 * @param file the file to change
843 * @param username the new user owner name
844 * @param groupname the new group owner name
845 * @throws IOException
846 */
847 public static void setOwner(File file, String username,
848 String groupname) throws IOException {
849 if (username == null && groupname == null) {
850 throw new IOException("username == null && groupname == null");
851 }
852 String arg = (username == null ? "" : username)
853 + (groupname == null ? "" : ":" + groupname);
854 String [] cmd = Shell.getSetOwnerCommand(arg);
855 execCommand(file, cmd);
856 }
857
858 /**
859 * Platform independent implementation for {@link File#setReadable(boolean)}
860 * File#setReadable does not work as expected on Windows.
861 * @param f input file
862 * @param readable
863 * @return true on success, false otherwise
864 */
865 public static boolean setReadable(File f, boolean readable) {
866 if (Shell.WINDOWS) {
867 try {
868 String permission = readable ? "u+r" : "u-r";
869 FileUtil.chmod(f.getCanonicalPath(), permission, false);
870 return true;
871 } catch (IOException ex) {
872 return false;
873 }
874 } else {
875 return f.setReadable(readable);
876 }
877 }
878
879 /**
880 * Platform independent implementation for {@link File#setWritable(boolean)}
881 * File#setWritable does not work as expected on Windows.
882 * @param f input file
883 * @param writable
884 * @return true on success, false otherwise
885 */
886 public static boolean setWritable(File f, boolean writable) {
887 if (Shell.WINDOWS) {
888 try {
889 String permission = writable ? "u+w" : "u-w";
890 FileUtil.chmod(f.getCanonicalPath(), permission, false);
891 return true;
892 } catch (IOException ex) {
893 return false;
894 }
895 } else {
896 return f.setWritable(writable);
897 }
898 }
899
900 /**
901 * Platform independent implementation for {@link File#setExecutable(boolean)}
902 * File#setExecutable does not work as expected on Windows.
903 * Note: revoking execute permission on folders does not have the same
904 * behavior on Windows as on Unix platforms. Creating, deleting or renaming
905 * a file within that folder will still succeed on Windows.
906 * @param f input file
907 * @param executable
908 * @return true on success, false otherwise
909 */
910 public static boolean setExecutable(File f, boolean executable) {
911 if (Shell.WINDOWS) {
912 try {
913 String permission = executable ? "u+x" : "u-x";
914 FileUtil.chmod(f.getCanonicalPath(), permission, false);
915 return true;
916 } catch (IOException ex) {
917 return false;
918 }
919 } else {
920 return f.setExecutable(executable);
921 }
922 }
923
924 /**
925 * Platform independent implementation for {@link File#canRead()}
926 * @param f input file
927 * @return On Unix, same as {@link File#canRead()}
928 * On Windows, true if process has read access on the path
929 */
930 public static boolean canRead(File f) {
931 if (Shell.WINDOWS) {
932 try {
933 return NativeIO.Windows.access(f.getCanonicalPath(),
934 NativeIO.Windows.AccessRight.ACCESS_READ);
935 } catch (IOException e) {
936 return false;
937 }
938 } else {
939 return f.canRead();
940 }
941 }
942
943 /**
944 * Platform independent implementation for {@link File#canWrite()}
945 * @param f input file
946 * @return On Unix, same as {@link File#canWrite()}
947 * On Windows, true if process has write access on the path
948 */
949 public static boolean canWrite(File f) {
950 if (Shell.WINDOWS) {
951 try {
952 return NativeIO.Windows.access(f.getCanonicalPath(),
953 NativeIO.Windows.AccessRight.ACCESS_WRITE);
954 } catch (IOException e) {
955 return false;
956 }
957 } else {
958 return f.canWrite();
959 }
960 }
961
962 /**
963 * Platform independent implementation for {@link File#canExecute()}
964 * @param f input file
965 * @return On Unix, same as {@link File#canExecute()}
966 * On Windows, true if process has execute access on the path
967 */
968 public static boolean canExecute(File f) {
969 if (Shell.WINDOWS) {
970 try {
971 return NativeIO.Windows.access(f.getCanonicalPath(),
972 NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
973 } catch (IOException e) {
974 return false;
975 }
976 } else {
977 return f.canExecute();
978 }
979 }
980
981 /**
982 * Set permissions to the required value. Uses the java primitives instead
983 * of forking if group == other.
984 * @param f the file to change
985 * @param permission the new permissions
986 * @throws IOException
987 */
988 public static void setPermission(File f, FsPermission permission
989 ) throws IOException {
990 FsAction user = permission.getUserAction();
991 FsAction group = permission.getGroupAction();
992 FsAction other = permission.getOtherAction();
993
994 // use the native/fork if the group/other permissions are different
995 // or if the native is available or on Windows
996 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
997 execSetPermission(f, permission);
998 return;
999 }
1000
1001 boolean rv = true;
1002
1003 // read perms
1004 rv = f.setReadable(group.implies(FsAction.READ), false);
1005 checkReturnValue(rv, f, permission);
1006 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1007 rv = f.setReadable(user.implies(FsAction.READ), true);
1008 checkReturnValue(rv, f, permission);
1009 }
1010
1011 // write perms
1012 rv = f.setWritable(group.implies(FsAction.WRITE), false);
1013 checkReturnValue(rv, f, permission);
1014 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1015 rv = f.setWritable(user.implies(FsAction.WRITE), true);
1016 checkReturnValue(rv, f, permission);
1017 }
1018
1019 // exec perms
1020 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1021 checkReturnValue(rv, f, permission);
1022 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1023 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1024 checkReturnValue(rv, f, permission);
1025 }
1026 }
1027
1028 private static void checkReturnValue(boolean rv, File p,
1029 FsPermission permission
1030 ) throws IOException {
1031 if (!rv) {
1032 throw new IOException("Failed to set permissions of path: " + p +
1033 " to " +
1034 String.format("%04o", permission.toShort()));
1035 }
1036 }
1037
1038 private static void execSetPermission(File f,
1039 FsPermission permission
1040 ) throws IOException {
1041 if (NativeIO.isAvailable()) {
1042 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1043 } else {
1044 execCommand(f, Shell.getSetPermissionCommand(
1045 String.format("%04o", permission.toShort()), false));
1046 }
1047 }
1048
1049 static String execCommand(File f, String... cmd) throws IOException {
1050 String[] args = new String[cmd.length + 1];
1051 System.arraycopy(cmd, 0, args, 0, cmd.length);
1052 args[cmd.length] = f.getCanonicalPath();
1053 String output = Shell.execCommand(args);
1054 return output;
1055 }
1056
1057 /**
1058 * Create a tmp file for a base file.
1059 * @param basefile the base file of the tmp
1060 * @param prefix file name prefix of tmp
1061 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1062 * @return a newly created tmp file
1063 * @exception IOException If a tmp file cannot created
1064 * @see java.io.File#createTempFile(String, String, File)
1065 * @see java.io.File#deleteOnExit()
1066 */
1067 public static final File createLocalTempFile(final File basefile,
1068 final String prefix,
1069 final boolean isDeleteOnExit)
1070 throws IOException {
1071 File tmp = File.createTempFile(prefix + basefile.getName(),
1072 "", basefile.getParentFile());
1073 if (isDeleteOnExit) {
1074 tmp.deleteOnExit();
1075 }
1076 return tmp;
1077 }
1078
1079 /**
1080 * Move the src file to the name specified by target.
1081 * @param src the source file
1082 * @param target the target file
1083 * @exception IOException If this operation fails
1084 */
1085 public static void replaceFile(File src, File target) throws IOException {
1086 /* renameTo() has two limitations on Windows platform.
1087 * src.renameTo(target) fails if
1088 * 1) If target already exists OR
1089 * 2) If target is already open for reading/writing.
1090 */
1091 if (!src.renameTo(target)) {
1092 int retries = 5;
1093 while (target.exists() && !target.delete() && retries-- >= 0) {
1094 try {
1095 Thread.sleep(1000);
1096 } catch (InterruptedException e) {
1097 throw new IOException("replaceFile interrupted.");
1098 }
1099 }
1100 if (!src.renameTo(target)) {
1101 throw new IOException("Unable to rename " + src +
1102 " to " + target);
1103 }
1104 }
1105 }
1106
1107 /**
1108 * A wrapper for {@link File#listFiles()}. This java.io API returns null
1109 * when a dir is not a directory or for any I/O error. Instead of having
1110 * null check everywhere File#listFiles() is used, we will add utility API
1111 * to get around this problem. For the majority of cases where we prefer
1112 * an IOException to be thrown.
1113 * @param dir directory for which listing should be performed
1114 * @return list of files or empty list
1115 * @exception IOException for invalid directory or for a bad disk.
1116 */
1117 public static File[] listFiles(File dir) throws IOException {
1118 File[] files = dir.listFiles();
1119 if(files == null) {
1120 throw new IOException("Invalid directory or I/O error occurred for dir: "
1121 + dir.toString());
1122 }
1123 return files;
1124 }
1125
1126 /**
1127 * A wrapper for {@link File#list()}. This java.io API returns null
1128 * when a dir is not a directory or for any I/O error. Instead of having
1129 * null check everywhere File#list() is used, we will add utility API
1130 * to get around this problem. For the majority of cases where we prefer
1131 * an IOException to be thrown.
1132 * @param dir directory for which listing should be performed
1133 * @return list of file names or empty string list
1134 * @exception IOException for invalid directory or for a bad disk.
1135 */
1136 public static String[] list(File dir) throws IOException {
1137 String[] fileNames = dir.list();
1138 if(fileNames == null) {
1139 throw new IOException("Invalid directory or I/O error occurred for dir: "
1140 + dir.toString());
1141 }
1142 return fileNames;
1143 }
1144
1145 /**
1146 * Create a jar file at the given path, containing a manifest with a classpath
1147 * that references all specified entries.
1148 *
1149 * Some platforms may have an upper limit on command line length. For example,
1150 * the maximum command line length on Windows is 8191 characters, but the
1151 * length of the classpath may exceed this. To work around this limitation,
1152 * use this method to create a small intermediate jar with a manifest that
1153 * contains the full classpath. It returns the absolute path to the new jar,
1154 * which the caller may set as the classpath for a new process.
1155 *
1156 * Environment variable evaluation is not supported within a jar manifest, so
1157 * this method expands environment variables before inserting classpath entries
1158 * to the manifest. The method parses environment variables according to
1159 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
1160 * environment variables are case-insensitive. For example, %VAR% and %var%
1161 * evaluate to the same value.
1162 *
1163 * Specifying the classpath in a jar manifest does not support wildcards, so
1164 * this method expands wildcards internally. Any classpath entry that ends
1165 * with * is translated to all files at that path with extension .jar or .JAR.
1166 *
1167 * @param inputClassPath String input classpath to bundle into the jar manifest
1168 * @param pwd Path to working directory to save jar
1169 * @param callerEnv Map<String, String> caller's environment variables to use
1170 * for expansion
1171 * @return String absolute path to new jar
1172 * @throws IOException if there is an I/O error while writing the jar file
1173 */
1174 public static String createJarWithClassPath(String inputClassPath, Path pwd,
1175 Map<String, String> callerEnv) throws IOException {
1176 // Replace environment variables, case-insensitive on Windows
1177 @SuppressWarnings("unchecked")
1178 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1179 callerEnv;
1180 String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1181 for (int i = 0; i < classPathEntries.length; ++i) {
1182 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1183 StringUtils.ENV_VAR_PATTERN, env);
1184 }
1185 File workingDir = new File(pwd.toString());
1186 if (!workingDir.mkdirs()) {
1187 // If mkdirs returns false because the working directory already exists,
1188 // then this is acceptable. If it returns false due to some other I/O
1189 // error, then this method will fail later with an IOException while saving
1190 // the jar.
1191 LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1192 }
1193
1194 // Append all entries
1195 List<String> classPathEntryList = new ArrayList<String>(
1196 classPathEntries.length);
1197 for (String classPathEntry: classPathEntries) {
1198 if (classPathEntry.endsWith("*")) {
1199 // Append all jars that match the wildcard
1200 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1201 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1202 .globStatus(globPath);
1203 if (wildcardJars != null) {
1204 for (FileStatus wildcardJar: wildcardJars) {
1205 classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1206 .toExternalForm());
1207 }
1208 }
1209 } else {
1210 // Append just this entry
1211 String classPathEntryUrl = new File(classPathEntry).toURI().toURL()
1212 .toExternalForm();
1213
1214 // File.toURI only appends trailing '/' if it can determine that it is a
1215 // directory that already exists. (See JavaDocs.) If this entry had a
1216 // trailing '/' specified by the caller, then guarantee that the
1217 // classpath entry in the manifest has a trailing '/', and thus refers to
1218 // a directory instead of a file. This can happen if the caller is
1219 // creating a classpath jar referencing a directory that hasn't been
1220 // created yet, but will definitely be created before running.
1221 if (classPathEntry.endsWith(Path.SEPARATOR) &&
1222 !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1223 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1224 }
1225 classPathEntryList.add(classPathEntryUrl);
1226 }
1227 }
1228 String jarClassPath = StringUtils.join(" ", classPathEntryList);
1229
1230 // Create the manifest
1231 Manifest jarManifest = new Manifest();
1232 jarManifest.getMainAttributes().putValue(
1233 Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1234 jarManifest.getMainAttributes().putValue(
1235 Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1236
1237 // Write the manifest to output JAR file
1238 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1239 FileOutputStream fos = null;
1240 BufferedOutputStream bos = null;
1241 JarOutputStream jos = null;
1242 try {
1243 fos = new FileOutputStream(classPathJar);
1244 bos = new BufferedOutputStream(fos);
1245 jos = new JarOutputStream(bos, jarManifest);
1246 } finally {
1247 IOUtils.cleanup(LOG, jos, bos, fos);
1248 }
1249
1250 return classPathJar.getCanonicalPath();
1251 }
1252 }