001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.fs;
019
020 import java.io.FileNotFoundException;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024 import java.net.URI;
025 import java.security.PrivilegedExceptionAction;
026 import java.util.ArrayList;
027 import java.util.Arrays;
028 import java.util.EnumSet;
029 import java.util.HashSet;
030 import java.util.IdentityHashMap;
031 import java.util.List;
032 import java.util.Map;
033 import java.util.Set;
034 import java.util.Stack;
035 import java.util.TreeSet;
036 import java.util.Map.Entry;
037
038 import org.apache.commons.logging.Log;
039 import org.apache.commons.logging.LogFactory;
040 import org.apache.hadoop.HadoopIllegalArgumentException;
041 import org.apache.hadoop.classification.InterfaceAudience;
042 import org.apache.hadoop.classification.InterfaceStability;
043 import org.apache.hadoop.conf.Configuration;
044 import org.apache.hadoop.fs.FileSystem.Statistics;
045 import org.apache.hadoop.fs.Options.CreateOpts;
046 import org.apache.hadoop.fs.permission.FsPermission;
047 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
048 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
049 import org.apache.hadoop.io.IOUtils;
050 import org.apache.hadoop.ipc.RpcClientException;
051 import org.apache.hadoop.ipc.RpcServerException;
052 import org.apache.hadoop.ipc.UnexpectedServerException;
053 import org.apache.hadoop.fs.InvalidPathException;
054 import org.apache.hadoop.security.AccessControlException;
055 import org.apache.hadoop.security.UserGroupInformation;
056 import org.apache.hadoop.security.token.Token;
057 import org.apache.hadoop.util.ShutdownHookManager;
058
059 /**
060 * The FileContext class provides an interface to the application writer for
061 * using the Hadoop file system.
062 * It provides a set of methods for the usual operation: create, open,
063 * list, etc
064 *
065 * <p>
066 * <b> *** Path Names *** </b>
067 * <p>
068 *
069 * The Hadoop file system supports a URI name space and URI names.
070 * It offers a forest of file systems that can be referenced using fully
071 * qualified URIs.
072 * Two common Hadoop file systems implementations are
073 * <ul>
074 * <li> the local file system: file:///path
075 * <li> the hdfs file system hdfs://nnAddress:nnPort/path
076 * </ul>
077 *
078 * While URI names are very flexible, it requires knowing the name or address
079 * of the server. For convenience one often wants to access the default system
080 * in one's environment without knowing its name/address. This has an
081 * additional benefit that it allows one to change one's default fs
082 * (e.g. admin moves application from cluster1 to cluster2).
083 * <p>
084 *
085 * To facilitate this, Hadoop supports a notion of a default file system.
086 * The user can set his default file system, although this is
087 * typically set up for you in your environment via your default config.
088 * A default file system implies a default scheme and authority; slash-relative
089 * names (such as /for/bar) are resolved relative to that default FS.
090 * Similarly a user can also have working-directory-relative names (i.e. names
091 * not starting with a slash). While the working directory is generally in the
092 * same default FS, the wd can be in a different FS.
093 * <p>
094 * Hence Hadoop path names can be one of:
095 * <ul>
096 * <li> fully qualified URI: scheme://authority/path
097 * <li> slash relative names: /path relative to the default file system
098 * <li> wd-relative names: path relative to the working dir
099 * </ul>
100 * Relative paths with scheme (scheme:foo/bar) are illegal.
101 *
102 * <p>
103 * <b>****The Role of the FileContext and configuration defaults****</b>
104 * <p>
105 * The FileContext provides file namespace context for resolving file names;
106 * it also contains the umask for permissions, In that sense it is like the
107 * per-process file-related state in Unix system.
108 * These two properties
109 * <ul>
110 * <li> default file system i.e your slash)
111 * <li> umask
112 * </ul>
113 * in general, are obtained from the default configuration file
114 * in your environment, (@see {@link Configuration}).
115 *
116 * No other configuration parameters are obtained from the default config as
117 * far as the file context layer is concerned. All file system instances
118 * (i.e. deployments of file systems) have default properties; we call these
119 * server side (SS) defaults. Operation like create allow one to select many
120 * properties: either pass them in as explicit parameters or use
121 * the SS properties.
122 * <p>
123 * The file system related SS defaults are
124 * <ul>
125 * <li> the home directory (default is "/user/userName")
126 * <li> the initial wd (only for local fs)
127 * <li> replication factor
128 * <li> block size
129 * <li> buffer size
130 * <li> encryptDataTransfer
131 * <li> checksum option. (checksumType and bytesPerChecksum)
132 * </ul>
133 *
134 * <p>
135 * <b> *** Usage Model for the FileContext class *** </b>
136 * <p>
137 * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
138 * Unspecified values come from core-defaults.xml in the release jar.
139 * <ul>
140 * <li> myFContext = FileContext.getFileContext(); // uses the default config
141 * // which has your default FS
142 * <li> myFContext.create(path, ...);
143 * <li> myFContext.setWorkingDir(path)
144 * <li> myFContext.open (path, ...);
145 * </ul>
146 * Example 2: Get a FileContext with a specific URI as the default FS
147 * <ul>
148 * <li> myFContext = FileContext.getFileContext(URI)
149 * <li> myFContext.create(path, ...);
150 * ...
151 * </ul>
152 * Example 3: FileContext with local file system as the default
153 * <ul>
154 * <li> myFContext = FileContext.getLocalFSFileContext()
155 * <li> myFContext.create(path, ...);
156 * <li> ...
157 * </ul>
158 * Example 4: Use a specific config, ignoring $HADOOP_CONFIG
159 * Generally you should not need use a config unless you are doing
160 * <ul>
161 * <li> configX = someConfigSomeOnePassedToYou.
162 * <li> myFContext = getFileContext(configX); // configX is not changed,
163 * // is passed down
164 * <li> myFContext.create(path, ...);
165 * <li>...
166 * </ul>
167 *
168 */
169
170 @InterfaceAudience.Public
171 @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
172 public final class FileContext {
173
174 public static final Log LOG = LogFactory.getLog(FileContext.class);
175 /**
176 * Default permission for directory and symlink
177 * In previous versions, this default permission was also used to
178 * create files, so files created end up with ugo+x permission.
179 * See HADOOP-9155 for detail.
180 * Two new constants are added to solve this, please use
181 * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
182 * {@link FileContext#FILE_DEFAULT_PERM} for file.
183 * This constant is kept for compatibility.
184 */
185 public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
186 /**
187 * Default permission for directory
188 */
189 public static final FsPermission DIR_DEFAULT_PERM = FsPermission.getDirDefault();
190 /**
191 * Default permission for file
192 */
193 public static final FsPermission FILE_DEFAULT_PERM = FsPermission.getFileDefault();
194
195 /**
196 * Priority of the FileContext shutdown hook.
197 */
198 public static final int SHUTDOWN_HOOK_PRIORITY = 20;
199
200 /**
201 * List of files that should be deleted on JVM shutdown.
202 */
203 static final Map<FileContext, Set<Path>> DELETE_ON_EXIT =
204 new IdentityHashMap<FileContext, Set<Path>>();
205
206 /** JVM shutdown hook thread. */
207 static final FileContextFinalizer FINALIZER =
208 new FileContextFinalizer();
209
210 private static final PathFilter DEFAULT_FILTER = new PathFilter() {
211 @Override
212 public boolean accept(final Path file) {
213 return true;
214 }
215 };
216
217 /**
218 * The FileContext is defined by.
219 * 1) defaultFS (slash)
220 * 2) wd
221 * 3) umask
222 */
223 private final AbstractFileSystem defaultFS; //default FS for this FileContext.
224 private Path workingDir; // Fully qualified
225 private FsPermission umask;
226 private final Configuration conf;
227 private final UserGroupInformation ugi;
228 final boolean resolveSymlinks;
229
230 private FileContext(final AbstractFileSystem defFs,
231 final FsPermission theUmask, final Configuration aConf) {
232 defaultFS = defFs;
233 umask = FsPermission.getUMask(aConf);
234 conf = aConf;
235 try {
236 ugi = UserGroupInformation.getCurrentUser();
237 } catch (IOException e) {
238 LOG.error("Exception in getCurrentUser: ",e);
239 throw new RuntimeException("Failed to get the current user " +
240 "while creating a FileContext", e);
241 }
242 /*
243 * Init the wd.
244 * WorkingDir is implemented at the FileContext layer
245 * NOT at the AbstractFileSystem layer.
246 * If the DefaultFS, such as localFilesystem has a notion of
247 * builtin WD, we use that as the initial WD.
248 * Otherwise the WD is initialized to the home directory.
249 */
250 workingDir = defaultFS.getInitialWorkingDirectory();
251 if (workingDir == null) {
252 workingDir = defaultFS.getHomeDirectory();
253 }
254 resolveSymlinks = conf.getBoolean(
255 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
256 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
257 util = new Util(); // for the inner class
258 }
259
260 /*
261 * Remove relative part - return "absolute":
262 * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
263 * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
264 * ("/foo/bar") are returned unchanged.
265 *
266 * Applications that use FileContext should use #makeQualified() since
267 * they really want a fully qualified URI.
268 * Hence this method is not called makeAbsolute() and
269 * has been deliberately declared private.
270 */
271 private Path fixRelativePart(Path p) {
272 if (p.isUriPathAbsolute()) {
273 return p;
274 } else {
275 return new Path(workingDir, p);
276 }
277 }
278
279 /**
280 * Delete all the paths that were marked as delete-on-exit.
281 */
282 static void processDeleteOnExit() {
283 synchronized (DELETE_ON_EXIT) {
284 Set<Entry<FileContext, Set<Path>>> set = DELETE_ON_EXIT.entrySet();
285 for (Entry<FileContext, Set<Path>> entry : set) {
286 FileContext fc = entry.getKey();
287 Set<Path> paths = entry.getValue();
288 for (Path path : paths) {
289 try {
290 fc.delete(path, true);
291 } catch (IOException e) {
292 LOG.warn("Ignoring failure to deleteOnExit for path " + path);
293 }
294 }
295 }
296 DELETE_ON_EXIT.clear();
297 }
298 }
299
300 /**
301 * Get the file system of supplied path.
302 *
303 * @param absOrFqPath - absolute or fully qualified path
304 * @return the file system of the path
305 *
306 * @throws UnsupportedFileSystemException If the file system for
307 * <code>absOrFqPath</code> is not supported.
308 * @throws IOExcepton If the file system for <code>absOrFqPath</code> could
309 * not be instantiated.
310 */
311 protected AbstractFileSystem getFSofPath(final Path absOrFqPath)
312 throws UnsupportedFileSystemException, IOException {
313 absOrFqPath.checkNotSchemeWithRelative();
314 absOrFqPath.checkNotRelative();
315
316 try {
317 // Is it the default FS for this FileContext?
318 defaultFS.checkPath(absOrFqPath);
319 return defaultFS;
320 } catch (Exception e) { // it is different FileSystem
321 return getAbstractFileSystem(ugi, absOrFqPath.toUri(), conf);
322 }
323 }
324
325 private static AbstractFileSystem getAbstractFileSystem(
326 UserGroupInformation user, final URI uri, final Configuration conf)
327 throws UnsupportedFileSystemException, IOException {
328 try {
329 return user.doAs(new PrivilegedExceptionAction<AbstractFileSystem>() {
330 @Override
331 public AbstractFileSystem run() throws UnsupportedFileSystemException {
332 return AbstractFileSystem.get(uri, conf);
333 }
334 });
335 } catch (InterruptedException ex) {
336 LOG.error(ex);
337 throw new IOException("Failed to get the AbstractFileSystem for path: "
338 + uri, ex);
339 }
340 }
341
342 /**
343 * Protected Static Factory methods for getting a FileContexts
344 * that take a AbstractFileSystem as input. To be used for testing.
345 */
346
347 /**
348 * Create a FileContext with specified FS as default using the specified
349 * config.
350 *
351 * @param defFS
352 * @param aConf
353 * @return new FileContext with specifed FS as default.
354 */
355 public static FileContext getFileContext(final AbstractFileSystem defFS,
356 final Configuration aConf) {
357 return new FileContext(defFS, FsPermission.getUMask(aConf), aConf);
358 }
359
360 /**
361 * Create a FileContext for specified file system using the default config.
362 *
363 * @param defaultFS
364 * @return a FileContext with the specified AbstractFileSystem
365 * as the default FS.
366 */
367 protected static FileContext getFileContext(
368 final AbstractFileSystem defaultFS) {
369 return getFileContext(defaultFS, new Configuration());
370 }
371
372 /**
373 * Static Factory methods for getting a FileContext.
374 * Note new file contexts are created for each call.
375 * The only singleton is the local FS context using the default config.
376 *
377 * Methods that use the default config: the default config read from the
378 * $HADOOP_CONFIG/core.xml,
379 * Unspecified key-values for config are defaulted from core-defaults.xml
380 * in the release jar.
381 *
382 * The keys relevant to the FileContext layer are extracted at time of
383 * construction. Changes to the config after the call are ignore
384 * by the FileContext layer.
385 * The conf is passed to lower layers like AbstractFileSystem and HDFS which
386 * pick up their own config variables.
387 */
388
389 /**
390 * Create a FileContext using the default config read from the
391 * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
392 * from core-defaults.xml in the release jar.
393 *
394 * @throws UnsupportedFileSystemException If the file system from the default
395 * configuration is not supported
396 */
397 public static FileContext getFileContext()
398 throws UnsupportedFileSystemException {
399 return getFileContext(new Configuration());
400 }
401
402 /**
403 * @return a FileContext for the local file system using the default config.
404 * @throws UnsupportedFileSystemException If the file system for
405 * {@link FsConstants#LOCAL_FS_URI} is not supported.
406 */
407 public static FileContext getLocalFSFileContext()
408 throws UnsupportedFileSystemException {
409 return getFileContext(FsConstants.LOCAL_FS_URI);
410 }
411
412 /**
413 * Create a FileContext for specified URI using the default config.
414 *
415 * @param defaultFsUri
416 * @return a FileContext with the specified URI as the default FS.
417 *
418 * @throws UnsupportedFileSystemException If the file system for
419 * <code>defaultFsUri</code> is not supported
420 */
421 public static FileContext getFileContext(final URI defaultFsUri)
422 throws UnsupportedFileSystemException {
423 return getFileContext(defaultFsUri, new Configuration());
424 }
425
426 /**
427 * Create a FileContext for specified default URI using the specified config.
428 *
429 * @param defaultFsUri
430 * @param aConf
431 * @return new FileContext for specified uri
432 * @throws UnsupportedFileSystemException If the file system with specified is
433 * not supported
434 * @throws RuntimeException If the file system specified is supported but
435 * could not be instantiated, or if login fails.
436 */
437 public static FileContext getFileContext(final URI defaultFsUri,
438 final Configuration aConf) throws UnsupportedFileSystemException {
439 UserGroupInformation currentUser = null;
440 AbstractFileSystem defaultAfs = null;
441 try {
442 currentUser = UserGroupInformation.getCurrentUser();
443 defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
444 } catch (UnsupportedFileSystemException ex) {
445 throw ex;
446 } catch (IOException ex) {
447 LOG.error(ex);
448 throw new RuntimeException(ex);
449 }
450 return getFileContext(defaultAfs, aConf);
451 }
452
453 /**
454 * Create a FileContext using the passed config. Generally it is better to use
455 * {@link #getFileContext(URI, Configuration)} instead of this one.
456 *
457 *
458 * @param aConf
459 * @return new FileContext
460 * @throws UnsupportedFileSystemException If file system in the config
461 * is not supported
462 */
463 public static FileContext getFileContext(final Configuration aConf)
464 throws UnsupportedFileSystemException {
465 return getFileContext(
466 URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)),
467 aConf);
468 }
469
470 /**
471 * @param aConf - from which the FileContext is configured
472 * @return a FileContext for the local file system using the specified config.
473 *
474 * @throws UnsupportedFileSystemException If default file system in the config
475 * is not supported
476 *
477 */
478 public static FileContext getLocalFSFileContext(final Configuration aConf)
479 throws UnsupportedFileSystemException {
480 return getFileContext(FsConstants.LOCAL_FS_URI, aConf);
481 }
482
483 /* This method is needed for tests. */
484 @InterfaceAudience.Private
485 @InterfaceStability.Unstable /* return type will change to AFS once
486 HADOOP-6223 is completed */
487 public AbstractFileSystem getDefaultFileSystem() {
488 return defaultFS;
489 }
490
491 /**
492 * Set the working directory for wd-relative names (such a "foo/bar"). Working
493 * directory feature is provided by simply prefixing relative names with the
494 * working dir. Note this is different from Unix where the wd is actually set
495 * to the inode. Hence setWorkingDir does not follow symlinks etc. This works
496 * better in a distributed environment that has multiple independent roots.
497 * {@link #getWorkingDirectory()} should return what setWorkingDir() set.
498 *
499 * @param newWDir new working directory
500 * @throws IOException
501 * <br>
502 * NewWdir can be one of:
503 * <ul>
504 * <li>relative path: "foo/bar";</li>
505 * <li>absolute without scheme: "/foo/bar"</li>
506 * <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
507 * </ul>
508 * <br>
509 * Illegal WDs:
510 * <ul>
511 * <li>relative with scheme: "xx:foo/bar"</li>
512 * <li>non existent directory</li>
513 * </ul>
514 */
515 public void setWorkingDirectory(final Path newWDir) throws IOException {
516 newWDir.checkNotSchemeWithRelative();
517 /* wd is stored as a fully qualified path. We check if the given
518 * path is not relative first since resolve requires and returns
519 * an absolute path.
520 */
521 final Path newWorkingDir = new Path(workingDir, newWDir);
522 FileStatus status = getFileStatus(newWorkingDir);
523 if (status.isFile()) {
524 throw new FileNotFoundException("Cannot setWD to a file");
525 }
526 workingDir = newWorkingDir;
527 }
528
529 /**
530 * Gets the working directory for wd-relative names (such a "foo/bar").
531 */
532 public Path getWorkingDirectory() {
533 return workingDir;
534 }
535
536 /**
537 * Gets the ugi in the file-context
538 * @return UserGroupInformation
539 */
540 public UserGroupInformation getUgi() {
541 return ugi;
542 }
543
544 /**
545 * Return the current user's home directory in this file system.
546 * The default implementation returns "/user/$USER/".
547 * @return the home directory
548 */
549 public Path getHomeDirectory() {
550 return defaultFS.getHomeDirectory();
551 }
552
553 /**
554 *
555 * @return the umask of this FileContext
556 */
557 public FsPermission getUMask() {
558 return umask;
559 }
560
561 /**
562 * Set umask to the supplied parameter.
563 * @param newUmask the new umask
564 */
565 public void setUMask(final FsPermission newUmask) {
566 umask = newUmask;
567 }
568
569
570 /**
571 * Resolve the path following any symlinks or mount points
572 * @param f to be resolved
573 * @return fully qualified resolved path
574 *
575 * @throws FileNotFoundException If <code>f</code> does not exist
576 * @throws AccessControlException if access denied
577 * @throws IOException If an IO Error occurred
578 *
579 * Exceptions applicable to file systems accessed over RPC:
580 * @throws RpcClientException If an exception occurred in the RPC client
581 * @throws RpcServerException If an exception occurred in the RPC server
582 * @throws UnexpectedServerException If server implementation throws
583 * undeclared exception to RPC server
584 *
585 * RuntimeExceptions:
586 * @throws InvalidPathException If path <code>f</code> is not valid
587 */
588 public Path resolvePath(final Path f) throws FileNotFoundException,
589 UnresolvedLinkException, AccessControlException, IOException {
590 return resolve(f);
591 }
592
593 /**
594 * Make the path fully qualified if it is isn't.
595 * A Fully-qualified path has scheme and authority specified and an absolute
596 * path.
597 * Use the default file system and working dir in this FileContext to qualify.
598 * @param path
599 * @return qualified path
600 */
601 public Path makeQualified(final Path path) {
602 return path.makeQualified(defaultFS.getUri(), getWorkingDirectory());
603 }
604
605 /**
606 * Create or overwrite file on indicated path and returns an output stream for
607 * writing into the file.
608 *
609 * @param f the file name to open
610 * @param createFlag gives the semantics of create; see {@link CreateFlag}
611 * @param opts file creation options; see {@link Options.CreateOpts}.
612 * <ul>
613 * <li>Progress - to report progress on the operation - default null
614 * <li>Permission - umask is applied against permisssion: default is
615 * FsPermissions:getDefault()
616 *
617 * <li>CreateParent - create missing parent path; default is to not
618 * to create parents
619 * <li>The defaults for the following are SS defaults of the file
620 * server implementing the target path. Not all parameters make sense
621 * for all kinds of file system - eg. localFS ignores Blocksize,
622 * replication, checksum
623 * <ul>
624 * <li>BufferSize - buffersize used in FSDataOutputStream
625 * <li>Blocksize - block size for file blocks
626 * <li>ReplicationFactor - replication for blocks
627 * <li>ChecksumParam - Checksum parameters. server default is used
628 * if not specified.
629 * </ul>
630 * </ul>
631 *
632 * @return {@link FSDataOutputStream} for created file
633 *
634 * @throws AccessControlException If access is denied
635 * @throws FileAlreadyExistsException If file <code>f</code> already exists
636 * @throws FileNotFoundException If parent of <code>f</code> does not exist
637 * and <code>createParent</code> is false
638 * @throws ParentNotDirectoryException If parent of <code>f</code> is not a
639 * directory.
640 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
641 * not supported
642 * @throws IOException If an I/O error occurred
643 *
644 * Exceptions applicable to file systems accessed over RPC:
645 * @throws RpcClientException If an exception occurred in the RPC client
646 * @throws RpcServerException If an exception occurred in the RPC server
647 * @throws UnexpectedServerException If server implementation throws
648 * undeclared exception to RPC server
649 *
650 * RuntimeExceptions:
651 * @throws InvalidPathException If path <code>f</code> is not valid
652 */
653 public FSDataOutputStream create(final Path f,
654 final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
655 throws AccessControlException, FileAlreadyExistsException,
656 FileNotFoundException, ParentNotDirectoryException,
657 UnsupportedFileSystemException, IOException {
658 Path absF = fixRelativePart(f);
659
660 // If one of the options is a permission, extract it & apply umask
661 // If not, add a default Perms and apply umask;
662 // AbstractFileSystem#create
663
664 CreateOpts.Perms permOpt =
665 (CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts);
666 FsPermission permission = (permOpt != null) ? permOpt.getValue() :
667 FILE_DEFAULT_PERM;
668 permission = permission.applyUMask(umask);
669
670 final CreateOpts[] updatedOpts =
671 CreateOpts.setOpt(CreateOpts.perms(permission), opts);
672 return new FSLinkResolver<FSDataOutputStream>() {
673 @Override
674 public FSDataOutputStream next(final AbstractFileSystem fs, final Path p)
675 throws IOException {
676 return fs.create(p, createFlag, updatedOpts);
677 }
678 }.resolve(this, absF);
679 }
680
681 /**
682 * Make(create) a directory and all the non-existent parents.
683 *
684 * @param dir - the dir to make
685 * @param permission - permissions is set permission&~umask
686 * @param createParent - if true then missing parent dirs are created if false
687 * then parent must exist
688 *
689 * @throws AccessControlException If access is denied
690 * @throws FileAlreadyExistsException If directory <code>dir</code> already
691 * exists
692 * @throws FileNotFoundException If parent of <code>dir</code> does not exist
693 * and <code>createParent</code> is false
694 * @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
695 * directory
696 * @throws UnsupportedFileSystemException If file system for <code>dir</code>
697 * is not supported
698 * @throws IOException If an I/O error occurred
699 *
700 * Exceptions applicable to file systems accessed over RPC:
701 * @throws RpcClientException If an exception occurred in the RPC client
702 * @throws UnexpectedServerException If server implementation throws
703 * undeclared exception to RPC server
704 *
705 * RuntimeExceptions:
706 * @throws InvalidPathException If path <code>dir</code> is not valid
707 */
708 public void mkdir(final Path dir, final FsPermission permission,
709 final boolean createParent) throws AccessControlException,
710 FileAlreadyExistsException, FileNotFoundException,
711 ParentNotDirectoryException, UnsupportedFileSystemException,
712 IOException {
713 final Path absDir = fixRelativePart(dir);
714 final FsPermission absFerms = (permission == null ?
715 FsPermission.getDirDefault() : permission).applyUMask(umask);
716 new FSLinkResolver<Void>() {
717 @Override
718 public Void next(final AbstractFileSystem fs, final Path p)
719 throws IOException, UnresolvedLinkException {
720 fs.mkdir(p, absFerms, createParent);
721 return null;
722 }
723 }.resolve(this, absDir);
724 }
725
726 /**
727 * Delete a file.
728 * @param f the path to delete.
729 * @param recursive if path is a directory and set to
730 * true, the directory is deleted else throws an exception. In
731 * case of a file the recursive can be set to either true or false.
732 *
733 * @throws AccessControlException If access is denied
734 * @throws FileNotFoundException If <code>f</code> does not exist
735 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
736 * not supported
737 * @throws IOException If an I/O error occurred
738 *
739 * Exceptions applicable to file systems accessed over RPC:
740 * @throws RpcClientException If an exception occurred in the RPC client
741 * @throws RpcServerException If an exception occurred in the RPC server
742 * @throws UnexpectedServerException If server implementation throws
743 * undeclared exception to RPC server
744 *
745 * RuntimeExceptions:
746 * @throws InvalidPathException If path <code>f</code> is invalid
747 */
748 public boolean delete(final Path f, final boolean recursive)
749 throws AccessControlException, FileNotFoundException,
750 UnsupportedFileSystemException, IOException {
751 Path absF = fixRelativePart(f);
752 return new FSLinkResolver<Boolean>() {
753 @Override
754 public Boolean next(final AbstractFileSystem fs, final Path p)
755 throws IOException, UnresolvedLinkException {
756 return Boolean.valueOf(fs.delete(p, recursive));
757 }
758 }.resolve(this, absF);
759 }
760
761 /**
762 * Opens an FSDataInputStream at the indicated Path using
763 * default buffersize.
764 * @param f the file name to open
765 *
766 * @throws AccessControlException If access is denied
767 * @throws FileNotFoundException If file <code>f</code> does not exist
768 * @throws UnsupportedFileSystemException If file system for <code>f</code>
769 * is not supported
770 * @throws IOException If an I/O error occurred
771 *
772 * Exceptions applicable to file systems accessed over RPC:
773 * @throws RpcClientException If an exception occurred in the RPC client
774 * @throws RpcServerException If an exception occurred in the RPC server
775 * @throws UnexpectedServerException If server implementation throws
776 * undeclared exception to RPC server
777 */
778 public FSDataInputStream open(final Path f) throws AccessControlException,
779 FileNotFoundException, UnsupportedFileSystemException, IOException {
780 final Path absF = fixRelativePart(f);
781 return new FSLinkResolver<FSDataInputStream>() {
782 @Override
783 public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
784 throws IOException, UnresolvedLinkException {
785 return fs.open(p);
786 }
787 }.resolve(this, absF);
788 }
789
790 /**
791 * Opens an FSDataInputStream at the indicated Path.
792 *
793 * @param f the file name to open
794 * @param bufferSize the size of the buffer to be used.
795 *
796 * @throws AccessControlException If access is denied
797 * @throws FileNotFoundException If file <code>f</code> does not exist
798 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
799 * not supported
800 * @throws IOException If an I/O error occurred
801 *
802 * Exceptions applicable to file systems accessed over RPC:
803 * @throws RpcClientException If an exception occurred in the RPC client
804 * @throws RpcServerException If an exception occurred in the RPC server
805 * @throws UnexpectedServerException If server implementation throws
806 * undeclared exception to RPC server
807 */
808 public FSDataInputStream open(final Path f, final int bufferSize)
809 throws AccessControlException, FileNotFoundException,
810 UnsupportedFileSystemException, IOException {
811 final Path absF = fixRelativePart(f);
812 return new FSLinkResolver<FSDataInputStream>() {
813 @Override
814 public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
815 throws IOException, UnresolvedLinkException {
816 return fs.open(p, bufferSize);
817 }
818 }.resolve(this, absF);
819 }
820
821 /**
822 * Set replication for an existing file.
823 *
824 * @param f file name
825 * @param replication new replication
826 *
827 * @return true if successful
828 *
829 * @throws AccessControlException If access is denied
830 * @throws FileNotFoundException If file <code>f</code> does not exist
831 * @throws IOException If an I/O error occurred
832 *
833 * Exceptions applicable to file systems accessed over RPC:
834 * @throws RpcClientException If an exception occurred in the RPC client
835 * @throws RpcServerException If an exception occurred in the RPC server
836 * @throws UnexpectedServerException If server implementation throws
837 * undeclared exception to RPC server
838 */
839 public boolean setReplication(final Path f, final short replication)
840 throws AccessControlException, FileNotFoundException,
841 IOException {
842 final Path absF = fixRelativePart(f);
843 return new FSLinkResolver<Boolean>() {
844 @Override
845 public Boolean next(final AbstractFileSystem fs, final Path p)
846 throws IOException, UnresolvedLinkException {
847 return Boolean.valueOf(fs.setReplication(p, replication));
848 }
849 }.resolve(this, absF);
850 }
851
852 /**
853 * Renames Path src to Path dst
854 * <ul>
855 * <li
856 * <li>Fails if src is a file and dst is a directory.
857 * <li>Fails if src is a directory and dst is a file.
858 * <li>Fails if the parent of dst does not exist or is a file.
859 * </ul>
860 * <p>
861 * If OVERWRITE option is not passed as an argument, rename fails if the dst
862 * already exists.
863 * <p>
864 * If OVERWRITE option is passed as an argument, rename overwrites the dst if
865 * it is a file or an empty directory. Rename fails if dst is a non-empty
866 * directory.
867 * <p>
868 * Note that atomicity of rename is dependent on the file system
869 * implementation. Please refer to the file system documentation for details
870 * <p>
871 *
872 * @param src path to be renamed
873 * @param dst new path after rename
874 *
875 * @throws AccessControlException If access is denied
876 * @throws FileAlreadyExistsException If <code>dst</code> already exists and
877 * <code>options</options> has {@link Options.Rename#OVERWRITE}
878 * option false.
879 * @throws FileNotFoundException If <code>src</code> does not exist
880 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
881 * directory
882 * @throws UnsupportedFileSystemException If file system for <code>src</code>
883 * and <code>dst</code> is not supported
884 * @throws IOException If an I/O error occurred
885 *
886 * Exceptions applicable to file systems accessed over RPC:
887 * @throws RpcClientException If an exception occurred in the RPC client
888 * @throws RpcServerException If an exception occurred in the RPC server
889 * @throws UnexpectedServerException If server implementation throws
890 * undeclared exception to RPC server
891 */
892 public void rename(final Path src, final Path dst,
893 final Options.Rename... options) throws AccessControlException,
894 FileAlreadyExistsException, FileNotFoundException,
895 ParentNotDirectoryException, UnsupportedFileSystemException,
896 IOException {
897 final Path absSrc = fixRelativePart(src);
898 final Path absDst = fixRelativePart(dst);
899 AbstractFileSystem srcFS = getFSofPath(absSrc);
900 AbstractFileSystem dstFS = getFSofPath(absDst);
901 if(!srcFS.getUri().equals(dstFS.getUri())) {
902 throw new IOException("Renames across AbstractFileSystems not supported");
903 }
904 try {
905 srcFS.rename(absSrc, absDst, options);
906 } catch (UnresolvedLinkException e) {
907 /* We do not know whether the source or the destination path
908 * was unresolved. Resolve the source path up until the final
909 * path component, then fully resolve the destination.
910 */
911 final Path source = resolveIntermediate(absSrc);
912 new FSLinkResolver<Void>() {
913 @Override
914 public Void next(final AbstractFileSystem fs, final Path p)
915 throws IOException, UnresolvedLinkException {
916 fs.rename(source, p, options);
917 return null;
918 }
919 }.resolve(this, absDst);
920 }
921 }
922
923 /**
924 * Set permission of a path.
925 * @param f
926 * @param permission - the new absolute permission (umask is not applied)
927 *
928 * @throws AccessControlException If access is denied
929 * @throws FileNotFoundException If <code>f</code> does not exist
930 * @throws UnsupportedFileSystemException If file system for <code>f</code>
931 * is not supported
932 * @throws IOException If an I/O error occurred
933 *
934 * Exceptions applicable to file systems accessed over RPC:
935 * @throws RpcClientException If an exception occurred in the RPC client
936 * @throws RpcServerException If an exception occurred in the RPC server
937 * @throws UnexpectedServerException If server implementation throws
938 * undeclared exception to RPC server
939 */
940 public void setPermission(final Path f, final FsPermission permission)
941 throws AccessControlException, FileNotFoundException,
942 UnsupportedFileSystemException, IOException {
943 final Path absF = fixRelativePart(f);
944 new FSLinkResolver<Void>() {
945 @Override
946 public Void next(final AbstractFileSystem fs, final Path p)
947 throws IOException, UnresolvedLinkException {
948 fs.setPermission(p, permission);
949 return null;
950 }
951 }.resolve(this, absF);
952 }
953
954 /**
955 * Set owner of a path (i.e. a file or a directory). The parameters username
956 * and groupname cannot both be null.
957 *
958 * @param f The path
959 * @param username If it is null, the original username remains unchanged.
960 * @param groupname If it is null, the original groupname remains unchanged.
961 *
962 * @throws AccessControlException If access is denied
963 * @throws FileNotFoundException If <code>f</code> does not exist
964 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
965 * not supported
966 * @throws IOException If an I/O error occurred
967 *
968 * Exceptions applicable to file systems accessed over RPC:
969 * @throws RpcClientException If an exception occurred in the RPC client
970 * @throws RpcServerException If an exception occurred in the RPC server
971 * @throws UnexpectedServerException If server implementation throws
972 * undeclared exception to RPC server
973 *
974 * RuntimeExceptions:
975 * @throws HadoopIllegalArgumentException If <code>username</code> or
976 * <code>groupname</code> is invalid.
977 */
978 public void setOwner(final Path f, final String username,
979 final String groupname) throws AccessControlException,
980 UnsupportedFileSystemException, FileNotFoundException,
981 IOException {
982 if ((username == null) && (groupname == null)) {
983 throw new HadoopIllegalArgumentException(
984 "username and groupname cannot both be null");
985 }
986 final Path absF = fixRelativePart(f);
987 new FSLinkResolver<Void>() {
988 @Override
989 public Void next(final AbstractFileSystem fs, final Path p)
990 throws IOException, UnresolvedLinkException {
991 fs.setOwner(p, username, groupname);
992 return null;
993 }
994 }.resolve(this, absF);
995 }
996
997 /**
998 * Set access time of a file.
999 * @param f The path
1000 * @param mtime Set the modification time of this file.
1001 * The number of milliseconds since epoch (Jan 1, 1970).
1002 * A value of -1 means that this call should not set modification time.
1003 * @param atime Set the access time of this file.
1004 * The number of milliseconds since Jan 1, 1970.
1005 * A value of -1 means that this call should not set access time.
1006 *
1007 * @throws AccessControlException If access is denied
1008 * @throws FileNotFoundException If <code>f</code> does not exist
1009 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1010 * not supported
1011 * @throws IOException If an I/O error occurred
1012 *
1013 * Exceptions applicable to file systems accessed over RPC:
1014 * @throws RpcClientException If an exception occurred in the RPC client
1015 * @throws RpcServerException If an exception occurred in the RPC server
1016 * @throws UnexpectedServerException If server implementation throws
1017 * undeclared exception to RPC server
1018 */
1019 public void setTimes(final Path f, final long mtime, final long atime)
1020 throws AccessControlException, FileNotFoundException,
1021 UnsupportedFileSystemException, IOException {
1022 final Path absF = fixRelativePart(f);
1023 new FSLinkResolver<Void>() {
1024 @Override
1025 public Void next(final AbstractFileSystem fs, final Path p)
1026 throws IOException, UnresolvedLinkException {
1027 fs.setTimes(p, mtime, atime);
1028 return null;
1029 }
1030 }.resolve(this, absF);
1031 }
1032
1033 /**
1034 * Get the checksum of a file.
1035 *
1036 * @param f file path
1037 *
1038 * @return The file checksum. The default return value is null,
1039 * which indicates that no checksum algorithm is implemented
1040 * in the corresponding FileSystem.
1041 *
1042 * @throws AccessControlException If access is denied
1043 * @throws FileNotFoundException If <code>f</code> does not exist
1044 * @throws IOException If an I/O error occurred
1045 *
1046 * Exceptions applicable to file systems accessed over RPC:
1047 * @throws RpcClientException If an exception occurred in the RPC client
1048 * @throws RpcServerException If an exception occurred in the RPC server
1049 * @throws UnexpectedServerException If server implementation throws
1050 * undeclared exception to RPC server
1051 */
1052 public FileChecksum getFileChecksum(final Path f)
1053 throws AccessControlException, FileNotFoundException,
1054 IOException {
1055 final Path absF = fixRelativePart(f);
1056 return new FSLinkResolver<FileChecksum>() {
1057 @Override
1058 public FileChecksum next(final AbstractFileSystem fs, final Path p)
1059 throws IOException, UnresolvedLinkException {
1060 return fs.getFileChecksum(p);
1061 }
1062 }.resolve(this, absF);
1063 }
1064
1065 /**
1066 * Set the verify checksum flag for the file system denoted by the path.
1067 * This is only applicable if the
1068 * corresponding FileSystem supports checksum. By default doesn't do anything.
1069 * @param verifyChecksum
1070 * @param f set the verifyChecksum for the Filesystem containing this path
1071 *
1072 * @throws AccessControlException If access is denied
1073 * @throws FileNotFoundException If <code>f</code> does not exist
1074 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1075 * not supported
1076 * @throws IOException If an I/O error occurred
1077 *
1078 * Exceptions applicable to file systems accessed over RPC:
1079 * @throws RpcClientException If an exception occurred in the RPC client
1080 * @throws RpcServerException If an exception occurred in the RPC server
1081 * @throws UnexpectedServerException If server implementation throws
1082 * undeclared exception to RPC server
1083 */
1084 public void setVerifyChecksum(final boolean verifyChecksum, final Path f)
1085 throws AccessControlException, FileNotFoundException,
1086 UnsupportedFileSystemException, IOException {
1087 final Path absF = resolve(fixRelativePart(f));
1088 getFSofPath(absF).setVerifyChecksum(verifyChecksum);
1089 }
1090
1091 /**
1092 * Return a file status object that represents the path.
1093 * @param f The path we want information from
1094 *
1095 * @return a FileStatus object
1096 *
1097 * @throws AccessControlException If access is denied
1098 * @throws FileNotFoundException If <code>f</code> does not exist
1099 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1100 * not supported
1101 * @throws IOException If an I/O error occurred
1102 *
1103 * Exceptions applicable to file systems accessed over RPC:
1104 * @throws RpcClientException If an exception occurred in the RPC client
1105 * @throws RpcServerException If an exception occurred in the RPC server
1106 * @throws UnexpectedServerException If server implementation throws
1107 * undeclared exception to RPC server
1108 */
1109 public FileStatus getFileStatus(final Path f) throws AccessControlException,
1110 FileNotFoundException, UnsupportedFileSystemException, IOException {
1111 final Path absF = fixRelativePart(f);
1112 return new FSLinkResolver<FileStatus>() {
1113 @Override
1114 public FileStatus next(final AbstractFileSystem fs, final Path p)
1115 throws IOException, UnresolvedLinkException {
1116 return fs.getFileStatus(p);
1117 }
1118 }.resolve(this, absF);
1119 }
1120
1121 /**
1122 * Return a file status object that represents the path. If the path
1123 * refers to a symlink then the FileStatus of the symlink is returned.
1124 * The behavior is equivalent to #getFileStatus() if the underlying
1125 * file system does not support symbolic links.
1126 * @param f The path we want information from.
1127 * @return A FileStatus object
1128 *
1129 * @throws AccessControlException If access is denied
1130 * @throws FileNotFoundException If <code>f</code> does not exist
1131 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1132 * not supported
1133 * @throws IOException If an I/O error occurred
1134 */
1135 public FileStatus getFileLinkStatus(final Path f)
1136 throws AccessControlException, FileNotFoundException,
1137 UnsupportedFileSystemException, IOException {
1138 final Path absF = fixRelativePart(f);
1139 return new FSLinkResolver<FileStatus>() {
1140 @Override
1141 public FileStatus next(final AbstractFileSystem fs, final Path p)
1142 throws IOException, UnresolvedLinkException {
1143 FileStatus fi = fs.getFileLinkStatus(p);
1144 if (fi.isSymlink()) {
1145 fi.setSymlink(FSLinkResolver.qualifySymlinkTarget(fs.getUri(), p,
1146 fi.getSymlink()));
1147 }
1148 return fi;
1149 }
1150 }.resolve(this, absF);
1151 }
1152
1153 /**
1154 * Returns the target of the given symbolic link as it was specified
1155 * when the link was created. Links in the path leading up to the
1156 * final path component are resolved transparently.
1157 *
1158 * @param f the path to return the target of
1159 * @return The un-interpreted target of the symbolic link.
1160 *
1161 * @throws AccessControlException If access is denied
1162 * @throws FileNotFoundException If path <code>f</code> does not exist
1163 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1164 * not supported
1165 * @throws IOException If the given path does not refer to a symlink
1166 * or an I/O error occurred
1167 */
1168 public Path getLinkTarget(final Path f) throws AccessControlException,
1169 FileNotFoundException, UnsupportedFileSystemException, IOException {
1170 final Path absF = fixRelativePart(f);
1171 return new FSLinkResolver<Path>() {
1172 @Override
1173 public Path next(final AbstractFileSystem fs, final Path p)
1174 throws IOException, UnresolvedLinkException {
1175 FileStatus fi = fs.getFileLinkStatus(p);
1176 return fi.getSymlink();
1177 }
1178 }.resolve(this, absF);
1179 }
1180
1181 /**
1182 * Return blockLocation of the given file for the given offset and len.
1183 * For a nonexistent file or regions, null will be returned.
1184 *
1185 * This call is most helpful with DFS, where it returns
1186 * hostnames of machines that contain the given file.
1187 *
1188 * @param f - get blocklocations of this file
1189 * @param start position (byte offset)
1190 * @param len (in bytes)
1191 *
1192 * @return block locations for given file at specified offset of len
1193 *
1194 * @throws AccessControlException If access is denied
1195 * @throws FileNotFoundException If <code>f</code> does not exist
1196 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1197 * not supported
1198 * @throws IOException If an I/O error occurred
1199 *
1200 * Exceptions applicable to file systems accessed over RPC:
1201 * @throws RpcClientException If an exception occurred in the RPC client
1202 * @throws RpcServerException If an exception occurred in the RPC server
1203 * @throws UnexpectedServerException If server implementation throws
1204 * undeclared exception to RPC server
1205 *
1206 * RuntimeExceptions:
1207 * @throws InvalidPathException If path <code>f</code> is invalid
1208 */
1209 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
1210 @InterfaceStability.Evolving
1211 public BlockLocation[] getFileBlockLocations(final Path f, final long start,
1212 final long len) throws AccessControlException, FileNotFoundException,
1213 UnsupportedFileSystemException, IOException {
1214 final Path absF = fixRelativePart(f);
1215 return new FSLinkResolver<BlockLocation[]>() {
1216 @Override
1217 public BlockLocation[] next(final AbstractFileSystem fs, final Path p)
1218 throws IOException, UnresolvedLinkException {
1219 return fs.getFileBlockLocations(p, start, len);
1220 }
1221 }.resolve(this, absF);
1222 }
1223
1224 /**
1225 * Returns a status object describing the use and capacity of the
1226 * file system denoted by the Parh argument p.
1227 * If the file system has multiple partitions, the
1228 * use and capacity of the partition pointed to by the specified
1229 * path is reflected.
1230 *
1231 * @param f Path for which status should be obtained. null means the
1232 * root partition of the default file system.
1233 *
1234 * @return a FsStatus object
1235 *
1236 * @throws AccessControlException If access is denied
1237 * @throws FileNotFoundException If <code>f</code> does not exist
1238 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1239 * not supported
1240 * @throws IOException If an I/O error occurred
1241 *
1242 * Exceptions applicable to file systems accessed over RPC:
1243 * @throws RpcClientException If an exception occurred in the RPC client
1244 * @throws RpcServerException If an exception occurred in the RPC server
1245 * @throws UnexpectedServerException If server implementation throws
1246 * undeclared exception to RPC server
1247 */
1248 public FsStatus getFsStatus(final Path f) throws AccessControlException,
1249 FileNotFoundException, UnsupportedFileSystemException, IOException {
1250 if (f == null) {
1251 return defaultFS.getFsStatus();
1252 }
1253 final Path absF = fixRelativePart(f);
1254 return new FSLinkResolver<FsStatus>() {
1255 @Override
1256 public FsStatus next(final AbstractFileSystem fs, final Path p)
1257 throws IOException, UnresolvedLinkException {
1258 return fs.getFsStatus(p);
1259 }
1260 }.resolve(this, absF);
1261 }
1262
1263 /**
1264 * Creates a symbolic link to an existing file. An exception is thrown if
1265 * the symlink exits, the user does not have permission to create symlink,
1266 * or the underlying file system does not support symlinks.
1267 *
1268 * Symlink permissions are ignored, access to a symlink is determined by
1269 * the permissions of the symlink target.
1270 *
1271 * Symlinks in paths leading up to the final path component are resolved
1272 * transparently. If the final path component refers to a symlink some
1273 * functions operate on the symlink itself, these are:
1274 * - delete(f) and deleteOnExit(f) - Deletes the symlink.
1275 * - rename(src, dst) - If src refers to a symlink, the symlink is
1276 * renamed. If dst refers to a symlink, the symlink is over-written.
1277 * - getLinkTarget(f) - Returns the target of the symlink.
1278 * - getFileLinkStatus(f) - Returns a FileStatus object describing
1279 * the symlink.
1280 * Some functions, create() and mkdir(), expect the final path component
1281 * does not exist. If they are given a path that refers to a symlink that
1282 * does exist they behave as if the path referred to an existing file or
1283 * directory. All other functions fully resolve, ie follow, the symlink.
1284 * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
1285 * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
1286 * getFsStatus, getFileStatus, exists, and listStatus.
1287 *
1288 * Symlink targets are stored as given to createSymlink, assuming the
1289 * underlying file system is capable of storing a fully qualified URI.
1290 * Dangling symlinks are permitted. FileContext supports four types of
1291 * symlink targets, and resolves them as follows
1292 * <pre>
1293 * Given a path referring to a symlink of form:
1294 *
1295 * <---X--->
1296 * fs://host/A/B/link
1297 * <-----Y----->
1298 *
1299 * In this path X is the scheme and authority that identify the file system,
1300 * and Y is the path leading up to the final path component "link". If Y is
1301 * a symlink itself then let Y' be the target of Y and X' be the scheme and
1302 * authority of Y'. Symlink targets may:
1303 *
1304 * 1. Fully qualified URIs
1305 *
1306 * fs://hostX/A/B/file Resolved according to the target file system.
1307 *
1308 * 2. Partially qualified URIs (eg scheme but no host)
1309 *
1310 * fs:///A/B/file Resolved according to the target file system. Eg resolving
1311 * a symlink to hdfs:///A results in an exception because
1312 * HDFS URIs must be fully qualified, while a symlink to
1313 * file:///A will not since Hadoop's local file systems
1314 * require partially qualified URIs.
1315 *
1316 * 3. Relative paths
1317 *
1318 * path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path
1319 * is "../B/file" then [Y'][path] is hdfs://host/B/file
1320 *
1321 * 4. Absolute paths
1322 *
1323 * path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
1324 * is "/file" then [X][path] is hdfs://host/file
1325 * </pre>
1326 *
1327 * @param target the target of the symbolic link
1328 * @param link the path to be created that points to target
1329 * @param createParent if true then missing parent dirs are created if
1330 * false then parent must exist
1331 *
1332 *
1333 * @throws AccessControlException If access is denied
1334 * @throws FileAlreadyExistsException If file <code>linkcode> already exists
1335 * @throws FileNotFoundException If <code>target</code> does not exist
1336 * @throws ParentNotDirectoryException If parent of <code>link</code> is not a
1337 * directory.
1338 * @throws UnsupportedFileSystemException If file system for
1339 * <code>target</code> or <code>link</code> is not supported
1340 * @throws IOException If an I/O error occurred
1341 */
1342 @SuppressWarnings("deprecation")
1343 public void createSymlink(final Path target, final Path link,
1344 final boolean createParent) throws AccessControlException,
1345 FileAlreadyExistsException, FileNotFoundException,
1346 ParentNotDirectoryException, UnsupportedFileSystemException,
1347 IOException {
1348 if (!FileSystem.isSymlinksEnabled()) {
1349 throw new UnsupportedOperationException("Symlinks not supported");
1350 }
1351 final Path nonRelLink = fixRelativePart(link);
1352 new FSLinkResolver<Void>() {
1353 @Override
1354 public Void next(final AbstractFileSystem fs, final Path p)
1355 throws IOException, UnresolvedLinkException {
1356 fs.createSymlink(target, p, createParent);
1357 return null;
1358 }
1359 }.resolve(this, nonRelLink);
1360 }
1361
1362 /**
1363 * List the statuses of the files/directories in the given path if the path is
1364 * a directory.
1365 *
1366 * @param f is the path
1367 *
1368 * @return an iterator that traverses statuses of the files/directories
1369 * in the given path
1370 *
1371 * @throws AccessControlException If access is denied
1372 * @throws FileNotFoundException If <code>f</code> does not exist
1373 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1374 * not supported
1375 * @throws IOException If an I/O error occurred
1376 *
1377 * Exceptions applicable to file systems accessed over RPC:
1378 * @throws RpcClientException If an exception occurred in the RPC client
1379 * @throws RpcServerException If an exception occurred in the RPC server
1380 * @throws UnexpectedServerException If server implementation throws
1381 * undeclared exception to RPC server
1382 */
1383 public RemoteIterator<FileStatus> listStatus(final Path f) throws
1384 AccessControlException, FileNotFoundException,
1385 UnsupportedFileSystemException, IOException {
1386 final Path absF = fixRelativePart(f);
1387 return new FSLinkResolver<RemoteIterator<FileStatus>>() {
1388 @Override
1389 public RemoteIterator<FileStatus> next(
1390 final AbstractFileSystem fs, final Path p)
1391 throws IOException, UnresolvedLinkException {
1392 return fs.listStatusIterator(p);
1393 }
1394 }.resolve(this, absF);
1395 }
1396
1397 /**
1398 * @return an iterator over the corrupt files under the given path
1399 * (may contain duplicates if a file has more than one corrupt block)
1400 * @throws IOException
1401 */
1402 public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1403 throws IOException {
1404 final Path absF = fixRelativePart(path);
1405 return new FSLinkResolver<RemoteIterator<Path>>() {
1406 @Override
1407 public RemoteIterator<Path> next(final AbstractFileSystem fs,
1408 final Path p)
1409 throws IOException, UnresolvedLinkException {
1410 return fs.listCorruptFileBlocks(p);
1411 }
1412 }.resolve(this, absF);
1413 }
1414
1415 /**
1416 * List the statuses of the files/directories in the given path if the path is
1417 * a directory.
1418 * Return the file's status and block locations If the path is a file.
1419 *
1420 * If a returned status is a file, it contains the file's block locations.
1421 *
1422 * @param f is the path
1423 *
1424 * @return an iterator that traverses statuses of the files/directories
1425 * in the given path
1426 * If any IO exception (for example the input directory gets deleted while
1427 * listing is being executed), next() or hasNext() of the returned iterator
1428 * may throw a RuntimeException with the io exception as the cause.
1429 *
1430 * @throws AccessControlException If access is denied
1431 * @throws FileNotFoundException If <code>f</code> does not exist
1432 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1433 * not supported
1434 * @throws IOException If an I/O error occurred
1435 *
1436 * Exceptions applicable to file systems accessed over RPC:
1437 * @throws RpcClientException If an exception occurred in the RPC client
1438 * @throws RpcServerException If an exception occurred in the RPC server
1439 * @throws UnexpectedServerException If server implementation throws
1440 * undeclared exception to RPC server
1441 */
1442 public RemoteIterator<LocatedFileStatus> listLocatedStatus(
1443 final Path f) throws
1444 AccessControlException, FileNotFoundException,
1445 UnsupportedFileSystemException, IOException {
1446 final Path absF = fixRelativePart(f);
1447 return new FSLinkResolver<RemoteIterator<LocatedFileStatus>>() {
1448 @Override
1449 public RemoteIterator<LocatedFileStatus> next(
1450 final AbstractFileSystem fs, final Path p)
1451 throws IOException, UnresolvedLinkException {
1452 return fs.listLocatedStatus(p);
1453 }
1454 }.resolve(this, absF);
1455 }
1456
1457 /**
1458 * Mark a path to be deleted on JVM shutdown.
1459 *
1460 * @param f the existing path to delete.
1461 *
1462 * @return true if deleteOnExit is successful, otherwise false.
1463 *
1464 * @throws AccessControlException If access is denied
1465 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1466 * not supported
1467 * @throws IOException If an I/O error occurred
1468 *
1469 * Exceptions applicable to file systems accessed over RPC:
1470 * @throws RpcClientException If an exception occurred in the RPC client
1471 * @throws RpcServerException If an exception occurred in the RPC server
1472 * @throws UnexpectedServerException If server implementation throws
1473 * undeclared exception to RPC server
1474 */
1475 public boolean deleteOnExit(Path f) throws AccessControlException,
1476 IOException {
1477 if (!this.util().exists(f)) {
1478 return false;
1479 }
1480 synchronized (DELETE_ON_EXIT) {
1481 if (DELETE_ON_EXIT.isEmpty()) {
1482 ShutdownHookManager.get().addShutdownHook(FINALIZER, SHUTDOWN_HOOK_PRIORITY);
1483 }
1484
1485 Set<Path> set = DELETE_ON_EXIT.get(this);
1486 if (set == null) {
1487 set = new TreeSet<Path>();
1488 DELETE_ON_EXIT.put(this, set);
1489 }
1490 set.add(f);
1491 }
1492 return true;
1493 }
1494
1495 private final Util util;
1496 public Util util() {
1497 return util;
1498 }
1499
1500
1501 /**
1502 * Utility/library methods built over the basic FileContext methods.
1503 * Since this are library functions, the oprtation are not atomic
1504 * and some of them may partially complete if other threads are making
1505 * changes to the same part of the name space.
1506 */
1507 public class Util {
1508 /**
1509 * Does the file exist?
1510 * Note: Avoid using this method if you already have FileStatus in hand.
1511 * Instead reuse the FileStatus
1512 * @param f the file or dir to be checked
1513 *
1514 * @throws AccessControlException If access is denied
1515 * @throws IOException If an I/O error occurred
1516 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1517 * not supported
1518 *
1519 * Exceptions applicable to file systems accessed over RPC:
1520 * @throws RpcClientException If an exception occurred in the RPC client
1521 * @throws RpcServerException If an exception occurred in the RPC server
1522 * @throws UnexpectedServerException If server implementation throws
1523 * undeclared exception to RPC server
1524 */
1525 public boolean exists(final Path f) throws AccessControlException,
1526 UnsupportedFileSystemException, IOException {
1527 try {
1528 FileStatus fs = FileContext.this.getFileStatus(f);
1529 assert fs != null;
1530 return true;
1531 } catch (FileNotFoundException e) {
1532 return false;
1533 }
1534 }
1535
1536 /**
1537 * Return a list of file status objects that corresponds to supplied paths
1538 * excluding those non-existent paths.
1539 *
1540 * @param paths list of paths we want information from
1541 *
1542 * @return a list of FileStatus objects
1543 *
1544 * @throws AccessControlException If access is denied
1545 * @throws IOException If an I/O error occurred
1546 *
1547 * Exceptions applicable to file systems accessed over RPC:
1548 * @throws RpcClientException If an exception occurred in the RPC client
1549 * @throws RpcServerException If an exception occurred in the RPC server
1550 * @throws UnexpectedServerException If server implementation throws
1551 * undeclared exception to RPC server
1552 */
1553 private FileStatus[] getFileStatus(Path[] paths)
1554 throws AccessControlException, IOException {
1555 if (paths == null) {
1556 return null;
1557 }
1558 ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
1559 for (int i = 0; i < paths.length; i++) {
1560 try {
1561 results.add(FileContext.this.getFileStatus(paths[i]));
1562 } catch (FileNotFoundException fnfe) {
1563 // ignoring
1564 }
1565 }
1566 return results.toArray(new FileStatus[results.size()]);
1567 }
1568
1569
1570 /**
1571 * Return the {@link ContentSummary} of path f.
1572 * @param f path
1573 *
1574 * @return the {@link ContentSummary} of path f.
1575 *
1576 * @throws AccessControlException If access is denied
1577 * @throws FileNotFoundException If <code>f</code> does not exist
1578 * @throws UnsupportedFileSystemException If file system for
1579 * <code>f</code> is not supported
1580 * @throws IOException If an I/O error occurred
1581 *
1582 * Exceptions applicable to file systems accessed over RPC:
1583 * @throws RpcClientException If an exception occurred in the RPC client
1584 * @throws RpcServerException If an exception occurred in the RPC server
1585 * @throws UnexpectedServerException If server implementation throws
1586 * undeclared exception to RPC server
1587 */
1588 public ContentSummary getContentSummary(Path f)
1589 throws AccessControlException, FileNotFoundException,
1590 UnsupportedFileSystemException, IOException {
1591 FileStatus status = FileContext.this.getFileStatus(f);
1592 if (status.isFile()) {
1593 return new ContentSummary(status.getLen(), 1, 0);
1594 }
1595 long[] summary = {0, 0, 1};
1596 RemoteIterator<FileStatus> statusIterator =
1597 FileContext.this.listStatus(f);
1598 while(statusIterator.hasNext()) {
1599 FileStatus s = statusIterator.next();
1600 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1601 new ContentSummary(s.getLen(), 1, 0);
1602 summary[0] += c.getLength();
1603 summary[1] += c.getFileCount();
1604 summary[2] += c.getDirectoryCount();
1605 }
1606 return new ContentSummary(summary[0], summary[1], summary[2]);
1607 }
1608
1609 /**
1610 * See {@link #listStatus(Path[], PathFilter)}
1611 */
1612 public FileStatus[] listStatus(Path[] files) throws AccessControlException,
1613 FileNotFoundException, IOException {
1614 return listStatus(files, DEFAULT_FILTER);
1615 }
1616
1617 /**
1618 * Filter files/directories in the given path using the user-supplied path
1619 * filter.
1620 *
1621 * @param f is the path name
1622 * @param filter is the user-supplied path filter
1623 *
1624 * @return an array of FileStatus objects for the files under the given path
1625 * after applying the filter
1626 *
1627 * @throws AccessControlException If access is denied
1628 * @throws FileNotFoundException If <code>f</code> does not exist
1629 * @throws UnsupportedFileSystemException If file system for
1630 * <code>pathPattern</code> is not supported
1631 * @throws IOException If an I/O error occurred
1632 *
1633 * Exceptions applicable to file systems accessed over RPC:
1634 * @throws RpcClientException If an exception occurred in the RPC client
1635 * @throws RpcServerException If an exception occurred in the RPC server
1636 * @throws UnexpectedServerException If server implementation throws
1637 * undeclared exception to RPC server
1638 */
1639 public FileStatus[] listStatus(Path f, PathFilter filter)
1640 throws AccessControlException, FileNotFoundException,
1641 UnsupportedFileSystemException, IOException {
1642 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1643 listStatus(results, f, filter);
1644 return results.toArray(new FileStatus[results.size()]);
1645 }
1646
1647 /**
1648 * Filter files/directories in the given list of paths using user-supplied
1649 * path filter.
1650 *
1651 * @param files is a list of paths
1652 * @param filter is the filter
1653 *
1654 * @return a list of statuses for the files under the given paths after
1655 * applying the filter
1656 *
1657 * @throws AccessControlException If access is denied
1658 * @throws FileNotFoundException If a file in <code>files</code> does not
1659 * exist
1660 * @throws IOException If an I/O error occurred
1661 *
1662 * Exceptions applicable to file systems accessed over RPC:
1663 * @throws RpcClientException If an exception occurred in the RPC client
1664 * @throws RpcServerException If an exception occurred in the RPC server
1665 * @throws UnexpectedServerException If server implementation throws
1666 * undeclared exception to RPC server
1667 */
1668 public FileStatus[] listStatus(Path[] files, PathFilter filter)
1669 throws AccessControlException, FileNotFoundException, IOException {
1670 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1671 for (int i = 0; i < files.length; i++) {
1672 listStatus(results, files[i], filter);
1673 }
1674 return results.toArray(new FileStatus[results.size()]);
1675 }
1676
1677 /*
1678 * Filter files/directories in the given path using the user-supplied path
1679 * filter. Results are added to the given array <code>results</code>.
1680 */
1681 private void listStatus(ArrayList<FileStatus> results, Path f,
1682 PathFilter filter) throws AccessControlException,
1683 FileNotFoundException, IOException {
1684 FileStatus[] listing = listStatus(f);
1685 if (listing != null) {
1686 for (int i = 0; i < listing.length; i++) {
1687 if (filter.accept(listing[i].getPath())) {
1688 results.add(listing[i]);
1689 }
1690 }
1691 }
1692 }
1693
1694 /**
1695 * List the statuses of the files/directories in the given path
1696 * if the path is a directory.
1697 *
1698 * @param f is the path
1699 *
1700 * @return an array that contains statuses of the files/directories
1701 * in the given path
1702 *
1703 * @throws AccessControlException If access is denied
1704 * @throws FileNotFoundException If <code>f</code> does not exist
1705 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1706 * not supported
1707 * @throws IOException If an I/O error occurred
1708 *
1709 * Exceptions applicable to file systems accessed over RPC:
1710 * @throws RpcClientException If an exception occurred in the RPC client
1711 * @throws RpcServerException If an exception occurred in the RPC server
1712 * @throws UnexpectedServerException If server implementation throws
1713 * undeclared exception to RPC server
1714 */
1715 public FileStatus[] listStatus(final Path f) throws AccessControlException,
1716 FileNotFoundException, UnsupportedFileSystemException,
1717 IOException {
1718 final Path absF = fixRelativePart(f);
1719 return new FSLinkResolver<FileStatus[]>() {
1720 @Override
1721 public FileStatus[] next(final AbstractFileSystem fs, final Path p)
1722 throws IOException, UnresolvedLinkException {
1723 return fs.listStatus(p);
1724 }
1725 }.resolve(FileContext.this, absF);
1726 }
1727
1728 /**
1729 * List the statuses and block locations of the files in the given path.
1730 *
1731 * If the path is a directory,
1732 * if recursive is false, returns files in the directory;
1733 * if recursive is true, return files in the subtree rooted at the path.
1734 * The subtree is traversed in the depth-first order.
1735 * If the path is a file, return the file's status and block locations.
1736 * Files across symbolic links are also returned.
1737 *
1738 * @param f is the path
1739 * @param recursive if the subdirectories need to be traversed recursively
1740 *
1741 * @return an iterator that traverses statuses of the files
1742 * If any IO exception (for example a sub-directory gets deleted while
1743 * listing is being executed), next() or hasNext() of the returned iterator
1744 * may throw a RuntimeException with the IO exception as the cause.
1745 *
1746 * @throws AccessControlException If access is denied
1747 * @throws FileNotFoundException If <code>f</code> does not exist
1748 * @throws UnsupportedFileSystemException If file system for <code>f</code>
1749 * is not supported
1750 * @throws IOException If an I/O error occurred
1751 *
1752 * Exceptions applicable to file systems accessed over RPC:
1753 * @throws RpcClientException If an exception occurred in the RPC client
1754 * @throws RpcServerException If an exception occurred in the RPC server
1755 * @throws UnexpectedServerException If server implementation throws
1756 * undeclared exception to RPC server
1757 */
1758 public RemoteIterator<LocatedFileStatus> listFiles(
1759 final Path f, final boolean recursive) throws AccessControlException,
1760 FileNotFoundException, UnsupportedFileSystemException,
1761 IOException {
1762 return new RemoteIterator<LocatedFileStatus>() {
1763 private Stack<RemoteIterator<LocatedFileStatus>> itors =
1764 new Stack<RemoteIterator<LocatedFileStatus>>();
1765 RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f);
1766 LocatedFileStatus curFile;
1767
1768 /**
1769 * Returns <tt>true</tt> if the iterator has more files.
1770 *
1771 * @return <tt>true</tt> if the iterator has more files.
1772 * @throws AccessControlException if not allowed to access next
1773 * file's status or locations
1774 * @throws FileNotFoundException if next file does not exist any more
1775 * @throws UnsupportedFileSystemException if next file's
1776 * fs is unsupported
1777 * @throws IOException for all other IO errors
1778 * for example, NameNode is not avaialbe or
1779 * NameNode throws IOException due to an error
1780 * while getting the status or block locations
1781 */
1782 @Override
1783 public boolean hasNext() throws IOException {
1784 while (curFile == null) {
1785 if (curItor.hasNext()) {
1786 handleFileStat(curItor.next());
1787 } else if (!itors.empty()) {
1788 curItor = itors.pop();
1789 } else {
1790 return false;
1791 }
1792 }
1793 return true;
1794 }
1795
1796 /**
1797 * Process the input stat.
1798 * If it is a file, return the file stat.
1799 * If it is a directory, traverse the directory if recursive is true;
1800 * ignore it if recursive is false.
1801 * If it is a symlink, resolve the symlink first and then process it
1802 * depending on if it is a file or directory.
1803 * @param stat input status
1804 * @throws AccessControlException if access is denied
1805 * @throws FileNotFoundException if file is not found
1806 * @throws UnsupportedFileSystemException if fs is not supported
1807 * @throws IOException for all other IO errors
1808 */
1809 private void handleFileStat(LocatedFileStatus stat)
1810 throws IOException {
1811 if (stat.isFile()) { // file
1812 curFile = stat;
1813 } else if (stat.isSymlink()) { // symbolic link
1814 // resolve symbolic link
1815 FileStatus symstat = FileContext.this.getFileStatus(
1816 stat.getSymlink());
1817 if (symstat.isFile() || (recursive && symstat.isDirectory())) {
1818 itors.push(curItor);
1819 curItor = listLocatedStatus(stat.getPath());
1820 }
1821 } else if (recursive) { // directory
1822 itors.push(curItor);
1823 curItor = listLocatedStatus(stat.getPath());
1824 }
1825 }
1826
1827 /**
1828 * Returns the next file's status with its block locations
1829 *
1830 * @throws AccessControlException if not allowed to access next
1831 * file's status or locations
1832 * @throws FileNotFoundException if next file does not exist any more
1833 * @throws UnsupportedFileSystemException if next file's
1834 * fs is unsupported
1835 * @throws IOException for all other IO errors
1836 * for example, NameNode is not avaialbe or
1837 * NameNode throws IOException due to an error
1838 * while getting the status or block locations
1839 */
1840 @Override
1841 public LocatedFileStatus next() throws IOException {
1842 if (hasNext()) {
1843 LocatedFileStatus result = curFile;
1844 curFile = null;
1845 return result;
1846 }
1847 throw new java.util.NoSuchElementException("No more entry in " + f);
1848 }
1849 };
1850 }
1851
1852 /**
1853 * <p>Return all the files that match filePattern and are not checksum
1854 * files. Results are sorted by their names.
1855 *
1856 * <p>
1857 * A filename pattern is composed of <i>regular</i> characters and
1858 * <i>special pattern matching</i> characters, which are:
1859 *
1860 * <dl>
1861 * <dd>
1862 * <dl>
1863 * <p>
1864 * <dt> <tt> ? </tt>
1865 * <dd> Matches any single character.
1866 *
1867 * <p>
1868 * <dt> <tt> * </tt>
1869 * <dd> Matches zero or more characters.
1870 *
1871 * <p>
1872 * <dt> <tt> [<i>abc</i>] </tt>
1873 * <dd> Matches a single character from character set
1874 * <tt>{<i>a,b,c</i>}</tt>.
1875 *
1876 * <p>
1877 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1878 * <dd> Matches a single character from the character range
1879 * <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be
1880 * lexicographically less than or equal to character <tt><i>b</i></tt>.
1881 *
1882 * <p>
1883 * <dt> <tt> [^<i>a</i>] </tt>
1884 * <dd> Matches a single char that is not from character set or range
1885 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
1886 * immediately to the right of the opening bracket.
1887 *
1888 * <p>
1889 * <dt> <tt> \<i>c</i> </tt>
1890 * <dd> Removes (escapes) any special meaning of character <i>c</i>.
1891 *
1892 * <p>
1893 * <dt> <tt> {ab,cd} </tt>
1894 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1895 *
1896 * <p>
1897 * <dt> <tt> {ab,c{de,fh}} </tt>
1898 * <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt>
1899 *
1900 * </dl>
1901 * </dd>
1902 * </dl>
1903 *
1904 * @param pathPattern a regular expression specifying a pth pattern
1905 *
1906 * @return an array of paths that match the path pattern
1907 *
1908 * @throws AccessControlException If access is denied
1909 * @throws UnsupportedFileSystemException If file system for
1910 * <code>pathPattern</code> is not supported
1911 * @throws IOException If an I/O error occurred
1912 *
1913 * Exceptions applicable to file systems accessed over RPC:
1914 * @throws RpcClientException If an exception occurred in the RPC client
1915 * @throws RpcServerException If an exception occurred in the RPC server
1916 * @throws UnexpectedServerException If server implementation throws
1917 * undeclared exception to RPC server
1918 */
1919 public FileStatus[] globStatus(Path pathPattern)
1920 throws AccessControlException, UnsupportedFileSystemException,
1921 IOException {
1922 return globStatus(pathPattern, DEFAULT_FILTER);
1923 }
1924
1925 /**
1926 * Return an array of FileStatus objects whose path names match pathPattern
1927 * and is accepted by the user-supplied path filter. Results are sorted by
1928 * their path names.
1929 * Return null if pathPattern has no glob and the path does not exist.
1930 * Return an empty array if pathPattern has a glob and no path matches it.
1931 *
1932 * @param pathPattern regular expression specifying the path pattern
1933 * @param filter user-supplied path filter
1934 *
1935 * @return an array of FileStatus objects
1936 *
1937 * @throws AccessControlException If access is denied
1938 * @throws UnsupportedFileSystemException If file system for
1939 * <code>pathPattern</code> is not supported
1940 * @throws IOException If an I/O error occurred
1941 *
1942 * Exceptions applicable to file systems accessed over RPC:
1943 * @throws RpcClientException If an exception occurred in the RPC client
1944 * @throws RpcServerException If an exception occurred in the RPC server
1945 * @throws UnexpectedServerException If server implementation throws
1946 * undeclared exception to RPC server
1947 */
1948 public FileStatus[] globStatus(final Path pathPattern,
1949 final PathFilter filter) throws AccessControlException,
1950 UnsupportedFileSystemException, IOException {
1951 URI uri = getFSofPath(fixRelativePart(pathPattern)).getUri();
1952
1953 String filename = pathPattern.toUri().getPath();
1954
1955 List<String> filePatterns = GlobExpander.expand(filename);
1956 if (filePatterns.size() == 1) {
1957 Path absPathPattern = fixRelativePart(pathPattern);
1958 return globStatusInternal(uri, new Path(absPathPattern.toUri()
1959 .getPath()), filter);
1960 } else {
1961 List<FileStatus> results = new ArrayList<FileStatus>();
1962 for (String iFilePattern : filePatterns) {
1963 Path iAbsFilePattern = fixRelativePart(new Path(iFilePattern));
1964 FileStatus[] files = globStatusInternal(uri, iAbsFilePattern, filter);
1965 for (FileStatus file : files) {
1966 results.add(file);
1967 }
1968 }
1969 return results.toArray(new FileStatus[results.size()]);
1970 }
1971 }
1972
1973 /**
1974 *
1975 * @param uri for all the inPathPattern
1976 * @param inPathPattern - without the scheme & authority (take from uri)
1977 * @param filter
1978 *
1979 * @return an array of FileStatus objects
1980 *
1981 * @throws AccessControlException If access is denied
1982 * @throws IOException If an I/O error occurred
1983 */
1984 private FileStatus[] globStatusInternal(final URI uri,
1985 final Path inPathPattern, final PathFilter filter)
1986 throws AccessControlException, IOException
1987 {
1988 Path[] parents = new Path[1];
1989 int level = 0;
1990
1991 assert(inPathPattern.toUri().getScheme() == null &&
1992 inPathPattern.toUri().getAuthority() == null &&
1993 inPathPattern.isUriPathAbsolute());
1994
1995
1996 String filename = inPathPattern.toUri().getPath();
1997
1998 // path has only zero component
1999 if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
2000 Path p = inPathPattern.makeQualified(uri, null);
2001 return getFileStatus(new Path[]{p});
2002 }
2003
2004 // path has at least one component
2005 String[] components = filename.split(Path.SEPARATOR);
2006
2007 // Path is absolute, first component is "/" hence first component
2008 // is the uri root
2009 parents[0] = new Path(new Path(uri), new Path("/"));
2010 level = 1;
2011
2012 // glob the paths that match the parent path, ie. [0, components.length-1]
2013 boolean[] hasGlob = new boolean[]{false};
2014 Path[] relParentPaths =
2015 globPathsLevel(parents, components, level, hasGlob);
2016 FileStatus[] results;
2017
2018 if (relParentPaths == null || relParentPaths.length == 0) {
2019 results = null;
2020 } else {
2021 // fix the pathes to be abs
2022 Path[] parentPaths = new Path [relParentPaths.length];
2023 for(int i=0; i<relParentPaths.length; i++) {
2024 parentPaths[i] = relParentPaths[i].makeQualified(uri, null);
2025 }
2026
2027 // Now work on the last component of the path
2028 GlobFilter fp =
2029 new GlobFilter(components[components.length - 1], filter);
2030 if (fp.hasPattern()) { // last component has a pattern
2031 // list parent directories and then glob the results
2032 try {
2033 results = listStatus(parentPaths, fp);
2034 } catch (FileNotFoundException e) {
2035 results = null;
2036 }
2037 hasGlob[0] = true;
2038 } else { // last component does not have a pattern
2039 // get all the path names
2040 ArrayList<Path> filteredPaths =
2041 new ArrayList<Path>(parentPaths.length);
2042 for (int i = 0; i < parentPaths.length; i++) {
2043 parentPaths[i] = new Path(parentPaths[i],
2044 components[components.length - 1]);
2045 if (fp.accept(parentPaths[i])) {
2046 filteredPaths.add(parentPaths[i]);
2047 }
2048 }
2049 // get all their statuses
2050 results = getFileStatus(
2051 filteredPaths.toArray(new Path[filteredPaths.size()]));
2052 }
2053 }
2054
2055 // Decide if the pathPattern contains a glob or not
2056 if (results == null) {
2057 if (hasGlob[0]) {
2058 results = new FileStatus[0];
2059 }
2060 } else {
2061 if (results.length == 0) {
2062 if (!hasGlob[0]) {
2063 results = null;
2064 }
2065 } else {
2066 Arrays.sort(results);
2067 }
2068 }
2069 return results;
2070 }
2071
2072 /*
2073 * For a path of N components, return a list of paths that match the
2074 * components [<code>level</code>, <code>N-1</code>].
2075 */
2076 private Path[] globPathsLevel(Path[] parents, String[] filePattern,
2077 int level, boolean[] hasGlob) throws AccessControlException,
2078 FileNotFoundException, IOException {
2079 if (level == filePattern.length - 1) {
2080 return parents;
2081 }
2082 if (parents == null || parents.length == 0) {
2083 return null;
2084 }
2085 GlobFilter fp = new GlobFilter(filePattern[level]);
2086 if (fp.hasPattern()) {
2087 try {
2088 parents = FileUtil.stat2Paths(listStatus(parents, fp));
2089 } catch (FileNotFoundException e) {
2090 parents = null;
2091 }
2092 hasGlob[0] = true;
2093 } else {
2094 for (int i = 0; i < parents.length; i++) {
2095 parents[i] = new Path(parents[i], filePattern[level]);
2096 }
2097 }
2098 return globPathsLevel(parents, filePattern, level + 1, hasGlob);
2099 }
2100
2101 /**
2102 * Copy file from src to dest. See
2103 * {@link #copy(Path, Path, boolean, boolean)}
2104 */
2105 public boolean copy(final Path src, final Path dst)
2106 throws AccessControlException, FileAlreadyExistsException,
2107 FileNotFoundException, ParentNotDirectoryException,
2108 UnsupportedFileSystemException, IOException {
2109 return copy(src, dst, false, false);
2110 }
2111
2112 /**
2113 * Copy from src to dst, optionally deleting src and overwriting dst.
2114 * @param src
2115 * @param dst
2116 * @param deleteSource - delete src if true
2117 * @param overwrite overwrite dst if true; throw IOException if dst exists
2118 * and overwrite is false.
2119 *
2120 * @return true if copy is successful
2121 *
2122 * @throws AccessControlException If access is denied
2123 * @throws FileAlreadyExistsException If <code>dst</code> already exists
2124 * @throws FileNotFoundException If <code>src</code> does not exist
2125 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not
2126 * a directory
2127 * @throws UnsupportedFileSystemException If file system for
2128 * <code>src</code> or <code>dst</code> is not supported
2129 * @throws IOException If an I/O error occurred
2130 *
2131 * Exceptions applicable to file systems accessed over RPC:
2132 * @throws RpcClientException If an exception occurred in the RPC client
2133 * @throws RpcServerException If an exception occurred in the RPC server
2134 * @throws UnexpectedServerException If server implementation throws
2135 * undeclared exception to RPC server
2136 *
2137 * RuntimeExceptions:
2138 * @throws InvalidPathException If path <code>dst</code> is invalid
2139 */
2140 public boolean copy(final Path src, final Path dst, boolean deleteSource,
2141 boolean overwrite) throws AccessControlException,
2142 FileAlreadyExistsException, FileNotFoundException,
2143 ParentNotDirectoryException, UnsupportedFileSystemException,
2144 IOException {
2145 src.checkNotSchemeWithRelative();
2146 dst.checkNotSchemeWithRelative();
2147 Path qSrc = makeQualified(src);
2148 Path qDst = makeQualified(dst);
2149 checkDest(qSrc.getName(), qDst, overwrite);
2150 FileStatus fs = FileContext.this.getFileStatus(qSrc);
2151 if (fs.isDirectory()) {
2152 checkDependencies(qSrc, qDst);
2153 mkdir(qDst, FsPermission.getDirDefault(), true);
2154 FileStatus[] contents = listStatus(qSrc);
2155 for (FileStatus content : contents) {
2156 copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
2157 content.getPath().getName())), deleteSource, overwrite);
2158 }
2159 } else {
2160 InputStream in=null;
2161 OutputStream out = null;
2162 try {
2163 in = open(qSrc);
2164 EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
2165 CreateFlag.CREATE, CreateFlag.OVERWRITE) :
2166 EnumSet.of(CreateFlag.CREATE);
2167 out = create(qDst, createFlag);
2168 IOUtils.copyBytes(in, out, conf, true);
2169 } catch (IOException e) {
2170 IOUtils.closeStream(out);
2171 IOUtils.closeStream(in);
2172 throw e;
2173 }
2174 }
2175 if (deleteSource) {
2176 return delete(qSrc, true);
2177 } else {
2178 return true;
2179 }
2180 }
2181 }
2182
2183 /**
2184 * Check if copying srcName to dst would overwrite an existing
2185 * file or directory.
2186 * @param srcName File or directory to be copied.
2187 * @param dst Destination to copy srcName to.
2188 * @param overwrite Whether it's ok to overwrite an existing file.
2189 * @throws AccessControlException If access is denied.
2190 * @throws IOException If dst is an existing directory, or dst is an
2191 * existing file and the overwrite option is not passed.
2192 */
2193 private void checkDest(String srcName, Path dst, boolean overwrite)
2194 throws AccessControlException, IOException {
2195 try {
2196 FileStatus dstFs = getFileStatus(dst);
2197 if (dstFs.isDirectory()) {
2198 if (null == srcName) {
2199 throw new IOException("Target " + dst + " is a directory");
2200 }
2201 // Recurse to check if dst/srcName exists.
2202 checkDest(null, new Path(dst, srcName), overwrite);
2203 } else if (!overwrite) {
2204 throw new IOException("Target " + new Path(dst, srcName)
2205 + " already exists");
2206 }
2207 } catch (FileNotFoundException e) {
2208 // dst does not exist - OK to copy.
2209 }
2210 }
2211
2212 //
2213 // If the destination is a subdirectory of the source, then
2214 // generate exception
2215 //
2216 private static void checkDependencies(Path qualSrc, Path qualDst)
2217 throws IOException {
2218 if (isSameFS(qualSrc, qualDst)) {
2219 String srcq = qualSrc.toString() + Path.SEPARATOR;
2220 String dstq = qualDst.toString() + Path.SEPARATOR;
2221 if (dstq.startsWith(srcq)) {
2222 if (srcq.length() == dstq.length()) {
2223 throw new IOException("Cannot copy " + qualSrc + " to itself.");
2224 } else {
2225 throw new IOException("Cannot copy " + qualSrc +
2226 " to its subdirectory " + qualDst);
2227 }
2228 }
2229 }
2230 }
2231
2232 /**
2233 * Are qualSrc and qualDst of the same file system?
2234 * @param qualPath1 - fully qualified path
2235 * @param qualPath2 - fully qualified path
2236 * @return
2237 */
2238 private static boolean isSameFS(Path qualPath1, Path qualPath2) {
2239 URI srcUri = qualPath1.toUri();
2240 URI dstUri = qualPath2.toUri();
2241 return (srcUri.getScheme().equals(dstUri.getScheme()) &&
2242 !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri
2243 .getAuthority().equals(dstUri.getAuthority())));
2244 }
2245
2246 /**
2247 * Deletes all the paths in deleteOnExit on JVM shutdown.
2248 */
2249 static class FileContextFinalizer implements Runnable {
2250 @Override
2251 public synchronized void run() {
2252 processDeleteOnExit();
2253 }
2254 }
2255
2256 /**
2257 * Resolves all symbolic links in the specified path.
2258 * Returns the new path object.
2259 */
2260 protected Path resolve(final Path f) throws FileNotFoundException,
2261 UnresolvedLinkException, AccessControlException, IOException {
2262 return new FSLinkResolver<Path>() {
2263 @Override
2264 public Path next(final AbstractFileSystem fs, final Path p)
2265 throws IOException, UnresolvedLinkException {
2266 return fs.resolvePath(p);
2267 }
2268 }.resolve(this, f);
2269 }
2270
2271 /**
2272 * Resolves all symbolic links in the specified path leading up
2273 * to, but not including the final path component.
2274 * @param f path to resolve
2275 * @return the new path object.
2276 */
2277 protected Path resolveIntermediate(final Path f) throws IOException {
2278 return new FSLinkResolver<FileStatus>() {
2279 @Override
2280 public FileStatus next(final AbstractFileSystem fs, final Path p)
2281 throws IOException, UnresolvedLinkException {
2282 return fs.getFileLinkStatus(p);
2283 }
2284 }.resolve(this, f).getPath();
2285 }
2286
2287 /**
2288 * Returns the list of AbstractFileSystems accessed in the path. The list may
2289 * contain more than one AbstractFileSystems objects in case of symlinks.
2290 *
2291 * @param f
2292 * Path which needs to be resolved
2293 * @return List of AbstractFileSystems accessed in the path
2294 * @throws IOException
2295 */
2296 Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f)
2297 throws IOException {
2298 final Path absF = fixRelativePart(f);
2299 final HashSet<AbstractFileSystem> result
2300 = new HashSet<AbstractFileSystem>();
2301 new FSLinkResolver<Void>() {
2302 @Override
2303 public Void next(final AbstractFileSystem fs, final Path p)
2304 throws IOException, UnresolvedLinkException {
2305 result.add(fs);
2306 fs.getFileStatus(p);
2307 return null;
2308 }
2309 }.resolve(this, absF);
2310 return result;
2311 }
2312
2313 /**
2314 * Get the statistics for a particular file system
2315 *
2316 * @param uri
2317 * the uri to lookup the statistics. Only scheme and authority part
2318 * of the uri are used as the key to store and lookup.
2319 * @return a statistics object
2320 */
2321 public static Statistics getStatistics(URI uri) {
2322 return AbstractFileSystem.getStatistics(uri);
2323 }
2324
2325 /**
2326 * Clears all the statistics stored in AbstractFileSystem, for all the file
2327 * systems.
2328 */
2329 public static void clearStatistics() {
2330 AbstractFileSystem.clearStatistics();
2331 }
2332
2333 /**
2334 * Prints the statistics to standard output. File System is identified by the
2335 * scheme and authority.
2336 */
2337 public static void printStatistics() {
2338 AbstractFileSystem.printStatistics();
2339 }
2340
2341 /**
2342 * @return Map of uri and statistics for each filesystem instantiated. The uri
2343 * consists of scheme and authority for the filesystem.
2344 */
2345 public static Map<URI, Statistics> getAllStatistics() {
2346 return AbstractFileSystem.getAllStatistics();
2347 }
2348
2349 /**
2350 * Get delegation tokens for the file systems accessed for a given
2351 * path.
2352 * @param p Path for which delegations tokens are requested.
2353 * @param renewer the account name that is allowed to renew the token.
2354 * @return List of delegation tokens.
2355 * @throws IOException
2356 */
2357 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
2358 public List<Token<?>> getDelegationTokens(
2359 Path p, String renewer) throws IOException {
2360 Set<AbstractFileSystem> afsSet = resolveAbstractFileSystems(p);
2361 List<Token<?>> tokenList =
2362 new ArrayList<Token<?>>();
2363 for (AbstractFileSystem afs : afsSet) {
2364 List<Token<?>> afsTokens = afs.getDelegationTokens(renewer);
2365 tokenList.addAll(afsTokens);
2366 }
2367 return tokenList;
2368 }
2369 }