001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.fs;
019
020 import java.io.Closeable;
021 import java.io.FileNotFoundException;
022 import java.io.IOException;
023 import java.lang.ref.WeakReference;
024 import java.net.URI;
025 import java.net.URISyntaxException;
026 import java.security.PrivilegedExceptionAction;
027 import java.util.ArrayList;
028 import java.util.Arrays;
029 import java.util.Collections;
030 import java.util.EnumSet;
031 import java.util.HashMap;
032 import java.util.HashSet;
033 import java.util.IdentityHashMap;
034 import java.util.Iterator;
035 import java.util.LinkedList;
036 import java.util.List;
037 import java.util.Map;
038 import java.util.NoSuchElementException;
039 import java.util.ServiceLoader;
040 import java.util.Set;
041 import java.util.Stack;
042 import java.util.TreeSet;
043 import java.util.concurrent.atomic.AtomicInteger;
044 import java.util.concurrent.atomic.AtomicLong;
045
046 import org.apache.commons.logging.Log;
047 import org.apache.commons.logging.LogFactory;
048 import org.apache.hadoop.classification.InterfaceAudience;
049 import org.apache.hadoop.classification.InterfaceStability;
050 import org.apache.hadoop.conf.Configuration;
051 import org.apache.hadoop.conf.Configured;
052 import org.apache.hadoop.fs.Options.ChecksumOpt;
053 import org.apache.hadoop.fs.Options.Rename;
054 import org.apache.hadoop.fs.permission.FsPermission;
055 import org.apache.hadoop.io.MultipleIOException;
056 import org.apache.hadoop.io.Text;
057 import org.apache.hadoop.net.NetUtils;
058 import org.apache.hadoop.security.AccessControlException;
059 import org.apache.hadoop.security.Credentials;
060 import org.apache.hadoop.security.SecurityUtil;
061 import org.apache.hadoop.security.UserGroupInformation;
062 import org.apache.hadoop.security.token.Token;
063 import org.apache.hadoop.util.DataChecksum;
064 import org.apache.hadoop.util.Progressable;
065 import org.apache.hadoop.util.ReflectionUtils;
066 import org.apache.hadoop.util.ShutdownHookManager;
067
068 import com.google.common.annotations.VisibleForTesting;
069
070 /****************************************************************
071 * An abstract base class for a fairly generic filesystem. It
072 * may be implemented as a distributed filesystem, or as a "local"
073 * one that reflects the locally-connected disk. The local version
074 * exists for small Hadoop instances and for testing.
075 *
076 * <p>
077 *
078 * All user code that may potentially use the Hadoop Distributed
079 * File System should be written to use a FileSystem object. The
080 * Hadoop DFS is a multi-machine system that appears as a single
081 * disk. It's useful because of its fault tolerance and potentially
082 * very large capacity.
083 *
084 * <p>
085 * The local implementation is {@link LocalFileSystem} and distributed
086 * implementation is DistributedFileSystem.
087 *****************************************************************/
088 @InterfaceAudience.Public
089 @InterfaceStability.Stable
090 public abstract class FileSystem extends Configured implements Closeable {
091 public static final String FS_DEFAULT_NAME_KEY =
092 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
093 public static final String DEFAULT_FS =
094 CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
095
096 public static final Log LOG = LogFactory.getLog(FileSystem.class);
097
098 /**
099 * Priority of the FileSystem shutdown hook.
100 */
101 public static final int SHUTDOWN_HOOK_PRIORITY = 10;
102
103 /** FileSystem cache */
104 static final Cache CACHE = new Cache();
105
106 /** The key this instance is stored under in the cache. */
107 private Cache.Key key;
108
109 /** Recording statistics per a FileSystem class */
110 private static final Map<Class<? extends FileSystem>, Statistics>
111 statisticsTable =
112 new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
113
114 /**
115 * The statistics for this file system.
116 */
117 protected Statistics statistics;
118
119 /**
120 * A cache of files that should be deleted when filsystem is closed
121 * or the JVM is exited.
122 */
123 private Set<Path> deleteOnExit = new TreeSet<Path>();
124
125 boolean resolveSymlinks;
126 /**
127 * This method adds a file system for testing so that we can find it later. It
128 * is only for testing.
129 * @param uri the uri to store it under
130 * @param conf the configuration to store it under
131 * @param fs the file system to store
132 * @throws IOException
133 */
134 static void addFileSystemForTesting(URI uri, Configuration conf,
135 FileSystem fs) throws IOException {
136 CACHE.map.put(new Cache.Key(uri, conf), fs);
137 }
138
139 /**
140 * Get a filesystem instance based on the uri, the passed
141 * configuration and the user
142 * @param uri of the filesystem
143 * @param conf the configuration to use
144 * @param user to perform the get as
145 * @return the filesystem instance
146 * @throws IOException
147 * @throws InterruptedException
148 */
149 public static FileSystem get(final URI uri, final Configuration conf,
150 final String user) throws IOException, InterruptedException {
151 String ticketCachePath =
152 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
153 UserGroupInformation ugi =
154 UserGroupInformation.getBestUGI(ticketCachePath, user);
155 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
156 @Override
157 public FileSystem run() throws IOException {
158 return get(uri, conf);
159 }
160 });
161 }
162
163 /**
164 * Returns the configured filesystem implementation.
165 * @param conf the configuration to use
166 */
167 public static FileSystem get(Configuration conf) throws IOException {
168 return get(getDefaultUri(conf), conf);
169 }
170
171 /** Get the default filesystem URI from a configuration.
172 * @param conf the configuration to use
173 * @return the uri of the default filesystem
174 */
175 public static URI getDefaultUri(Configuration conf) {
176 return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
177 }
178
179 /** Set the default filesystem URI in a configuration.
180 * @param conf the configuration to alter
181 * @param uri the new default filesystem uri
182 */
183 public static void setDefaultUri(Configuration conf, URI uri) {
184 conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
185 }
186
187 /** Set the default filesystem URI in a configuration.
188 * @param conf the configuration to alter
189 * @param uri the new default filesystem uri
190 */
191 public static void setDefaultUri(Configuration conf, String uri) {
192 setDefaultUri(conf, URI.create(fixName(uri)));
193 }
194
195 /** Called after a new FileSystem instance is constructed.
196 * @param name a uri whose authority section names the host, port, etc.
197 * for this FileSystem
198 * @param conf the configuration
199 */
200 public void initialize(URI name, Configuration conf) throws IOException {
201 statistics = getStatistics(name.getScheme(), getClass());
202 resolveSymlinks = conf.getBoolean(
203 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
204 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
205 }
206
207 /**
208 * Return the protocol scheme for the FileSystem.
209 * <p/>
210 * This implementation throws an <code>UnsupportedOperationException</code>.
211 *
212 * @return the protocol scheme for the FileSystem.
213 */
214 public String getScheme() {
215 throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
216 }
217
218 /** Returns a URI whose scheme and authority identify this FileSystem.*/
219 public abstract URI getUri();
220
221 /**
222 * Return a canonicalized form of this FileSystem's URI.
223 *
224 * The default implementation simply calls {@link #canonicalizeUri(URI)}
225 * on the filesystem's own URI, so subclasses typically only need to
226 * implement that method.
227 *
228 * @see #canonicalizeUri(URI)
229 */
230 protected URI getCanonicalUri() {
231 return canonicalizeUri(getUri());
232 }
233
234 /**
235 * Canonicalize the given URI.
236 *
237 * This is filesystem-dependent, but may for example consist of
238 * canonicalizing the hostname using DNS and adding the default
239 * port if not specified.
240 *
241 * The default implementation simply fills in the default port if
242 * not specified and if the filesystem has a default port.
243 *
244 * @return URI
245 * @see NetUtils#getCanonicalUri(URI, int)
246 */
247 protected URI canonicalizeUri(URI uri) {
248 if (uri.getPort() == -1 && getDefaultPort() > 0) {
249 // reconstruct the uri with the default port set
250 try {
251 uri = new URI(uri.getScheme(), uri.getUserInfo(),
252 uri.getHost(), getDefaultPort(),
253 uri.getPath(), uri.getQuery(), uri.getFragment());
254 } catch (URISyntaxException e) {
255 // Should never happen!
256 throw new AssertionError("Valid URI became unparseable: " +
257 uri);
258 }
259 }
260
261 return uri;
262 }
263
264 /**
265 * Get the default port for this file system.
266 * @return the default port or 0 if there isn't one
267 */
268 protected int getDefaultPort() {
269 return 0;
270 }
271
272 protected static FileSystem getFSofPath(final Path absOrFqPath,
273 final Configuration conf)
274 throws UnsupportedFileSystemException, IOException {
275 absOrFqPath.checkNotSchemeWithRelative();
276 absOrFqPath.checkNotRelative();
277
278 // Uses the default file system if not fully qualified
279 return get(absOrFqPath.toUri(), conf);
280 }
281
282 /**
283 * Get a canonical service name for this file system. The token cache is
284 * the only user of the canonical service name, and uses it to lookup this
285 * filesystem's service tokens.
286 * If file system provides a token of its own then it must have a canonical
287 * name, otherwise canonical name can be null.
288 *
289 * Default Impl: If the file system has child file systems
290 * (such as an embedded file system) then it is assumed that the fs has no
291 * tokens of its own and hence returns a null name; otherwise a service
292 * name is built using Uri and port.
293 *
294 * @return a service string that uniquely identifies this file system, null
295 * if the filesystem does not implement tokens
296 * @see SecurityUtil#buildDTServiceName(URI, int)
297 */
298 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
299 public String getCanonicalServiceName() {
300 return (getChildFileSystems() == null)
301 ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort())
302 : null;
303 }
304
305 /** @deprecated call #getUri() instead.*/
306 @Deprecated
307 public String getName() { return getUri().toString(); }
308
309 /** @deprecated call #get(URI,Configuration) instead. */
310 @Deprecated
311 public static FileSystem getNamed(String name, Configuration conf)
312 throws IOException {
313 return get(URI.create(fixName(name)), conf);
314 }
315
316 /** Update old-format filesystem names, for back-compatibility. This should
317 * eventually be replaced with a checkName() method that throws an exception
318 * for old-format names. */
319 private static String fixName(String name) {
320 // convert old-format name to new-format name
321 if (name.equals("local")) { // "local" is now "file:///".
322 LOG.warn("\"local\" is a deprecated filesystem name."
323 +" Use \"file:///\" instead.");
324 name = "file:///";
325 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
326 LOG.warn("\""+name+"\" is a deprecated filesystem name."
327 +" Use \"hdfs://"+name+"/\" instead.");
328 name = "hdfs://"+name;
329 }
330 return name;
331 }
332
333 /**
334 * Get the local file system.
335 * @param conf the configuration to configure the file system with
336 * @return a LocalFileSystem
337 */
338 public static LocalFileSystem getLocal(Configuration conf)
339 throws IOException {
340 return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
341 }
342
343 /** Returns the FileSystem for this URI's scheme and authority. The scheme
344 * of the URI determines a configuration property name,
345 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
346 * The entire URI is passed to the FileSystem instance's initialize method.
347 */
348 public static FileSystem get(URI uri, Configuration conf) throws IOException {
349 String scheme = uri.getScheme();
350 String authority = uri.getAuthority();
351
352 if (scheme == null && authority == null) { // use default FS
353 return get(conf);
354 }
355
356 if (scheme != null && authority == null) { // no authority
357 URI defaultUri = getDefaultUri(conf);
358 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
359 && defaultUri.getAuthority() != null) { // & default has authority
360 return get(defaultUri, conf); // return default
361 }
362 }
363
364 String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
365 if (conf.getBoolean(disableCacheName, false)) {
366 return createFileSystem(uri, conf);
367 }
368
369 return CACHE.get(uri, conf);
370 }
371
372 /**
373 * Returns the FileSystem for this URI's scheme and authority and the
374 * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
375 * @param uri of the filesystem
376 * @param conf the configuration to use
377 * @param user to perform the get as
378 * @return filesystem instance
379 * @throws IOException
380 * @throws InterruptedException
381 */
382 public static FileSystem newInstance(final URI uri, final Configuration conf,
383 final String user) throws IOException, InterruptedException {
384 String ticketCachePath =
385 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
386 UserGroupInformation ugi =
387 UserGroupInformation.getBestUGI(ticketCachePath, user);
388 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
389 @Override
390 public FileSystem run() throws IOException {
391 return newInstance(uri,conf);
392 }
393 });
394 }
395 /** Returns the FileSystem for this URI's scheme and authority. The scheme
396 * of the URI determines a configuration property name,
397 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
398 * The entire URI is passed to the FileSystem instance's initialize method.
399 * This always returns a new FileSystem object.
400 */
401 public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
402 String scheme = uri.getScheme();
403 String authority = uri.getAuthority();
404
405 if (scheme == null) { // no scheme: use default FS
406 return newInstance(conf);
407 }
408
409 if (authority == null) { // no authority
410 URI defaultUri = getDefaultUri(conf);
411 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
412 && defaultUri.getAuthority() != null) { // & default has authority
413 return newInstance(defaultUri, conf); // return default
414 }
415 }
416 return CACHE.getUnique(uri, conf);
417 }
418
419 /** Returns a unique configured filesystem implementation.
420 * This always returns a new FileSystem object.
421 * @param conf the configuration to use
422 */
423 public static FileSystem newInstance(Configuration conf) throws IOException {
424 return newInstance(getDefaultUri(conf), conf);
425 }
426
427 /**
428 * Get a unique local file system object
429 * @param conf the configuration to configure the file system with
430 * @return a LocalFileSystem
431 * This always returns a new FileSystem object.
432 */
433 public static LocalFileSystem newInstanceLocal(Configuration conf)
434 throws IOException {
435 return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
436 }
437
438 /**
439 * Close all cached filesystems. Be sure those filesystems are not
440 * used anymore.
441 *
442 * @throws IOException
443 */
444 public static void closeAll() throws IOException {
445 CACHE.closeAll();
446 }
447
448 /**
449 * Close all cached filesystems for a given UGI. Be sure those filesystems
450 * are not used anymore.
451 * @param ugi user group info to close
452 * @throws IOException
453 */
454 public static void closeAllForUGI(UserGroupInformation ugi)
455 throws IOException {
456 CACHE.closeAll(ugi);
457 }
458
459 /**
460 * Make sure that a path specifies a FileSystem.
461 * @param path to use
462 */
463 public Path makeQualified(Path path) {
464 checkPath(path);
465 return path.makeQualified(this.getUri(), this.getWorkingDirectory());
466 }
467
468 /**
469 * Get a new delegation token for this file system.
470 * This is an internal method that should have been declared protected
471 * but wasn't historically.
472 * Callers should use {@link #addDelegationTokens(String, Credentials)}
473 *
474 * @param renewer the account name that is allowed to renew the token.
475 * @return a new delegation token
476 * @throws IOException
477 */
478 @InterfaceAudience.Private()
479 public Token<?> getDelegationToken(String renewer) throws IOException {
480 return null;
481 }
482
483 /**
484 * Obtain all delegation tokens used by this FileSystem that are not
485 * already present in the given Credentials. Existing tokens will neither
486 * be verified as valid nor having the given renewer. Missing tokens will
487 * be acquired and added to the given Credentials.
488 *
489 * Default Impl: works for simple fs with its own token
490 * and also for an embedded fs whose tokens are those of its
491 * children file system (i.e. the embedded fs has not tokens of its
492 * own).
493 *
494 * @param renewer the user allowed to renew the delegation tokens
495 * @param credentials cache in which to add new delegation tokens
496 * @return list of new delegation tokens
497 * @throws IOException
498 */
499 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
500 public Token<?>[] addDelegationTokens(
501 final String renewer, Credentials credentials) throws IOException {
502 if (credentials == null) {
503 credentials = new Credentials();
504 }
505 final List<Token<?>> tokens = new ArrayList<Token<?>>();
506 collectDelegationTokens(renewer, credentials, tokens);
507 return tokens.toArray(new Token<?>[tokens.size()]);
508 }
509
510 /**
511 * Recursively obtain the tokens for this FileSystem and all descended
512 * FileSystems as determined by getChildFileSystems().
513 * @param renewer the user allowed to renew the delegation tokens
514 * @param credentials cache in which to add the new delegation tokens
515 * @param tokens list in which to add acquired tokens
516 * @throws IOException
517 */
518 private void collectDelegationTokens(final String renewer,
519 final Credentials credentials,
520 final List<Token<?>> tokens)
521 throws IOException {
522 final String serviceName = getCanonicalServiceName();
523 // Collect token of the this filesystem and then of its embedded children
524 if (serviceName != null) { // fs has token, grab it
525 final Text service = new Text(serviceName);
526 Token<?> token = credentials.getToken(service);
527 if (token == null) {
528 token = getDelegationToken(renewer);
529 if (token != null) {
530 tokens.add(token);
531 credentials.addToken(service, token);
532 }
533 }
534 }
535 // Now collect the tokens from the children
536 final FileSystem[] children = getChildFileSystems();
537 if (children != null) {
538 for (final FileSystem fs : children) {
539 fs.collectDelegationTokens(renewer, credentials, tokens);
540 }
541 }
542 }
543
544 /**
545 * Get all the immediate child FileSystems embedded in this FileSystem.
546 * It does not recurse and get grand children. If a FileSystem
547 * has multiple child FileSystems, then it should return a unique list
548 * of those FileSystems. Default is to return null to signify no children.
549 *
550 * @return FileSystems used by this FileSystem
551 */
552 @InterfaceAudience.LimitedPrivate({ "HDFS" })
553 @VisibleForTesting
554 public FileSystem[] getChildFileSystems() {
555 return null;
556 }
557
558 /** create a file with the provided permission
559 * The permission of the file is set to be the provided permission as in
560 * setPermission, not permission&~umask
561 *
562 * It is implemented using two RPCs. It is understood that it is inefficient,
563 * but the implementation is thread-safe. The other option is to change the
564 * value of umask in configuration to be 0, but it is not thread-safe.
565 *
566 * @param fs file system handle
567 * @param file the name of the file to be created
568 * @param permission the permission of the file
569 * @return an output stream
570 * @throws IOException
571 */
572 public static FSDataOutputStream create(FileSystem fs,
573 Path file, FsPermission permission) throws IOException {
574 // create the file with default permission
575 FSDataOutputStream out = fs.create(file);
576 // set its permission to the supplied one
577 fs.setPermission(file, permission);
578 return out;
579 }
580
581 /** create a directory with the provided permission
582 * The permission of the directory is set to be the provided permission as in
583 * setPermission, not permission&~umask
584 *
585 * @see #create(FileSystem, Path, FsPermission)
586 *
587 * @param fs file system handle
588 * @param dir the name of the directory to be created
589 * @param permission the permission of the directory
590 * @return true if the directory creation succeeds; false otherwise
591 * @throws IOException
592 */
593 public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
594 throws IOException {
595 // create the directory using the default permission
596 boolean result = fs.mkdirs(dir);
597 // set its permission to be the supplied one
598 fs.setPermission(dir, permission);
599 return result;
600 }
601
602 ///////////////////////////////////////////////////////////////
603 // FileSystem
604 ///////////////////////////////////////////////////////////////
605
606 protected FileSystem() {
607 super(null);
608 }
609
610 /**
611 * Check that a Path belongs to this FileSystem.
612 * @param path to check
613 */
614 protected void checkPath(Path path) {
615 URI uri = path.toUri();
616 String thatScheme = uri.getScheme();
617 if (thatScheme == null) // fs is relative
618 return;
619 URI thisUri = getCanonicalUri();
620 String thisScheme = thisUri.getScheme();
621 //authority and scheme are not case sensitive
622 if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
623 String thisAuthority = thisUri.getAuthority();
624 String thatAuthority = uri.getAuthority();
625 if (thatAuthority == null && // path's authority is null
626 thisAuthority != null) { // fs has an authority
627 URI defaultUri = getDefaultUri(getConf());
628 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) {
629 uri = defaultUri; // schemes match, so use this uri instead
630 } else {
631 uri = null; // can't determine auth of the path
632 }
633 }
634 if (uri != null) {
635 // canonicalize uri before comparing with this fs
636 uri = canonicalizeUri(uri);
637 thatAuthority = uri.getAuthority();
638 if (thisAuthority == thatAuthority || // authorities match
639 (thisAuthority != null &&
640 thisAuthority.equalsIgnoreCase(thatAuthority)))
641 return;
642 }
643 }
644 throw new IllegalArgumentException("Wrong FS: "+path+
645 ", expected: "+this.getUri());
646 }
647
648 /**
649 * Return an array containing hostnames, offset and size of
650 * portions of the given file. For a nonexistent
651 * file or regions, null will be returned.
652 *
653 * This call is most helpful with DFS, where it returns
654 * hostnames of machines that contain the given file.
655 *
656 * The FileSystem will simply return an elt containing 'localhost'.
657 *
658 * @param file FilesStatus to get data from
659 * @param start offset into the given file
660 * @param len length for which to get locations for
661 */
662 public BlockLocation[] getFileBlockLocations(FileStatus file,
663 long start, long len) throws IOException {
664 if (file == null) {
665 return null;
666 }
667
668 if (start < 0 || len < 0) {
669 throw new IllegalArgumentException("Invalid start or len parameter");
670 }
671
672 if (file.getLen() <= start) {
673 return new BlockLocation[0];
674
675 }
676 String[] name = { "localhost:50010" };
677 String[] host = { "localhost" };
678 return new BlockLocation[] {
679 new BlockLocation(name, host, 0, file.getLen()) };
680 }
681
682
683 /**
684 * Return an array containing hostnames, offset and size of
685 * portions of the given file. For a nonexistent
686 * file or regions, null will be returned.
687 *
688 * This call is most helpful with DFS, where it returns
689 * hostnames of machines that contain the given file.
690 *
691 * The FileSystem will simply return an elt containing 'localhost'.
692 *
693 * @param p path is used to identify an FS since an FS could have
694 * another FS that it could be delegating the call to
695 * @param start offset into the given file
696 * @param len length for which to get locations for
697 */
698 public BlockLocation[] getFileBlockLocations(Path p,
699 long start, long len) throws IOException {
700 if (p == null) {
701 throw new NullPointerException();
702 }
703 FileStatus file = getFileStatus(p);
704 return getFileBlockLocations(file, start, len);
705 }
706
707 /**
708 * Return a set of server default configuration values
709 * @return server default configuration values
710 * @throws IOException
711 * @deprecated use {@link #getServerDefaults(Path)} instead
712 */
713 @Deprecated
714 public FsServerDefaults getServerDefaults() throws IOException {
715 Configuration conf = getConf();
716 // CRC32 is chosen as default as it is available in all
717 // releases that support checksum.
718 // The client trash configuration is ignored.
719 return new FsServerDefaults(getDefaultBlockSize(),
720 conf.getInt("io.bytes.per.checksum", 512),
721 64 * 1024,
722 getDefaultReplication(),
723 conf.getInt("io.file.buffer.size", 4096),
724 false,
725 CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT,
726 DataChecksum.Type.CRC32);
727 }
728
729 /**
730 * Return a set of server default configuration values
731 * @param p path is used to identify an FS since an FS could have
732 * another FS that it could be delegating the call to
733 * @return server default configuration values
734 * @throws IOException
735 */
736 public FsServerDefaults getServerDefaults(Path p) throws IOException {
737 return getServerDefaults();
738 }
739
740 /**
741 * Return the fully-qualified path of path f resolving the path
742 * through any symlinks or mount point
743 * @param p path to be resolved
744 * @return fully qualified path
745 * @throws FileNotFoundException
746 */
747 public Path resolvePath(final Path p) throws IOException {
748 checkPath(p);
749 return getFileStatus(p).getPath();
750 }
751
752 /**
753 * Opens an FSDataInputStream at the indicated Path.
754 * @param f the file name to open
755 * @param bufferSize the size of the buffer to be used.
756 */
757 public abstract FSDataInputStream open(Path f, int bufferSize)
758 throws IOException;
759
760 /**
761 * Opens an FSDataInputStream at the indicated Path.
762 * @param f the file to open
763 */
764 public FSDataInputStream open(Path f) throws IOException {
765 return open(f, getConf().getInt("io.file.buffer.size", 4096));
766 }
767
768 /**
769 * Create an FSDataOutputStream at the indicated Path.
770 * Files are overwritten by default.
771 * @param f the file to create
772 */
773 public FSDataOutputStream create(Path f) throws IOException {
774 return create(f, true);
775 }
776
777 /**
778 * Create an FSDataOutputStream at the indicated Path.
779 * @param f the file to create
780 * @param overwrite if a file with this name already exists, then if true,
781 * the file will be overwritten, and if false an exception will be thrown.
782 */
783 public FSDataOutputStream create(Path f, boolean overwrite)
784 throws IOException {
785 return create(f, overwrite,
786 getConf().getInt("io.file.buffer.size", 4096),
787 getDefaultReplication(f),
788 getDefaultBlockSize(f));
789 }
790
791 /**
792 * Create an FSDataOutputStream at the indicated Path with write-progress
793 * reporting.
794 * Files are overwritten by default.
795 * @param f the file to create
796 * @param progress to report progress
797 */
798 public FSDataOutputStream create(Path f, Progressable progress)
799 throws IOException {
800 return create(f, true,
801 getConf().getInt("io.file.buffer.size", 4096),
802 getDefaultReplication(f),
803 getDefaultBlockSize(f), progress);
804 }
805
806 /**
807 * Create an FSDataOutputStream at the indicated Path.
808 * Files are overwritten by default.
809 * @param f the file to create
810 * @param replication the replication factor
811 */
812 public FSDataOutputStream create(Path f, short replication)
813 throws IOException {
814 return create(f, true,
815 getConf().getInt("io.file.buffer.size", 4096),
816 replication,
817 getDefaultBlockSize(f));
818 }
819
820 /**
821 * Create an FSDataOutputStream at the indicated Path with write-progress
822 * reporting.
823 * Files are overwritten by default.
824 * @param f the file to create
825 * @param replication the replication factor
826 * @param progress to report progress
827 */
828 public FSDataOutputStream create(Path f, short replication,
829 Progressable progress) throws IOException {
830 return create(f, true,
831 getConf().getInt(
832 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
833 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT),
834 replication,
835 getDefaultBlockSize(f), progress);
836 }
837
838
839 /**
840 * Create an FSDataOutputStream at the indicated Path.
841 * @param f the file name to create
842 * @param overwrite if a file with this name already exists, then if true,
843 * the file will be overwritten, and if false an error will be thrown.
844 * @param bufferSize the size of the buffer to be used.
845 */
846 public FSDataOutputStream create(Path f,
847 boolean overwrite,
848 int bufferSize
849 ) throws IOException {
850 return create(f, overwrite, bufferSize,
851 getDefaultReplication(f),
852 getDefaultBlockSize(f));
853 }
854
855 /**
856 * Create an FSDataOutputStream at the indicated Path with write-progress
857 * reporting.
858 * @param f the path of the file to open
859 * @param overwrite if a file with this name already exists, then if true,
860 * the file will be overwritten, and if false an error will be thrown.
861 * @param bufferSize the size of the buffer to be used.
862 */
863 public FSDataOutputStream create(Path f,
864 boolean overwrite,
865 int bufferSize,
866 Progressable progress
867 ) throws IOException {
868 return create(f, overwrite, bufferSize,
869 getDefaultReplication(f),
870 getDefaultBlockSize(f), progress);
871 }
872
873
874 /**
875 * Create an FSDataOutputStream at the indicated Path.
876 * @param f the file name to open
877 * @param overwrite if a file with this name already exists, then if true,
878 * the file will be overwritten, and if false an error will be thrown.
879 * @param bufferSize the size of the buffer to be used.
880 * @param replication required block replication for the file.
881 */
882 public FSDataOutputStream create(Path f,
883 boolean overwrite,
884 int bufferSize,
885 short replication,
886 long blockSize
887 ) throws IOException {
888 return create(f, overwrite, bufferSize, replication, blockSize, null);
889 }
890
891 /**
892 * Create an FSDataOutputStream at the indicated Path with write-progress
893 * reporting.
894 * @param f the file name to open
895 * @param overwrite if a file with this name already exists, then if true,
896 * the file will be overwritten, and if false an error will be thrown.
897 * @param bufferSize the size of the buffer to be used.
898 * @param replication required block replication for the file.
899 */
900 public FSDataOutputStream create(Path f,
901 boolean overwrite,
902 int bufferSize,
903 short replication,
904 long blockSize,
905 Progressable progress
906 ) throws IOException {
907 return this.create(f, FsPermission.getFileDefault().applyUMask(
908 FsPermission.getUMask(getConf())), overwrite, bufferSize,
909 replication, blockSize, progress);
910 }
911
912 /**
913 * Create an FSDataOutputStream at the indicated Path with write-progress
914 * reporting.
915 * @param f the file name to open
916 * @param permission
917 * @param overwrite if a file with this name already exists, then if true,
918 * the file will be overwritten, and if false an error will be thrown.
919 * @param bufferSize the size of the buffer to be used.
920 * @param replication required block replication for the file.
921 * @param blockSize
922 * @param progress
923 * @throws IOException
924 * @see #setPermission(Path, FsPermission)
925 */
926 public abstract FSDataOutputStream create(Path f,
927 FsPermission permission,
928 boolean overwrite,
929 int bufferSize,
930 short replication,
931 long blockSize,
932 Progressable progress) throws IOException;
933
934 /**
935 * Create an FSDataOutputStream at the indicated Path with write-progress
936 * reporting.
937 * @param f the file name to open
938 * @param permission
939 * @param flags {@link CreateFlag}s to use for this stream.
940 * @param bufferSize the size of the buffer to be used.
941 * @param replication required block replication for the file.
942 * @param blockSize
943 * @param progress
944 * @throws IOException
945 * @see #setPermission(Path, FsPermission)
946 */
947 public FSDataOutputStream create(Path f,
948 FsPermission permission,
949 EnumSet<CreateFlag> flags,
950 int bufferSize,
951 short replication,
952 long blockSize,
953 Progressable progress) throws IOException {
954 return create(f, permission, flags, bufferSize, replication,
955 blockSize, progress, null);
956 }
957
958 /**
959 * Create an FSDataOutputStream at the indicated Path with a custom
960 * checksum option
961 * @param f the file name to open
962 * @param permission
963 * @param flags {@link CreateFlag}s to use for this stream.
964 * @param bufferSize the size of the buffer to be used.
965 * @param replication required block replication for the file.
966 * @param blockSize
967 * @param progress
968 * @param checksumOpt checksum parameter. If null, the values
969 * found in conf will be used.
970 * @throws IOException
971 * @see #setPermission(Path, FsPermission)
972 */
973 public FSDataOutputStream create(Path f,
974 FsPermission permission,
975 EnumSet<CreateFlag> flags,
976 int bufferSize,
977 short replication,
978 long blockSize,
979 Progressable progress,
980 ChecksumOpt checksumOpt) throws IOException {
981 // Checksum options are ignored by default. The file systems that
982 // implement checksum need to override this method. The full
983 // support is currently only available in DFS.
984 return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
985 bufferSize, replication, blockSize, progress);
986 }
987
988 /*.
989 * This create has been added to support the FileContext that processes
990 * the permission
991 * with umask before calling this method.
992 * This a temporary method added to support the transition from FileSystem
993 * to FileContext for user applications.
994 */
995 @Deprecated
996 protected FSDataOutputStream primitiveCreate(Path f,
997 FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
998 short replication, long blockSize, Progressable progress,
999 ChecksumOpt checksumOpt) throws IOException {
1000
1001 boolean pathExists = exists(f);
1002 CreateFlag.validate(f, pathExists, flag);
1003
1004 // Default impl assumes that permissions do not matter and
1005 // nor does the bytesPerChecksum hence
1006 // calling the regular create is good enough.
1007 // FSs that implement permissions should override this.
1008
1009 if (pathExists && flag.contains(CreateFlag.APPEND)) {
1010 return append(f, bufferSize, progress);
1011 }
1012
1013 return this.create(f, absolutePermission,
1014 flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
1015 blockSize, progress);
1016 }
1017
1018 /**
1019 * This version of the mkdirs method assumes that the permission is absolute.
1020 * It has been added to support the FileContext that processes the permission
1021 * with umask before calling this method.
1022 * This a temporary method added to support the transition from FileSystem
1023 * to FileContext for user applications.
1024 */
1025 @Deprecated
1026 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
1027 throws IOException {
1028 // Default impl is to assume that permissions do not matter and hence
1029 // calling the regular mkdirs is good enough.
1030 // FSs that implement permissions should override this.
1031 return this.mkdirs(f, absolutePermission);
1032 }
1033
1034
1035 /**
1036 * This version of the mkdirs method assumes that the permission is absolute.
1037 * It has been added to support the FileContext that processes the permission
1038 * with umask before calling this method.
1039 * This a temporary method added to support the transition from FileSystem
1040 * to FileContext for user applications.
1041 */
1042 @Deprecated
1043 protected void primitiveMkdir(Path f, FsPermission absolutePermission,
1044 boolean createParent)
1045 throws IOException {
1046
1047 if (!createParent) { // parent must exist.
1048 // since the this.mkdirs makes parent dirs automatically
1049 // we must throw exception if parent does not exist.
1050 final FileStatus stat = getFileStatus(f.getParent());
1051 if (stat == null) {
1052 throw new FileNotFoundException("Missing parent:" + f);
1053 }
1054 if (!stat.isDirectory()) {
1055 throw new ParentNotDirectoryException("parent is not a dir");
1056 }
1057 // parent does exist - go ahead with mkdir of leaf
1058 }
1059 // Default impl is to assume that permissions do not matter and hence
1060 // calling the regular mkdirs is good enough.
1061 // FSs that implement permissions should override this.
1062 if (!this.mkdirs(f, absolutePermission)) {
1063 throw new IOException("mkdir of "+ f + " failed");
1064 }
1065 }
1066
1067 /**
1068 * Opens an FSDataOutputStream at the indicated Path with write-progress
1069 * reporting. Same as create(), except fails if parent directory doesn't
1070 * already exist.
1071 * @param f the file name to open
1072 * @param overwrite if a file with this name already exists, then if true,
1073 * the file will be overwritten, and if false an error will be thrown.
1074 * @param bufferSize the size of the buffer to be used.
1075 * @param replication required block replication for the file.
1076 * @param blockSize
1077 * @param progress
1078 * @throws IOException
1079 * @see #setPermission(Path, FsPermission)
1080 * @deprecated API only for 0.20-append
1081 */
1082 @Deprecated
1083 public FSDataOutputStream createNonRecursive(Path f,
1084 boolean overwrite,
1085 int bufferSize, short replication, long blockSize,
1086 Progressable progress) throws IOException {
1087 return this.createNonRecursive(f, FsPermission.getFileDefault(),
1088 overwrite, bufferSize, replication, blockSize, progress);
1089 }
1090
1091 /**
1092 * Opens an FSDataOutputStream at the indicated Path with write-progress
1093 * reporting. Same as create(), except fails if parent directory doesn't
1094 * already exist.
1095 * @param f the file name to open
1096 * @param permission
1097 * @param overwrite if a file with this name already exists, then if true,
1098 * the file will be overwritten, and if false an error will be thrown.
1099 * @param bufferSize the size of the buffer to be used.
1100 * @param replication required block replication for the file.
1101 * @param blockSize
1102 * @param progress
1103 * @throws IOException
1104 * @see #setPermission(Path, FsPermission)
1105 * @deprecated API only for 0.20-append
1106 */
1107 @Deprecated
1108 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1109 boolean overwrite, int bufferSize, short replication, long blockSize,
1110 Progressable progress) throws IOException {
1111 return createNonRecursive(f, permission,
1112 overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1113 : EnumSet.of(CreateFlag.CREATE), bufferSize,
1114 replication, blockSize, progress);
1115 }
1116
1117 /**
1118 * Opens an FSDataOutputStream at the indicated Path with write-progress
1119 * reporting. Same as create(), except fails if parent directory doesn't
1120 * already exist.
1121 * @param f the file name to open
1122 * @param permission
1123 * @param flags {@link CreateFlag}s to use for this stream.
1124 * @param bufferSize the size of the buffer to be used.
1125 * @param replication required block replication for the file.
1126 * @param blockSize
1127 * @param progress
1128 * @throws IOException
1129 * @see #setPermission(Path, FsPermission)
1130 * @deprecated API only for 0.20-append
1131 */
1132 @Deprecated
1133 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1134 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
1135 Progressable progress) throws IOException {
1136 throw new IOException("createNonRecursive unsupported for this filesystem "
1137 + this.getClass());
1138 }
1139
1140 /**
1141 * Creates the given Path as a brand-new zero-length file. If
1142 * create fails, or if it already existed, return false.
1143 *
1144 * @param f path to use for create
1145 */
1146 public boolean createNewFile(Path f) throws IOException {
1147 if (exists(f)) {
1148 return false;
1149 } else {
1150 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
1151 return true;
1152 }
1153 }
1154
1155 /**
1156 * Append to an existing file (optional operation).
1157 * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
1158 * @param f the existing file to be appended.
1159 * @throws IOException
1160 */
1161 public FSDataOutputStream append(Path f) throws IOException {
1162 return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
1163 }
1164 /**
1165 * Append to an existing file (optional operation).
1166 * Same as append(f, bufferSize, null).
1167 * @param f the existing file to be appended.
1168 * @param bufferSize the size of the buffer to be used.
1169 * @throws IOException
1170 */
1171 public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
1172 return append(f, bufferSize, null);
1173 }
1174
1175 /**
1176 * Append to an existing file (optional operation).
1177 * @param f the existing file to be appended.
1178 * @param bufferSize the size of the buffer to be used.
1179 * @param progress for reporting progress if it is not null.
1180 * @throws IOException
1181 */
1182 public abstract FSDataOutputStream append(Path f, int bufferSize,
1183 Progressable progress) throws IOException;
1184
1185 /**
1186 * Concat existing files together.
1187 * @param trg the path to the target destination.
1188 * @param psrcs the paths to the sources to use for the concatenation.
1189 * @throws IOException
1190 */
1191 public void concat(final Path trg, final Path [] psrcs) throws IOException {
1192 throw new UnsupportedOperationException("Not implemented by the " +
1193 getClass().getSimpleName() + " FileSystem implementation");
1194 }
1195
1196 /**
1197 * Get replication.
1198 *
1199 * @deprecated Use getFileStatus() instead
1200 * @param src file name
1201 * @return file replication
1202 * @throws IOException
1203 */
1204 @Deprecated
1205 public short getReplication(Path src) throws IOException {
1206 return getFileStatus(src).getReplication();
1207 }
1208
1209 /**
1210 * Set replication for an existing file.
1211 *
1212 * @param src file name
1213 * @param replication new replication
1214 * @throws IOException
1215 * @return true if successful;
1216 * false if file does not exist or is a directory
1217 */
1218 public boolean setReplication(Path src, short replication)
1219 throws IOException {
1220 return true;
1221 }
1222
1223 /**
1224 * Renames Path src to Path dst. Can take place on local fs
1225 * or remote DFS.
1226 * @param src path to be renamed
1227 * @param dst new path after rename
1228 * @throws IOException on failure
1229 * @return true if rename is successful
1230 */
1231 public abstract boolean rename(Path src, Path dst) throws IOException;
1232
1233 /**
1234 * Renames Path src to Path dst
1235 * <ul>
1236 * <li
1237 * <li>Fails if src is a file and dst is a directory.
1238 * <li>Fails if src is a directory and dst is a file.
1239 * <li>Fails if the parent of dst does not exist or is a file.
1240 * </ul>
1241 * <p>
1242 * If OVERWRITE option is not passed as an argument, rename fails
1243 * if the dst already exists.
1244 * <p>
1245 * If OVERWRITE option is passed as an argument, rename overwrites
1246 * the dst if it is a file or an empty directory. Rename fails if dst is
1247 * a non-empty directory.
1248 * <p>
1249 * Note that atomicity of rename is dependent on the file system
1250 * implementation. Please refer to the file system documentation for
1251 * details. This default implementation is non atomic.
1252 * <p>
1253 * This method is deprecated since it is a temporary method added to
1254 * support the transition from FileSystem to FileContext for user
1255 * applications.
1256 *
1257 * @param src path to be renamed
1258 * @param dst new path after rename
1259 * @throws IOException on failure
1260 */
1261 @Deprecated
1262 protected void rename(final Path src, final Path dst,
1263 final Rename... options) throws IOException {
1264 // Default implementation
1265 final FileStatus srcStatus = getFileLinkStatus(src);
1266 if (srcStatus == null) {
1267 throw new FileNotFoundException("rename source " + src + " not found.");
1268 }
1269
1270 boolean overwrite = false;
1271 if (null != options) {
1272 for (Rename option : options) {
1273 if (option == Rename.OVERWRITE) {
1274 overwrite = true;
1275 }
1276 }
1277 }
1278
1279 FileStatus dstStatus;
1280 try {
1281 dstStatus = getFileLinkStatus(dst);
1282 } catch (IOException e) {
1283 dstStatus = null;
1284 }
1285 if (dstStatus != null) {
1286 if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
1287 throw new IOException("Source " + src + " Destination " + dst
1288 + " both should be either file or directory");
1289 }
1290 if (!overwrite) {
1291 throw new FileAlreadyExistsException("rename destination " + dst
1292 + " already exists.");
1293 }
1294 // Delete the destination that is a file or an empty directory
1295 if (dstStatus.isDirectory()) {
1296 FileStatus[] list = listStatus(dst);
1297 if (list != null && list.length != 0) {
1298 throw new IOException(
1299 "rename cannot overwrite non empty destination directory " + dst);
1300 }
1301 }
1302 delete(dst, false);
1303 } else {
1304 final Path parent = dst.getParent();
1305 final FileStatus parentStatus = getFileStatus(parent);
1306 if (parentStatus == null) {
1307 throw new FileNotFoundException("rename destination parent " + parent
1308 + " not found.");
1309 }
1310 if (!parentStatus.isDirectory()) {
1311 throw new ParentNotDirectoryException("rename destination parent " + parent
1312 + " is a file.");
1313 }
1314 }
1315 if (!rename(src, dst)) {
1316 throw new IOException("rename from " + src + " to " + dst + " failed.");
1317 }
1318 }
1319
1320 /**
1321 * Delete a file
1322 * @deprecated Use {@link #delete(Path, boolean)} instead.
1323 */
1324 @Deprecated
1325 public boolean delete(Path f) throws IOException {
1326 return delete(f, true);
1327 }
1328
1329 /** Delete a file.
1330 *
1331 * @param f the path to delete.
1332 * @param recursive if path is a directory and set to
1333 * true, the directory is deleted else throws an exception. In
1334 * case of a file the recursive can be set to either true or false.
1335 * @return true if delete is successful else false.
1336 * @throws IOException
1337 */
1338 public abstract boolean delete(Path f, boolean recursive) throws IOException;
1339
1340 /**
1341 * Mark a path to be deleted when FileSystem is closed.
1342 * When the JVM shuts down,
1343 * all FileSystem objects will be closed automatically.
1344 * Then,
1345 * the marked path will be deleted as a result of closing the FileSystem.
1346 *
1347 * The path has to exist in the file system.
1348 *
1349 * @param f the path to delete.
1350 * @return true if deleteOnExit is successful, otherwise false.
1351 * @throws IOException
1352 */
1353 public boolean deleteOnExit(Path f) throws IOException {
1354 if (!exists(f)) {
1355 return false;
1356 }
1357 synchronized (deleteOnExit) {
1358 deleteOnExit.add(f);
1359 }
1360 return true;
1361 }
1362
1363 /**
1364 * Cancel the deletion of the path when the FileSystem is closed
1365 * @param f the path to cancel deletion
1366 */
1367 public boolean cancelDeleteOnExit(Path f) {
1368 synchronized (deleteOnExit) {
1369 return deleteOnExit.remove(f);
1370 }
1371 }
1372
1373 /**
1374 * Delete all files that were marked as delete-on-exit. This recursively
1375 * deletes all files in the specified paths.
1376 */
1377 protected void processDeleteOnExit() {
1378 synchronized (deleteOnExit) {
1379 for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
1380 Path path = iter.next();
1381 try {
1382 if (exists(path)) {
1383 delete(path, true);
1384 }
1385 }
1386 catch (IOException e) {
1387 LOG.info("Ignoring failure to deleteOnExit for path " + path);
1388 }
1389 iter.remove();
1390 }
1391 }
1392 }
1393
1394 /** Check if exists.
1395 * @param f source file
1396 */
1397 public boolean exists(Path f) throws IOException {
1398 try {
1399 return getFileStatus(f) != null;
1400 } catch (FileNotFoundException e) {
1401 return false;
1402 }
1403 }
1404
1405 /** True iff the named path is a directory.
1406 * Note: Avoid using this method. Instead reuse the FileStatus
1407 * returned by getFileStatus() or listStatus() methods.
1408 * @param f path to check
1409 */
1410 public boolean isDirectory(Path f) throws IOException {
1411 try {
1412 return getFileStatus(f).isDirectory();
1413 } catch (FileNotFoundException e) {
1414 return false; // f does not exist
1415 }
1416 }
1417
1418 /** True iff the named path is a regular file.
1419 * Note: Avoid using this method. Instead reuse the FileStatus
1420 * returned by getFileStatus() or listStatus() methods.
1421 * @param f path to check
1422 */
1423 public boolean isFile(Path f) throws IOException {
1424 try {
1425 return getFileStatus(f).isFile();
1426 } catch (FileNotFoundException e) {
1427 return false; // f does not exist
1428 }
1429 }
1430
1431 /** The number of bytes in a file. */
1432 /** @deprecated Use getFileStatus() instead */
1433 @Deprecated
1434 public long getLength(Path f) throws IOException {
1435 return getFileStatus(f).getLen();
1436 }
1437
1438 /** Return the {@link ContentSummary} of a given {@link Path}.
1439 * @param f path to use
1440 */
1441 public ContentSummary getContentSummary(Path f) throws IOException {
1442 FileStatus status = getFileStatus(f);
1443 if (status.isFile()) {
1444 // f is a file
1445 return new ContentSummary(status.getLen(), 1, 0);
1446 }
1447 // f is a directory
1448 long[] summary = {0, 0, 1};
1449 for(FileStatus s : listStatus(f)) {
1450 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1451 new ContentSummary(s.getLen(), 1, 0);
1452 summary[0] += c.getLength();
1453 summary[1] += c.getFileCount();
1454 summary[2] += c.getDirectoryCount();
1455 }
1456 return new ContentSummary(summary[0], summary[1], summary[2]);
1457 }
1458
1459 final private static PathFilter DEFAULT_FILTER = new PathFilter() {
1460 @Override
1461 public boolean accept(Path file) {
1462 return true;
1463 }
1464 };
1465
1466 /**
1467 * List the statuses of the files/directories in the given path if the path is
1468 * a directory.
1469 *
1470 * @param f given path
1471 * @return the statuses of the files/directories in the given patch
1472 * @throws FileNotFoundException when the path does not exist;
1473 * IOException see specific implementation
1474 */
1475 public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException,
1476 IOException;
1477
1478 /*
1479 * Filter files/directories in the given path using the user-supplied path
1480 * filter. Results are added to the given array <code>results</code>.
1481 */
1482 private void listStatus(ArrayList<FileStatus> results, Path f,
1483 PathFilter filter) throws FileNotFoundException, IOException {
1484 FileStatus listing[] = listStatus(f);
1485 if (listing == null) {
1486 throw new IOException("Error accessing " + f);
1487 }
1488
1489 for (int i = 0; i < listing.length; i++) {
1490 if (filter.accept(listing[i].getPath())) {
1491 results.add(listing[i]);
1492 }
1493 }
1494 }
1495
1496 /**
1497 * @return an iterator over the corrupt files under the given path
1498 * (may contain duplicates if a file has more than one corrupt block)
1499 * @throws IOException
1500 */
1501 public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1502 throws IOException {
1503 throw new UnsupportedOperationException(getClass().getCanonicalName() +
1504 " does not support" +
1505 " listCorruptFileBlocks");
1506 }
1507
1508 /**
1509 * Filter files/directories in the given path using the user-supplied path
1510 * filter.
1511 *
1512 * @param f
1513 * a path name
1514 * @param filter
1515 * the user-supplied path filter
1516 * @return an array of FileStatus objects for the files under the given path
1517 * after applying the filter
1518 * @throws FileNotFoundException when the path does not exist;
1519 * IOException see specific implementation
1520 */
1521 public FileStatus[] listStatus(Path f, PathFilter filter)
1522 throws FileNotFoundException, IOException {
1523 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1524 listStatus(results, f, filter);
1525 return results.toArray(new FileStatus[results.size()]);
1526 }
1527
1528 /**
1529 * Filter files/directories in the given list of paths using default
1530 * path filter.
1531 *
1532 * @param files
1533 * a list of paths
1534 * @return a list of statuses for the files under the given paths after
1535 * applying the filter default Path filter
1536 * @throws FileNotFoundException when the path does not exist;
1537 * IOException see specific implementation
1538 */
1539 public FileStatus[] listStatus(Path[] files)
1540 throws FileNotFoundException, IOException {
1541 return listStatus(files, DEFAULT_FILTER);
1542 }
1543
1544 /**
1545 * Filter files/directories in the given list of paths using user-supplied
1546 * path filter.
1547 *
1548 * @param files
1549 * a list of paths
1550 * @param filter
1551 * the user-supplied path filter
1552 * @return a list of statuses for the files under the given paths after
1553 * applying the filter
1554 * @throws FileNotFoundException when the path does not exist;
1555 * IOException see specific implementation
1556 */
1557 public FileStatus[] listStatus(Path[] files, PathFilter filter)
1558 throws FileNotFoundException, IOException {
1559 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1560 for (int i = 0; i < files.length; i++) {
1561 listStatus(results, files[i], filter);
1562 }
1563 return results.toArray(new FileStatus[results.size()]);
1564 }
1565
1566 /**
1567 * <p>Return all the files that match filePattern and are not checksum
1568 * files. Results are sorted by their names.
1569 *
1570 * <p>
1571 * A filename pattern is composed of <i>regular</i> characters and
1572 * <i>special pattern matching</i> characters, which are:
1573 *
1574 * <dl>
1575 * <dd>
1576 * <dl>
1577 * <p>
1578 * <dt> <tt> ? </tt>
1579 * <dd> Matches any single character.
1580 *
1581 * <p>
1582 * <dt> <tt> * </tt>
1583 * <dd> Matches zero or more characters.
1584 *
1585 * <p>
1586 * <dt> <tt> [<i>abc</i>] </tt>
1587 * <dd> Matches a single character from character set
1588 * <tt>{<i>a,b,c</i>}</tt>.
1589 *
1590 * <p>
1591 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1592 * <dd> Matches a single character from the character range
1593 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be
1594 * lexicographically less than or equal to character <tt><i>b</i></tt>.
1595 *
1596 * <p>
1597 * <dt> <tt> [^<i>a</i>] </tt>
1598 * <dd> Matches a single character that is not from character set or range
1599 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
1600 * immediately to the right of the opening bracket.
1601 *
1602 * <p>
1603 * <dt> <tt> \<i>c</i> </tt>
1604 * <dd> Removes (escapes) any special meaning of character <i>c</i>.
1605 *
1606 * <p>
1607 * <dt> <tt> {ab,cd} </tt>
1608 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1609 *
1610 * <p>
1611 * <dt> <tt> {ab,c{de,fh}} </tt>
1612 * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
1613 *
1614 * </dl>
1615 * </dd>
1616 * </dl>
1617 *
1618 * @param pathPattern a regular expression specifying a pth pattern
1619
1620 * @return an array of paths that match the path pattern
1621 * @throws IOException
1622 */
1623 public FileStatus[] globStatus(Path pathPattern) throws IOException {
1624 return new Globber(this, pathPattern, DEFAULT_FILTER).glob();
1625 }
1626
1627 /**
1628 * Return an array of FileStatus objects whose path names match pathPattern
1629 * and is accepted by the user-supplied path filter. Results are sorted by
1630 * their path names.
1631 * Return null if pathPattern has no glob and the path does not exist.
1632 * Return an empty array if pathPattern has a glob and no path matches it.
1633 *
1634 * @param pathPattern
1635 * a regular expression specifying the path pattern
1636 * @param filter
1637 * a user-supplied path filter
1638 * @return an array of FileStatus objects
1639 * @throws IOException if any I/O error occurs when fetching file status
1640 */
1641 public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
1642 throws IOException {
1643 return new Globber(this, pathPattern, filter).glob();
1644 }
1645
1646 /**
1647 * List the statuses of the files/directories in the given path if the path is
1648 * a directory.
1649 * Return the file's status and block locations If the path is a file.
1650 *
1651 * If a returned status is a file, it contains the file's block locations.
1652 *
1653 * @param f is the path
1654 *
1655 * @return an iterator that traverses statuses of the files/directories
1656 * in the given path
1657 *
1658 * @throws FileNotFoundException If <code>f</code> does not exist
1659 * @throws IOException If an I/O error occurred
1660 */
1661 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
1662 throws FileNotFoundException, IOException {
1663 return listLocatedStatus(f, DEFAULT_FILTER);
1664 }
1665
1666 /**
1667 * Listing a directory
1668 * The returned results include its block location if it is a file
1669 * The results are filtered by the given path filter
1670 * @param f a path
1671 * @param filter a path filter
1672 * @return an iterator that traverses statuses of the files/directories
1673 * in the given path
1674 * @throws FileNotFoundException if <code>f</code> does not exist
1675 * @throws IOException if any I/O error occurred
1676 */
1677 protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
1678 final PathFilter filter)
1679 throws FileNotFoundException, IOException {
1680 return new RemoteIterator<LocatedFileStatus>() {
1681 private final FileStatus[] stats = listStatus(f, filter);
1682 private int i = 0;
1683
1684 @Override
1685 public boolean hasNext() {
1686 return i<stats.length;
1687 }
1688
1689 @Override
1690 public LocatedFileStatus next() throws IOException {
1691 if (!hasNext()) {
1692 throw new NoSuchElementException("No more entry in " + f);
1693 }
1694 FileStatus result = stats[i++];
1695 BlockLocation[] locs = result.isFile() ?
1696 getFileBlockLocations(result.getPath(), 0, result.getLen()) :
1697 null;
1698 return new LocatedFileStatus(result, locs);
1699 }
1700 };
1701 }
1702
1703 /**
1704 * List the statuses and block locations of the files in the given path.
1705 *
1706 * If the path is a directory,
1707 * if recursive is false, returns files in the directory;
1708 * if recursive is true, return files in the subtree rooted at the path.
1709 * If the path is a file, return the file's status and block locations.
1710 *
1711 * @param f is the path
1712 * @param recursive if the subdirectories need to be traversed recursively
1713 *
1714 * @return an iterator that traverses statuses of the files
1715 *
1716 * @throws FileNotFoundException when the path does not exist;
1717 * IOException see specific implementation
1718 */
1719 public RemoteIterator<LocatedFileStatus> listFiles(
1720 final Path f, final boolean recursive)
1721 throws FileNotFoundException, IOException {
1722 return new RemoteIterator<LocatedFileStatus>() {
1723 private Stack<RemoteIterator<LocatedFileStatus>> itors =
1724 new Stack<RemoteIterator<LocatedFileStatus>>();
1725 private RemoteIterator<LocatedFileStatus> curItor =
1726 listLocatedStatus(f);
1727 private LocatedFileStatus curFile;
1728
1729 @Override
1730 public boolean hasNext() throws IOException {
1731 while (curFile == null) {
1732 if (curItor.hasNext()) {
1733 handleFileStat(curItor.next());
1734 } else if (!itors.empty()) {
1735 curItor = itors.pop();
1736 } else {
1737 return false;
1738 }
1739 }
1740 return true;
1741 }
1742
1743 /**
1744 * Process the input stat.
1745 * If it is a file, return the file stat.
1746 * If it is a directory, traverse the directory if recursive is true;
1747 * ignore it if recursive is false.
1748 * @param stat input status
1749 * @throws IOException if any IO error occurs
1750 */
1751 private void handleFileStat(LocatedFileStatus stat) throws IOException {
1752 if (stat.isFile()) { // file
1753 curFile = stat;
1754 } else if (recursive) { // directory
1755 itors.push(curItor);
1756 curItor = listLocatedStatus(stat.getPath());
1757 }
1758 }
1759
1760 @Override
1761 public LocatedFileStatus next() throws IOException {
1762 if (hasNext()) {
1763 LocatedFileStatus result = curFile;
1764 curFile = null;
1765 return result;
1766 }
1767 throw new java.util.NoSuchElementException("No more entry in " + f);
1768 }
1769 };
1770 }
1771
1772 /** Return the current user's home directory in this filesystem.
1773 * The default implementation returns "/user/$USER/".
1774 */
1775 public Path getHomeDirectory() {
1776 return this.makeQualified(
1777 new Path("/user/"+System.getProperty("user.name")));
1778 }
1779
1780
1781 /**
1782 * Set the current working directory for the given file system. All relative
1783 * paths will be resolved relative to it.
1784 *
1785 * @param new_dir
1786 */
1787 public abstract void setWorkingDirectory(Path new_dir);
1788
1789 /**
1790 * Get the current working directory for the given file system
1791 * @return the directory pathname
1792 */
1793 public abstract Path getWorkingDirectory();
1794
1795
1796 /**
1797 * Note: with the new FilesContext class, getWorkingDirectory()
1798 * will be removed.
1799 * The working directory is implemented in FilesContext.
1800 *
1801 * Some file systems like LocalFileSystem have an initial workingDir
1802 * that we use as the starting workingDir. For other file systems
1803 * like HDFS there is no built in notion of an initial workingDir.
1804 *
1805 * @return if there is built in notion of workingDir then it
1806 * is returned; else a null is returned.
1807 */
1808 protected Path getInitialWorkingDirectory() {
1809 return null;
1810 }
1811
1812 /**
1813 * Call {@link #mkdirs(Path, FsPermission)} with default permission.
1814 */
1815 public boolean mkdirs(Path f) throws IOException {
1816 return mkdirs(f, FsPermission.getDirDefault());
1817 }
1818
1819 /**
1820 * Make the given file and all non-existent parents into
1821 * directories. Has the semantics of Unix 'mkdir -p'.
1822 * Existence of the directory hierarchy is not an error.
1823 * @param f path to create
1824 * @param permission to apply to f
1825 */
1826 public abstract boolean mkdirs(Path f, FsPermission permission
1827 ) throws IOException;
1828
1829 /**
1830 * The src file is on the local disk. Add it to FS at
1831 * the given dst name and the source is kept intact afterwards
1832 * @param src path
1833 * @param dst path
1834 */
1835 public void copyFromLocalFile(Path src, Path dst)
1836 throws IOException {
1837 copyFromLocalFile(false, src, dst);
1838 }
1839
1840 /**
1841 * The src files is on the local disk. Add it to FS at
1842 * the given dst name, removing the source afterwards.
1843 * @param srcs path
1844 * @param dst path
1845 */
1846 public void moveFromLocalFile(Path[] srcs, Path dst)
1847 throws IOException {
1848 copyFromLocalFile(true, true, srcs, dst);
1849 }
1850
1851 /**
1852 * The src file is on the local disk. Add it to FS at
1853 * the given dst name, removing the source afterwards.
1854 * @param src path
1855 * @param dst path
1856 */
1857 public void moveFromLocalFile(Path src, Path dst)
1858 throws IOException {
1859 copyFromLocalFile(true, src, dst);
1860 }
1861
1862 /**
1863 * The src file is on the local disk. Add it to FS at
1864 * the given dst name.
1865 * delSrc indicates if the source should be removed
1866 * @param delSrc whether to delete the src
1867 * @param src path
1868 * @param dst path
1869 */
1870 public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
1871 throws IOException {
1872 copyFromLocalFile(delSrc, true, src, dst);
1873 }
1874
1875 /**
1876 * The src files are on the local disk. Add it to FS at
1877 * the given dst name.
1878 * delSrc indicates if the source should be removed
1879 * @param delSrc whether to delete the src
1880 * @param overwrite whether to overwrite an existing file
1881 * @param srcs array of paths which are source
1882 * @param dst path
1883 */
1884 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
1885 Path[] srcs, Path dst)
1886 throws IOException {
1887 Configuration conf = getConf();
1888 FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
1889 }
1890
1891 /**
1892 * The src file is on the local disk. Add it to FS at
1893 * the given dst name.
1894 * delSrc indicates if the source should be removed
1895 * @param delSrc whether to delete the src
1896 * @param overwrite whether to overwrite an existing file
1897 * @param src path
1898 * @param dst path
1899 */
1900 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
1901 Path src, Path dst)
1902 throws IOException {
1903 Configuration conf = getConf();
1904 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
1905 }
1906
1907 /**
1908 * The src file is under FS, and the dst is on the local disk.
1909 * Copy it from FS control to the local dst name.
1910 * @param src path
1911 * @param dst path
1912 */
1913 public void copyToLocalFile(Path src, Path dst) throws IOException {
1914 copyToLocalFile(false, src, dst);
1915 }
1916
1917 /**
1918 * The src file is under FS, and the dst is on the local disk.
1919 * Copy it from FS control to the local dst name.
1920 * Remove the source afterwards
1921 * @param src path
1922 * @param dst path
1923 */
1924 public void moveToLocalFile(Path src, Path dst) throws IOException {
1925 copyToLocalFile(true, src, dst);
1926 }
1927
1928 /**
1929 * The src file is under FS, and the dst is on the local disk.
1930 * Copy it from FS control to the local dst name.
1931 * delSrc indicates if the src will be removed or not.
1932 * @param delSrc whether to delete the src
1933 * @param src path
1934 * @param dst path
1935 */
1936 public void copyToLocalFile(boolean delSrc, Path src, Path dst)
1937 throws IOException {
1938 copyToLocalFile(delSrc, src, dst, false);
1939 }
1940
1941 /**
1942 * The src file is under FS, and the dst is on the local disk. Copy it from FS
1943 * control to the local dst name. delSrc indicates if the src will be removed
1944 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
1945 * as local file system or not. RawLocalFileSystem is non crc file system.So,
1946 * It will not create any crc files at local.
1947 *
1948 * @param delSrc
1949 * whether to delete the src
1950 * @param src
1951 * path
1952 * @param dst
1953 * path
1954 * @param useRawLocalFileSystem
1955 * whether to use RawLocalFileSystem as local file system or not.
1956 *
1957 * @throws IOException
1958 * - if any IO error
1959 */
1960 public void copyToLocalFile(boolean delSrc, Path src, Path dst,
1961 boolean useRawLocalFileSystem) throws IOException {
1962 Configuration conf = getConf();
1963 FileSystem local = null;
1964 if (useRawLocalFileSystem) {
1965 local = getLocal(conf).getRawFileSystem();
1966 } else {
1967 local = getLocal(conf);
1968 }
1969 FileUtil.copy(this, src, local, dst, delSrc, conf);
1970 }
1971
1972 /**
1973 * Returns a local File that the user can write output to. The caller
1974 * provides both the eventual FS target name and the local working
1975 * file. If the FS is local, we write directly into the target. If
1976 * the FS is remote, we write into the tmp local area.
1977 * @param fsOutputFile path of output file
1978 * @param tmpLocalFile path of local tmp file
1979 */
1980 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1981 throws IOException {
1982 return tmpLocalFile;
1983 }
1984
1985 /**
1986 * Called when we're all done writing to the target. A local FS will
1987 * do nothing, because we've written to exactly the right place. A remote
1988 * FS will copy the contents of tmpLocalFile to the correct target at
1989 * fsOutputFile.
1990 * @param fsOutputFile path of output file
1991 * @param tmpLocalFile path to local tmp file
1992 */
1993 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1994 throws IOException {
1995 moveFromLocalFile(tmpLocalFile, fsOutputFile);
1996 }
1997
1998 /**
1999 * No more filesystem operations are needed. Will
2000 * release any held locks.
2001 */
2002 @Override
2003 public void close() throws IOException {
2004 // delete all files that were marked as delete-on-exit.
2005 processDeleteOnExit();
2006 CACHE.remove(this.key, this);
2007 }
2008
2009 /** Return the total size of all files in the filesystem.*/
2010 public long getUsed() throws IOException{
2011 long used = 0;
2012 FileStatus[] files = listStatus(new Path("/"));
2013 for(FileStatus file:files){
2014 used += file.getLen();
2015 }
2016 return used;
2017 }
2018
2019 /**
2020 * Get the block size for a particular file.
2021 * @param f the filename
2022 * @return the number of bytes in a block
2023 */
2024 /** @deprecated Use getFileStatus() instead */
2025 @Deprecated
2026 public long getBlockSize(Path f) throws IOException {
2027 return getFileStatus(f).getBlockSize();
2028 }
2029
2030 /**
2031 * Return the number of bytes that large input files should be optimally
2032 * be split into to minimize i/o time.
2033 * @deprecated use {@link #getDefaultBlockSize(Path)} instead
2034 */
2035 @Deprecated
2036 public long getDefaultBlockSize() {
2037 // default to 32MB: large enough to minimize the impact of seeks
2038 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
2039 }
2040
2041 /** Return the number of bytes that large input files should be optimally
2042 * be split into to minimize i/o time. The given path will be used to
2043 * locate the actual filesystem. The full path does not have to exist.
2044 * @param f path of file
2045 * @return the default block size for the path's filesystem
2046 */
2047 public long getDefaultBlockSize(Path f) {
2048 return getDefaultBlockSize();
2049 }
2050
2051 /**
2052 * Get the default replication.
2053 * @deprecated use {@link #getDefaultReplication(Path)} instead
2054 */
2055 @Deprecated
2056 public short getDefaultReplication() { return 1; }
2057
2058 /**
2059 * Get the default replication for a path. The given path will be used to
2060 * locate the actual filesystem. The full path does not have to exist.
2061 * @param path of the file
2062 * @return default replication for the path's filesystem
2063 */
2064 public short getDefaultReplication(Path path) {
2065 return getDefaultReplication();
2066 }
2067
2068 /**
2069 * Return a file status object that represents the path.
2070 * @param f The path we want information from
2071 * @return a FileStatus object
2072 * @throws FileNotFoundException when the path does not exist;
2073 * IOException see specific implementation
2074 */
2075 public abstract FileStatus getFileStatus(Path f) throws IOException;
2076
2077 /**
2078 * See {@link FileContext#fixRelativePart}
2079 */
2080 protected Path fixRelativePart(Path p) {
2081 if (p.isUriPathAbsolute()) {
2082 return p;
2083 } else {
2084 return new Path(getWorkingDirectory(), p);
2085 }
2086 }
2087
2088 /**
2089 * See {@link FileContext#createSymlink(Path, Path, boolean)}
2090 */
2091 public void createSymlink(final Path target, final Path link,
2092 final boolean createParent) throws AccessControlException,
2093 FileAlreadyExistsException, FileNotFoundException,
2094 ParentNotDirectoryException, UnsupportedFileSystemException,
2095 IOException {
2096 // Supporting filesystems should override this method
2097 throw new UnsupportedOperationException(
2098 "Filesystem does not support symlinks!");
2099 }
2100
2101 /**
2102 * See {@link FileContext#getFileLinkStatus(Path)}
2103 */
2104 public FileStatus getFileLinkStatus(final Path f)
2105 throws AccessControlException, FileNotFoundException,
2106 UnsupportedFileSystemException, IOException {
2107 // Supporting filesystems should override this method
2108 return getFileStatus(f);
2109 }
2110
2111 /**
2112 * See {@link AbstractFileSystem#supportsSymlinks()}
2113 */
2114 public boolean supportsSymlinks() {
2115 return false;
2116 }
2117
2118 /**
2119 * See {@link FileContext#getLinkTarget(Path)}
2120 */
2121 public Path getLinkTarget(Path f) throws IOException {
2122 // Supporting filesystems should override this method
2123 throw new UnsupportedOperationException(
2124 "Filesystem does not support symlinks!");
2125 }
2126
2127 /**
2128 * See {@link AbstractFileSystem#getLinkTarget(Path)}
2129 */
2130 protected Path resolveLink(Path f) throws IOException {
2131 // Supporting filesystems should override this method
2132 throw new UnsupportedOperationException(
2133 "Filesystem does not support symlinks!");
2134 }
2135
2136 /**
2137 * Get the checksum of a file.
2138 *
2139 * @param f The file path
2140 * @return The file checksum. The default return value is null,
2141 * which indicates that no checksum algorithm is implemented
2142 * in the corresponding FileSystem.
2143 */
2144 public FileChecksum getFileChecksum(Path f) throws IOException {
2145 return null;
2146 }
2147
2148 /**
2149 * Set the verify checksum flag. This is only applicable if the
2150 * corresponding FileSystem supports checksum. By default doesn't do anything.
2151 * @param verifyChecksum
2152 */
2153 public void setVerifyChecksum(boolean verifyChecksum) {
2154 //doesn't do anything
2155 }
2156
2157 /**
2158 * Set the write checksum flag. This is only applicable if the
2159 * corresponding FileSystem supports checksum. By default doesn't do anything.
2160 * @param writeChecksum
2161 */
2162 public void setWriteChecksum(boolean writeChecksum) {
2163 //doesn't do anything
2164 }
2165
2166 /**
2167 * Returns a status object describing the use and capacity of the
2168 * file system. If the file system has multiple partitions, the
2169 * use and capacity of the root partition is reflected.
2170 *
2171 * @return a FsStatus object
2172 * @throws IOException
2173 * see specific implementation
2174 */
2175 public FsStatus getStatus() throws IOException {
2176 return getStatus(null);
2177 }
2178
2179 /**
2180 * Returns a status object describing the use and capacity of the
2181 * file system. If the file system has multiple partitions, the
2182 * use and capacity of the partition pointed to by the specified
2183 * path is reflected.
2184 * @param p Path for which status should be obtained. null means
2185 * the default partition.
2186 * @return a FsStatus object
2187 * @throws IOException
2188 * see specific implementation
2189 */
2190 public FsStatus getStatus(Path p) throws IOException {
2191 return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE);
2192 }
2193
2194 /**
2195 * Set permission of a path.
2196 * @param p
2197 * @param permission
2198 */
2199 public void setPermission(Path p, FsPermission permission
2200 ) throws IOException {
2201 }
2202
2203 /**
2204 * Set owner of a path (i.e. a file or a directory).
2205 * The parameters username and groupname cannot both be null.
2206 * @param p The path
2207 * @param username If it is null, the original username remains unchanged.
2208 * @param groupname If it is null, the original groupname remains unchanged.
2209 */
2210 public void setOwner(Path p, String username, String groupname
2211 ) throws IOException {
2212 }
2213
2214 /**
2215 * Set access time of a file
2216 * @param p The path
2217 * @param mtime Set the modification time of this file.
2218 * The number of milliseconds since Jan 1, 1970.
2219 * A value of -1 means that this call should not set modification time.
2220 * @param atime Set the access time of this file.
2221 * The number of milliseconds since Jan 1, 1970.
2222 * A value of -1 means that this call should not set access time.
2223 */
2224 public void setTimes(Path p, long mtime, long atime
2225 ) throws IOException {
2226 }
2227
2228 /**
2229 * Create a snapshot with a default name.
2230 * @param path The directory where snapshots will be taken.
2231 * @return the snapshot path.
2232 */
2233 public final Path createSnapshot(Path path) throws IOException {
2234 return createSnapshot(path, null);
2235 }
2236
2237 /**
2238 * Create a snapshot
2239 * @param path The directory where snapshots will be taken.
2240 * @param snapshotName The name of the snapshot
2241 * @return the snapshot path.
2242 */
2243 public Path createSnapshot(Path path, String snapshotName)
2244 throws IOException {
2245 throw new UnsupportedOperationException(getClass().getSimpleName()
2246 + " doesn't support createSnapshot");
2247 }
2248
2249 /**
2250 * Rename a snapshot
2251 * @param path The directory path where the snapshot was taken
2252 * @param snapshotOldName Old name of the snapshot
2253 * @param snapshotNewName New name of the snapshot
2254 * @throws IOException
2255 */
2256 public void renameSnapshot(Path path, String snapshotOldName,
2257 String snapshotNewName) throws IOException {
2258 throw new UnsupportedOperationException(getClass().getSimpleName()
2259 + " doesn't support renameSnapshot");
2260 }
2261
2262 /**
2263 * Delete a snapshot of a directory
2264 * @param path The directory that the to-be-deleted snapshot belongs to
2265 * @param snapshotName The name of the snapshot
2266 */
2267 public void deleteSnapshot(Path path, String snapshotName)
2268 throws IOException {
2269 throw new UnsupportedOperationException(getClass().getSimpleName()
2270 + " doesn't support deleteSnapshot");
2271 }
2272
2273 // making it volatile to be able to do a double checked locking
2274 private volatile static boolean FILE_SYSTEMS_LOADED = false;
2275
2276 private static final Map<String, Class<? extends FileSystem>>
2277 SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>();
2278
2279 private static void loadFileSystems() {
2280 synchronized (FileSystem.class) {
2281 if (!FILE_SYSTEMS_LOADED) {
2282 ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
2283 for (FileSystem fs : serviceLoader) {
2284 SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
2285 }
2286 FILE_SYSTEMS_LOADED = true;
2287 }
2288 }
2289 }
2290
2291 public static Class<? extends FileSystem> getFileSystemClass(String scheme,
2292 Configuration conf) throws IOException {
2293 if (!FILE_SYSTEMS_LOADED) {
2294 loadFileSystems();
2295 }
2296 Class<? extends FileSystem> clazz = null;
2297 if (conf != null) {
2298 clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null);
2299 }
2300 if (clazz == null) {
2301 clazz = SERVICE_FILE_SYSTEMS.get(scheme);
2302 }
2303 if (clazz == null) {
2304 throw new IOException("No FileSystem for scheme: " + scheme);
2305 }
2306 return clazz;
2307 }
2308
2309 private static FileSystem createFileSystem(URI uri, Configuration conf
2310 ) throws IOException {
2311 Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
2312 if (clazz == null) {
2313 throw new IOException("No FileSystem for scheme: " + uri.getScheme());
2314 }
2315 FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
2316 fs.initialize(uri, conf);
2317 return fs;
2318 }
2319
2320 /** Caching FileSystem objects */
2321 static class Cache {
2322 private final ClientFinalizer clientFinalizer = new ClientFinalizer();
2323
2324 private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
2325 private final Set<Key> toAutoClose = new HashSet<Key>();
2326
2327 /** A variable that makes all objects in the cache unique */
2328 private static AtomicLong unique = new AtomicLong(1);
2329
2330 FileSystem get(URI uri, Configuration conf) throws IOException{
2331 Key key = new Key(uri, conf);
2332 return getInternal(uri, conf, key);
2333 }
2334
2335 /** The objects inserted into the cache using this method are all unique */
2336 FileSystem getUnique(URI uri, Configuration conf) throws IOException{
2337 Key key = new Key(uri, conf, unique.getAndIncrement());
2338 return getInternal(uri, conf, key);
2339 }
2340
2341 private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
2342 FileSystem fs;
2343 synchronized (this) {
2344 fs = map.get(key);
2345 }
2346 if (fs != null) {
2347 return fs;
2348 }
2349
2350 fs = createFileSystem(uri, conf);
2351 synchronized (this) { // refetch the lock again
2352 FileSystem oldfs = map.get(key);
2353 if (oldfs != null) { // a file system is created while lock is releasing
2354 fs.close(); // close the new file system
2355 return oldfs; // return the old file system
2356 }
2357
2358 // now insert the new file system into the map
2359 if (map.isEmpty()
2360 && !ShutdownHookManager.get().isShutdownInProgress()) {
2361 ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY);
2362 }
2363 fs.key = key;
2364 map.put(key, fs);
2365 if (conf.getBoolean("fs.automatic.close", true)) {
2366 toAutoClose.add(key);
2367 }
2368 return fs;
2369 }
2370 }
2371
2372 synchronized void remove(Key key, FileSystem fs) {
2373 if (map.containsKey(key) && fs == map.get(key)) {
2374 map.remove(key);
2375 toAutoClose.remove(key);
2376 }
2377 }
2378
2379 synchronized void closeAll() throws IOException {
2380 closeAll(false);
2381 }
2382
2383 /**
2384 * Close all FileSystem instances in the Cache.
2385 * @param onlyAutomatic only close those that are marked for automatic closing
2386 */
2387 synchronized void closeAll(boolean onlyAutomatic) throws IOException {
2388 List<IOException> exceptions = new ArrayList<IOException>();
2389
2390 // Make a copy of the keys in the map since we'll be modifying
2391 // the map while iterating over it, which isn't safe.
2392 List<Key> keys = new ArrayList<Key>();
2393 keys.addAll(map.keySet());
2394
2395 for (Key key : keys) {
2396 final FileSystem fs = map.get(key);
2397
2398 if (onlyAutomatic && !toAutoClose.contains(key)) {
2399 continue;
2400 }
2401
2402 //remove from cache
2403 remove(key, fs);
2404
2405 if (fs != null) {
2406 try {
2407 fs.close();
2408 }
2409 catch(IOException ioe) {
2410 exceptions.add(ioe);
2411 }
2412 }
2413 }
2414
2415 if (!exceptions.isEmpty()) {
2416 throw MultipleIOException.createIOException(exceptions);
2417 }
2418 }
2419
2420 private class ClientFinalizer implements Runnable {
2421 @Override
2422 public synchronized void run() {
2423 try {
2424 closeAll(true);
2425 } catch (IOException e) {
2426 LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e);
2427 }
2428 }
2429 }
2430
2431 synchronized void closeAll(UserGroupInformation ugi) throws IOException {
2432 List<FileSystem> targetFSList = new ArrayList<FileSystem>();
2433 //Make a pass over the list and collect the filesystems to close
2434 //we cannot close inline since close() removes the entry from the Map
2435 for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
2436 final Key key = entry.getKey();
2437 final FileSystem fs = entry.getValue();
2438 if (ugi.equals(key.ugi) && fs != null) {
2439 targetFSList.add(fs);
2440 }
2441 }
2442 List<IOException> exceptions = new ArrayList<IOException>();
2443 //now make a pass over the target list and close each
2444 for (FileSystem fs : targetFSList) {
2445 try {
2446 fs.close();
2447 }
2448 catch(IOException ioe) {
2449 exceptions.add(ioe);
2450 }
2451 }
2452 if (!exceptions.isEmpty()) {
2453 throw MultipleIOException.createIOException(exceptions);
2454 }
2455 }
2456
2457 /** FileSystem.Cache.Key */
2458 static class Key {
2459 final String scheme;
2460 final String authority;
2461 final UserGroupInformation ugi;
2462 final long unique; // an artificial way to make a key unique
2463
2464 Key(URI uri, Configuration conf) throws IOException {
2465 this(uri, conf, 0);
2466 }
2467
2468 Key(URI uri, Configuration conf, long unique) throws IOException {
2469 scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
2470 authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
2471 this.unique = unique;
2472
2473 this.ugi = UserGroupInformation.getCurrentUser();
2474 }
2475
2476 @Override
2477 public int hashCode() {
2478 return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique;
2479 }
2480
2481 static boolean isEqual(Object a, Object b) {
2482 return a == b || (a != null && a.equals(b));
2483 }
2484
2485 @Override
2486 public boolean equals(Object obj) {
2487 if (obj == this) {
2488 return true;
2489 }
2490 if (obj != null && obj instanceof Key) {
2491 Key that = (Key)obj;
2492 return isEqual(this.scheme, that.scheme)
2493 && isEqual(this.authority, that.authority)
2494 && isEqual(this.ugi, that.ugi)
2495 && (this.unique == that.unique);
2496 }
2497 return false;
2498 }
2499
2500 @Override
2501 public String toString() {
2502 return "("+ugi.toString() + ")@" + scheme + "://" + authority;
2503 }
2504 }
2505 }
2506
2507 /**
2508 * Tracks statistics about how many reads, writes, and so forth have been
2509 * done in a FileSystem.
2510 *
2511 * Since there is only one of these objects per FileSystem, there will
2512 * typically be many threads writing to this object. Almost every operation
2513 * on an open file will involve a write to this object. In contrast, reading
2514 * statistics is done infrequently by most programs, and not at all by others.
2515 * Hence, this is optimized for writes.
2516 *
2517 * Each thread writes to its own thread-local area of memory. This removes
2518 * contention and allows us to scale up to many, many threads. To read
2519 * statistics, the reader thread totals up the contents of all of the
2520 * thread-local data areas.
2521 */
2522 public static final class Statistics {
2523 /**
2524 * Statistics data.
2525 *
2526 * There is only a single writer to thread-local StatisticsData objects.
2527 * Hence, volatile is adequate here-- we do not need AtomicLong or similar
2528 * to prevent lost updates.
2529 * The Java specification guarantees that updates to volatile longs will
2530 * be perceived as atomic with respect to other threads, which is all we
2531 * need.
2532 */
2533 private static class StatisticsData {
2534 volatile long bytesRead;
2535 volatile long bytesWritten;
2536 volatile int readOps;
2537 volatile int largeReadOps;
2538 volatile int writeOps;
2539 /**
2540 * Stores a weak reference to the thread owning this StatisticsData.
2541 * This allows us to remove StatisticsData objects that pertain to
2542 * threads that no longer exist.
2543 */
2544 final WeakReference<Thread> owner;
2545
2546 StatisticsData(WeakReference<Thread> owner) {
2547 this.owner = owner;
2548 }
2549
2550 /**
2551 * Add another StatisticsData object to this one.
2552 */
2553 void add(StatisticsData other) {
2554 this.bytesRead += other.bytesRead;
2555 this.bytesWritten += other.bytesWritten;
2556 this.readOps += other.readOps;
2557 this.largeReadOps += other.largeReadOps;
2558 this.writeOps += other.writeOps;
2559 }
2560
2561 /**
2562 * Negate the values of all statistics.
2563 */
2564 void negate() {
2565 this.bytesRead = -this.bytesRead;
2566 this.bytesWritten = -this.bytesWritten;
2567 this.readOps = -this.readOps;
2568 this.largeReadOps = -this.largeReadOps;
2569 this.writeOps = -this.writeOps;
2570 }
2571
2572 @Override
2573 public String toString() {
2574 return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
2575 + readOps + " read ops, " + largeReadOps + " large read ops, "
2576 + writeOps + " write ops";
2577 }
2578 }
2579
2580 private interface StatisticsAggregator<T> {
2581 void accept(StatisticsData data);
2582 T aggregate();
2583 }
2584
2585 private final String scheme;
2586
2587 /**
2588 * rootData is data that doesn't belong to any thread, but will be added
2589 * to the totals. This is useful for making copies of Statistics objects,
2590 * and for storing data that pertains to threads that have been garbage
2591 * collected. Protected by the Statistics lock.
2592 */
2593 private final StatisticsData rootData;
2594
2595 /**
2596 * Thread-local data.
2597 */
2598 private final ThreadLocal<StatisticsData> threadData;
2599
2600 /**
2601 * List of all thread-local data areas. Protected by the Statistics lock.
2602 */
2603 private LinkedList<StatisticsData> allData;
2604
2605 public Statistics(String scheme) {
2606 this.scheme = scheme;
2607 this.rootData = new StatisticsData(null);
2608 this.threadData = new ThreadLocal<StatisticsData>();
2609 this.allData = null;
2610 }
2611
2612 /**
2613 * Copy constructor.
2614 *
2615 * @param other The input Statistics object which is cloned.
2616 */
2617 public Statistics(Statistics other) {
2618 this.scheme = other.scheme;
2619 this.rootData = new StatisticsData(null);
2620 other.visitAll(new StatisticsAggregator<Void>() {
2621 @Override
2622 public void accept(StatisticsData data) {
2623 rootData.add(data);
2624 }
2625
2626 public Void aggregate() {
2627 return null;
2628 }
2629 });
2630 this.threadData = new ThreadLocal<StatisticsData>();
2631 }
2632
2633 /**
2634 * Get or create the thread-local data associated with the current thread.
2635 */
2636 private StatisticsData getThreadData() {
2637 StatisticsData data = threadData.get();
2638 if (data == null) {
2639 data = new StatisticsData(
2640 new WeakReference<Thread>(Thread.currentThread()));
2641 threadData.set(data);
2642 synchronized(this) {
2643 if (allData == null) {
2644 allData = new LinkedList<StatisticsData>();
2645 }
2646 allData.add(data);
2647 }
2648 }
2649 return data;
2650 }
2651
2652 /**
2653 * Increment the bytes read in the statistics
2654 * @param newBytes the additional bytes read
2655 */
2656 public void incrementBytesRead(long newBytes) {
2657 getThreadData().bytesRead += newBytes;
2658 }
2659
2660 /**
2661 * Increment the bytes written in the statistics
2662 * @param newBytes the additional bytes written
2663 */
2664 public void incrementBytesWritten(long newBytes) {
2665 getThreadData().bytesWritten += newBytes;
2666 }
2667
2668 /**
2669 * Increment the number of read operations
2670 * @param count number of read operations
2671 */
2672 public void incrementReadOps(int count) {
2673 getThreadData().readOps += count;
2674 }
2675
2676 /**
2677 * Increment the number of large read operations
2678 * @param count number of large read operations
2679 */
2680 public void incrementLargeReadOps(int count) {
2681 getThreadData().largeReadOps += count;
2682 }
2683
2684 /**
2685 * Increment the number of write operations
2686 * @param count number of write operations
2687 */
2688 public void incrementWriteOps(int count) {
2689 getThreadData().writeOps += count;
2690 }
2691
2692 /**
2693 * Apply the given aggregator to all StatisticsData objects associated with
2694 * this Statistics object.
2695 *
2696 * For each StatisticsData object, we will call accept on the visitor.
2697 * Finally, at the end, we will call aggregate to get the final total.
2698 *
2699 * @param The visitor to use.
2700 * @return The total.
2701 */
2702 private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
2703 visitor.accept(rootData);
2704 if (allData != null) {
2705 for (Iterator<StatisticsData> iter = allData.iterator();
2706 iter.hasNext(); ) {
2707 StatisticsData data = iter.next();
2708 visitor.accept(data);
2709 if (data.owner.get() == null) {
2710 /*
2711 * If the thread that created this thread-local data no
2712 * longer exists, remove the StatisticsData from our list
2713 * and fold the values into rootData.
2714 */
2715 rootData.add(data);
2716 iter.remove();
2717 }
2718 }
2719 }
2720 return visitor.aggregate();
2721 }
2722
2723 /**
2724 * Get the total number of bytes read
2725 * @return the number of bytes
2726 */
2727 public long getBytesRead() {
2728 return visitAll(new StatisticsAggregator<Long>() {
2729 private long bytesRead = 0;
2730
2731 @Override
2732 public void accept(StatisticsData data) {
2733 bytesRead += data.bytesRead;
2734 }
2735
2736 public Long aggregate() {
2737 return bytesRead;
2738 }
2739 });
2740 }
2741
2742 /**
2743 * Get the total number of bytes written
2744 * @return the number of bytes
2745 */
2746 public long getBytesWritten() {
2747 return visitAll(new StatisticsAggregator<Long>() {
2748 private long bytesWritten = 0;
2749
2750 @Override
2751 public void accept(StatisticsData data) {
2752 bytesWritten += data.bytesWritten;
2753 }
2754
2755 public Long aggregate() {
2756 return bytesWritten;
2757 }
2758 });
2759 }
2760
2761 /**
2762 * Get the number of file system read operations such as list files
2763 * @return number of read operations
2764 */
2765 public int getReadOps() {
2766 return visitAll(new StatisticsAggregator<Integer>() {
2767 private int readOps = 0;
2768
2769 @Override
2770 public void accept(StatisticsData data) {
2771 readOps += data.readOps;
2772 readOps += data.largeReadOps;
2773 }
2774
2775 public Integer aggregate() {
2776 return readOps;
2777 }
2778 });
2779 }
2780
2781 /**
2782 * Get the number of large file system read operations such as list files
2783 * under a large directory
2784 * @return number of large read operations
2785 */
2786 public int getLargeReadOps() {
2787 return visitAll(new StatisticsAggregator<Integer>() {
2788 private int largeReadOps = 0;
2789
2790 @Override
2791 public void accept(StatisticsData data) {
2792 largeReadOps += data.largeReadOps;
2793 }
2794
2795 public Integer aggregate() {
2796 return largeReadOps;
2797 }
2798 });
2799 }
2800
2801 /**
2802 * Get the number of file system write operations such as create, append
2803 * rename etc.
2804 * @return number of write operations
2805 */
2806 public int getWriteOps() {
2807 return visitAll(new StatisticsAggregator<Integer>() {
2808 private int writeOps = 0;
2809
2810 @Override
2811 public void accept(StatisticsData data) {
2812 writeOps += data.writeOps;
2813 }
2814
2815 public Integer aggregate() {
2816 return writeOps;
2817 }
2818 });
2819 }
2820
2821
2822 @Override
2823 public String toString() {
2824 return visitAll(new StatisticsAggregator<String>() {
2825 private StatisticsData total = new StatisticsData(null);
2826
2827 @Override
2828 public void accept(StatisticsData data) {
2829 total.add(data);
2830 }
2831
2832 public String aggregate() {
2833 return total.toString();
2834 }
2835 });
2836 }
2837
2838 /**
2839 * Resets all statistics to 0.
2840 *
2841 * In order to reset, we add up all the thread-local statistics data, and
2842 * set rootData to the negative of that.
2843 *
2844 * This may seem like a counterintuitive way to reset the statsitics. Why
2845 * can't we just zero out all the thread-local data? Well, thread-local
2846 * data can only be modified by the thread that owns it. If we tried to
2847 * modify the thread-local data from this thread, our modification might get
2848 * interleaved with a read-modify-write operation done by the thread that
2849 * owns the data. That would result in our update getting lost.
2850 *
2851 * The approach used here avoids this problem because it only ever reads
2852 * (not writes) the thread-local data. Both reads and writes to rootData
2853 * are done under the lock, so we're free to modify rootData from any thread
2854 * that holds the lock.
2855 */
2856 public void reset() {
2857 visitAll(new StatisticsAggregator<Void>() {
2858 private StatisticsData total = new StatisticsData(null);
2859
2860 @Override
2861 public void accept(StatisticsData data) {
2862 total.add(data);
2863 }
2864
2865 public Void aggregate() {
2866 total.negate();
2867 rootData.add(total);
2868 return null;
2869 }
2870 });
2871 }
2872
2873 /**
2874 * Get the uri scheme associated with this statistics object.
2875 * @return the schema associated with this set of statistics
2876 */
2877 public String getScheme() {
2878 return scheme;
2879 }
2880 }
2881
2882 /**
2883 * Get the Map of Statistics object indexed by URI Scheme.
2884 * @return a Map having a key as URI scheme and value as Statistics object
2885 * @deprecated use {@link #getAllStatistics} instead
2886 */
2887 @Deprecated
2888 public static synchronized Map<String, Statistics> getStatistics() {
2889 Map<String, Statistics> result = new HashMap<String, Statistics>();
2890 for(Statistics stat: statisticsTable.values()) {
2891 result.put(stat.getScheme(), stat);
2892 }
2893 return result;
2894 }
2895
2896 /**
2897 * Return the FileSystem classes that have Statistics
2898 */
2899 public static synchronized List<Statistics> getAllStatistics() {
2900 return new ArrayList<Statistics>(statisticsTable.values());
2901 }
2902
2903 /**
2904 * Get the statistics for a particular file system
2905 * @param cls the class to lookup
2906 * @return a statistics object
2907 */
2908 public static synchronized
2909 Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
2910 Statistics result = statisticsTable.get(cls);
2911 if (result == null) {
2912 result = new Statistics(scheme);
2913 statisticsTable.put(cls, result);
2914 }
2915 return result;
2916 }
2917
2918 /**
2919 * Reset all statistics for all file systems
2920 */
2921 public static synchronized void clearStatistics() {
2922 for(Statistics stat: statisticsTable.values()) {
2923 stat.reset();
2924 }
2925 }
2926
2927 /**
2928 * Print all statistics for all file systems
2929 */
2930 public static synchronized
2931 void printStatistics() throws IOException {
2932 for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
2933 statisticsTable.entrySet()) {
2934 System.out.println(" FileSystem " + pair.getKey().getName() +
2935 ": " + pair.getValue());
2936 }
2937 }
2938
2939 // Symlinks are temporarily disabled - see HADOOP-10020 and HADOOP-10052
2940 private static boolean symlinksEnabled = false;
2941
2942 private static Configuration conf = null;
2943
2944 @VisibleForTesting
2945 public static boolean areSymlinksEnabled() {
2946 return symlinksEnabled;
2947 }
2948
2949 @VisibleForTesting
2950 public static void enableSymlinks() {
2951 symlinksEnabled = true;
2952 }
2953 }