001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.fs;
019
020 import java.io.Closeable;
021 import java.io.FileNotFoundException;
022 import java.io.IOException;
023 import java.net.URI;
024 import java.security.PrivilegedExceptionAction;
025 import java.util.ArrayList;
026 import java.util.Arrays;
027 import java.util.EnumSet;
028 import java.util.HashMap;
029 import java.util.HashSet;
030 import java.util.IdentityHashMap;
031 import java.util.Iterator;
032 import java.util.List;
033 import java.util.Map;
034 import java.util.NoSuchElementException;
035 import java.util.Set;
036 import java.util.Stack;
037 import java.util.TreeSet;
038 import java.util.concurrent.atomic.AtomicInteger;
039 import java.util.concurrent.atomic.AtomicLong;
040
041 import org.apache.commons.logging.Log;
042 import org.apache.commons.logging.LogFactory;
043 import org.apache.hadoop.classification.InterfaceAudience;
044 import org.apache.hadoop.classification.InterfaceStability;
045 import org.apache.hadoop.conf.Configuration;
046 import org.apache.hadoop.conf.Configured;
047 import org.apache.hadoop.fs.Options.Rename;
048 import org.apache.hadoop.fs.permission.FsPermission;
049 import org.apache.hadoop.io.MultipleIOException;
050 import org.apache.hadoop.net.NetUtils;
051 import org.apache.hadoop.security.Credentials;
052 import org.apache.hadoop.security.SecurityUtil;
053 import org.apache.hadoop.security.UserGroupInformation;
054 import org.apache.hadoop.security.token.Token;
055 import org.apache.hadoop.util.Progressable;
056 import org.apache.hadoop.util.ReflectionUtils;
057
058 /****************************************************************
059 * An abstract base class for a fairly generic filesystem. It
060 * may be implemented as a distributed filesystem, or as a "local"
061 * one that reflects the locally-connected disk. The local version
062 * exists for small Hadoop instances and for testing.
063 *
064 * <p>
065 *
066 * All user code that may potentially use the Hadoop Distributed
067 * File System should be written to use a FileSystem object. The
068 * Hadoop DFS is a multi-machine system that appears as a single
069 * disk. It's useful because of its fault tolerance and potentially
070 * very large capacity.
071 *
072 * <p>
073 * The local implementation is {@link LocalFileSystem} and distributed
074 * implementation is DistributedFileSystem.
075 *****************************************************************/
076 @InterfaceAudience.Public
077 @InterfaceStability.Stable
078 public abstract class FileSystem extends Configured implements Closeable {
079 public static final String FS_DEFAULT_NAME_KEY =
080 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
081 public static final String DEFAULT_FS =
082 CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
083
084 public static final Log LOG = LogFactory.getLog(FileSystem.class);
085
086 /** FileSystem cache */
087 static final Cache CACHE = new Cache();
088
089 /** The key this instance is stored under in the cache. */
090 private Cache.Key key;
091
092 /** Recording statistics per a FileSystem class */
093 private static final Map<Class<? extends FileSystem>, Statistics>
094 statisticsTable =
095 new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
096
097 /**
098 * The statistics for this file system.
099 */
100 protected Statistics statistics;
101
102 /**
103 * A cache of files that should be deleted when filsystem is closed
104 * or the JVM is exited.
105 */
106 private Set<Path> deleteOnExit = new TreeSet<Path>();
107
108 /**
109 * This method adds a file system for testing so that we can find it later. It
110 * is only for testing.
111 * @param uri the uri to store it under
112 * @param conf the configuration to store it under
113 * @param fs the file system to store
114 * @throws IOException
115 */
116 static void addFileSystemForTesting(URI uri, Configuration conf,
117 FileSystem fs) throws IOException {
118 CACHE.map.put(new Cache.Key(uri, conf), fs);
119 }
120
121 /**
122 * Get a filesystem instance based on the uri, the passed
123 * configuration and the user
124 * @param uri of the filesystem
125 * @param conf the configuration to use
126 * @param user to perform the get as
127 * @return the filesystem instance
128 * @throws IOException
129 * @throws InterruptedException
130 */
131 public static FileSystem get(final URI uri, final Configuration conf,
132 final String user) throws IOException, InterruptedException {
133 UserGroupInformation ugi;
134 if (user == null) {
135 ugi = UserGroupInformation.getCurrentUser();
136 } else {
137 ugi = UserGroupInformation.createRemoteUser(user);
138 }
139 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
140 public FileSystem run() throws IOException {
141 return get(uri, conf);
142 }
143 });
144 }
145
146 /**
147 * Returns the configured filesystem implementation.
148 * @param conf the configuration to use
149 */
150 public static FileSystem get(Configuration conf) throws IOException {
151 return get(getDefaultUri(conf), conf);
152 }
153
154 /** Get the default filesystem URI from a configuration.
155 * @param conf the configuration to use
156 * @return the uri of the default filesystem
157 */
158 public static URI getDefaultUri(Configuration conf) {
159 return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
160 }
161
162 /** Set the default filesystem URI in a configuration.
163 * @param conf the configuration to alter
164 * @param uri the new default filesystem uri
165 */
166 public static void setDefaultUri(Configuration conf, URI uri) {
167 conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
168 }
169
170 /** Set the default filesystem URI in a configuration.
171 * @param conf the configuration to alter
172 * @param uri the new default filesystem uri
173 */
174 public static void setDefaultUri(Configuration conf, String uri) {
175 setDefaultUri(conf, URI.create(fixName(uri)));
176 }
177
178 /** Called after a new FileSystem instance is constructed.
179 * @param name a uri whose authority section names the host, port, etc.
180 * for this FileSystem
181 * @param conf the configuration
182 */
183 public void initialize(URI name, Configuration conf) throws IOException {
184 statistics = getStatistics(name.getScheme(), getClass());
185 }
186
187 /** Returns a URI whose scheme and authority identify this FileSystem.*/
188 public abstract URI getUri();
189
190 /**
191 * Resolve the uri's hostname and add the default port if not in the uri
192 * @return URI
193 * @see NetUtils#getCanonicalUri(URI, int)
194 */
195 protected URI getCanonicalUri() {
196 return NetUtils.getCanonicalUri(getUri(), getDefaultPort());
197 }
198
199 /**
200 * Get the default port for this file system.
201 * @return the default port or 0 if there isn't one
202 */
203 protected int getDefaultPort() {
204 return 0;
205 }
206
207 /**
208 * Get a canonical service name for this file system. The token cache is
209 * the only user of this value, and uses it to lookup this filesystem's
210 * service tokens. The token cache will not attempt to acquire tokens if the
211 * service is null.
212 * @return a service string that uniquely identifies this file system, null
213 * if the filesystem does not implement tokens
214 * @see SecurityUtil#buildDTServiceName(URI, int)
215 */
216 public String getCanonicalServiceName() {
217 return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort());
218 }
219
220 /** @deprecated call #getUri() instead.*/
221 @Deprecated
222 public String getName() { return getUri().toString(); }
223
224 /** @deprecated call #get(URI,Configuration) instead. */
225 @Deprecated
226 public static FileSystem getNamed(String name, Configuration conf)
227 throws IOException {
228 return get(URI.create(fixName(name)), conf);
229 }
230
231 /** Update old-format filesystem names, for back-compatibility. This should
232 * eventually be replaced with a checkName() method that throws an exception
233 * for old-format names. */
234 private static String fixName(String name) {
235 // convert old-format name to new-format name
236 if (name.equals("local")) { // "local" is now "file:///".
237 LOG.warn("\"local\" is a deprecated filesystem name."
238 +" Use \"file:///\" instead.");
239 name = "file:///";
240 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
241 LOG.warn("\""+name+"\" is a deprecated filesystem name."
242 +" Use \"hdfs://"+name+"/\" instead.");
243 name = "hdfs://"+name;
244 }
245 return name;
246 }
247
248 /**
249 * Get the local file system.
250 * @param conf the configuration to configure the file system with
251 * @return a LocalFileSystem
252 */
253 public static LocalFileSystem getLocal(Configuration conf)
254 throws IOException {
255 return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
256 }
257
258 /** Returns the FileSystem for this URI's scheme and authority. The scheme
259 * of the URI determines a configuration property name,
260 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
261 * The entire URI is passed to the FileSystem instance's initialize method.
262 */
263 public static FileSystem get(URI uri, Configuration conf) throws IOException {
264 String scheme = uri.getScheme();
265 String authority = uri.getAuthority();
266
267 if (scheme == null) { // no scheme: use default FS
268 return get(conf);
269 }
270
271 if (authority == null) { // no authority
272 URI defaultUri = getDefaultUri(conf);
273 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
274 && defaultUri.getAuthority() != null) { // & default has authority
275 return get(defaultUri, conf); // return default
276 }
277 }
278
279 String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
280 if (conf.getBoolean(disableCacheName, false)) {
281 return createFileSystem(uri, conf);
282 }
283
284 return CACHE.get(uri, conf);
285 }
286
287 /**
288 * Returns the FileSystem for this URI's scheme and authority and the
289 * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
290 * @param uri of the filesystem
291 * @param conf the configuration to use
292 * @param user to perform the get as
293 * @return filesystem instance
294 * @throws IOException
295 * @throws InterruptedException
296 */
297 public static FileSystem newInstance(final URI uri, final Configuration conf,
298 final String user) throws IOException, InterruptedException {
299 UserGroupInformation ugi;
300 if (user == null) {
301 ugi = UserGroupInformation.getCurrentUser();
302 } else {
303 ugi = UserGroupInformation.createRemoteUser(user);
304 }
305 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
306 public FileSystem run() throws IOException {
307 return newInstance(uri,conf);
308 }
309 });
310 }
311 /** Returns the FileSystem for this URI's scheme and authority. The scheme
312 * of the URI determines a configuration property name,
313 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
314 * The entire URI is passed to the FileSystem instance's initialize method.
315 * This always returns a new FileSystem object.
316 */
317 public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
318 String scheme = uri.getScheme();
319 String authority = uri.getAuthority();
320
321 if (scheme == null) { // no scheme: use default FS
322 return newInstance(conf);
323 }
324
325 if (authority == null) { // no authority
326 URI defaultUri = getDefaultUri(conf);
327 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
328 && defaultUri.getAuthority() != null) { // & default has authority
329 return newInstance(defaultUri, conf); // return default
330 }
331 }
332 return CACHE.getUnique(uri, conf);
333 }
334
335 /** Returns a unique configured filesystem implementation.
336 * This always returns a new FileSystem object.
337 * @param conf the configuration to use
338 */
339 public static FileSystem newInstance(Configuration conf) throws IOException {
340 return newInstance(getDefaultUri(conf), conf);
341 }
342
343 /**
344 * Get a unique local file system object
345 * @param conf the configuration to configure the file system with
346 * @return a LocalFileSystem
347 * This always returns a new FileSystem object.
348 */
349 public static LocalFileSystem newInstanceLocal(Configuration conf)
350 throws IOException {
351 return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
352 }
353
354 /**
355 * Close all cached filesystems. Be sure those filesystems are not
356 * used anymore.
357 *
358 * @throws IOException
359 */
360 public static void closeAll() throws IOException {
361 CACHE.closeAll();
362 }
363
364 /**
365 * Close all cached filesystems for a given UGI. Be sure those filesystems
366 * are not used anymore.
367 * @param ugi user group info to close
368 * @throws IOException
369 */
370 public static void closeAllForUGI(UserGroupInformation ugi)
371 throws IOException {
372 CACHE.closeAll(ugi);
373 }
374
375 /**
376 * Make sure that a path specifies a FileSystem.
377 * @param path to use
378 */
379 public Path makeQualified(Path path) {
380 checkPath(path);
381 return path.makeQualified(this.getUri(), this.getWorkingDirectory());
382 }
383
384 /**
385 * Deprecated - use @link {@link #getDelegationTokens(String)}
386 * Get a new delegation token for this file system.
387 * @param renewer the account name that is allowed to renew the token.
388 * @return a new delegation token
389 * @throws IOException
390 */
391 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
392 @Deprecated
393 public Token<?> getDelegationToken(String renewer) throws IOException {
394 return null;
395 }
396
397 /**
398 * Get one or more delegation tokens associated with the filesystem. Normally
399 * a file system returns a single delegation token. A file system that manages
400 * multiple file systems underneath, could return set of delegation tokens for
401 * all the file systems it manages.
402 *
403 * @param renewer the account name that is allowed to renew the token.
404 * @return list of new delegation tokens
405 * If delegation tokens not supported then return a list of size zero.
406 * @throws IOException
407 */
408 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
409 public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
410 return new ArrayList<Token<?>>(0);
411 }
412
413 /**
414 * @see #getDelegationTokens(String)
415 * This is similar to getDelegationTokens, with the added restriction that if
416 * a token is already present in the passed Credentials object - that token
417 * is returned instead of a new delegation token.
418 *
419 * If the token is found to be cached in the Credentials object, this API does
420 * not verify the token validity or the passed in renewer.
421 *
422 *
423 * @param renewer the account name that is allowed to renew the token.
424 * @param credentials a Credentials object containing already knowing
425 * delegationTokens.
426 * @return a list of delegation tokens.
427 * @throws IOException
428 */
429 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
430 public List<Token<?>> getDelegationTokens(String renewer,
431 Credentials credentials) throws IOException {
432 List<Token<?>> allTokens = getDelegationTokens(renewer);
433 List<Token<?>> newTokens = new ArrayList<Token<?>>();
434 if (allTokens != null) {
435 for (Token<?> token : allTokens) {
436 Token<?> knownToken = credentials.getToken(token.getService());
437 if (knownToken == null) {
438 newTokens.add(token);
439 } else {
440 newTokens.add(knownToken);
441 }
442 }
443 }
444 return newTokens;
445 }
446
447 /** create a file with the provided permission
448 * The permission of the file is set to be the provided permission as in
449 * setPermission, not permission&~umask
450 *
451 * It is implemented using two RPCs. It is understood that it is inefficient,
452 * but the implementation is thread-safe. The other option is to change the
453 * value of umask in configuration to be 0, but it is not thread-safe.
454 *
455 * @param fs file system handle
456 * @param file the name of the file to be created
457 * @param permission the permission of the file
458 * @return an output stream
459 * @throws IOException
460 */
461 public static FSDataOutputStream create(FileSystem fs,
462 Path file, FsPermission permission) throws IOException {
463 // create the file with default permission
464 FSDataOutputStream out = fs.create(file);
465 // set its permission to the supplied one
466 fs.setPermission(file, permission);
467 return out;
468 }
469
470 /** create a directory with the provided permission
471 * The permission of the directory is set to be the provided permission as in
472 * setPermission, not permission&~umask
473 *
474 * @see #create(FileSystem, Path, FsPermission)
475 *
476 * @param fs file system handle
477 * @param dir the name of the directory to be created
478 * @param permission the permission of the directory
479 * @return true if the directory creation succeeds; false otherwise
480 * @throws IOException
481 */
482 public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
483 throws IOException {
484 // create the directory using the default permission
485 boolean result = fs.mkdirs(dir);
486 // set its permission to be the supplied one
487 fs.setPermission(dir, permission);
488 return result;
489 }
490
491 ///////////////////////////////////////////////////////////////
492 // FileSystem
493 ///////////////////////////////////////////////////////////////
494
495 protected FileSystem() {
496 super(null);
497 }
498
499 /**
500 * Check that a Path belongs to this FileSystem.
501 * @param path to check
502 */
503 protected void checkPath(Path path) {
504 URI uri = path.toUri();
505 String thatScheme = uri.getScheme();
506 if (thatScheme == null) // fs is relative
507 return;
508 URI thisUri = getCanonicalUri();
509 String thisScheme = thisUri.getScheme();
510 //authority and scheme are not case sensitive
511 if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
512 String thisAuthority = thisUri.getAuthority();
513 String thatAuthority = uri.getAuthority();
514 if (thatAuthority == null && // path's authority is null
515 thisAuthority != null) { // fs has an authority
516 URI defaultUri = getDefaultUri(getConf());
517 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) {
518 uri = defaultUri; // schemes match, so use this uri instead
519 } else {
520 uri = null; // can't determine auth of the path
521 }
522 }
523 if (uri != null) {
524 // canonicalize uri before comparing with this fs
525 uri = NetUtils.getCanonicalUri(uri, getDefaultPort());
526 thatAuthority = uri.getAuthority();
527 if (thisAuthority == thatAuthority || // authorities match
528 (thisAuthority != null &&
529 thisAuthority.equalsIgnoreCase(thatAuthority)))
530 return;
531 }
532 }
533 throw new IllegalArgumentException("Wrong FS: "+path+
534 ", expected: "+this.getUri());
535 }
536
537 /**
538 * Return an array containing hostnames, offset and size of
539 * portions of the given file. For a nonexistent
540 * file or regions, null will be returned.
541 *
542 * This call is most helpful with DFS, where it returns
543 * hostnames of machines that contain the given file.
544 *
545 * The FileSystem will simply return an elt containing 'localhost'.
546 *
547 * @param file FilesStatus to get data from
548 * @param start offset into the given file
549 * @param len length for which to get locations for
550 */
551 public BlockLocation[] getFileBlockLocations(FileStatus file,
552 long start, long len) throws IOException {
553 if (file == null) {
554 return null;
555 }
556
557 if (start < 0 || len < 0) {
558 throw new IllegalArgumentException("Invalid start or len parameter");
559 }
560
561 if (file.getLen() < start) {
562 return new BlockLocation[0];
563
564 }
565 String[] name = { "localhost:50010" };
566 String[] host = { "localhost" };
567 return new BlockLocation[] {
568 new BlockLocation(name, host, 0, file.getLen()) };
569 }
570
571
572 /**
573 * Return an array containing hostnames, offset and size of
574 * portions of the given file. For a nonexistent
575 * file or regions, null will be returned.
576 *
577 * This call is most helpful with DFS, where it returns
578 * hostnames of machines that contain the given file.
579 *
580 * The FileSystem will simply return an elt containing 'localhost'.
581 *
582 * @param p path of file to get locations for
583 * @param start offset into the given file
584 * @param len length for which to get locations for
585 */
586 public BlockLocation[] getFileBlockLocations(Path p,
587 long start, long len) throws IOException {
588 if (p == null) {
589 throw new NullPointerException();
590 }
591 FileStatus file = getFileStatus(p);
592 return getFileBlockLocations(file, start, len);
593 }
594
595 /**
596 * Return a set of server default configuration values
597 * @return server default configuration values
598 * @throws IOException
599 */
600 public FsServerDefaults getServerDefaults() throws IOException {
601 Configuration conf = getConf();
602 return new FsServerDefaults(getDefaultBlockSize(),
603 conf.getInt("io.bytes.per.checksum", 512),
604 64 * 1024,
605 getDefaultReplication(),
606 conf.getInt("io.file.buffer.size", 4096));
607 }
608
609 /**
610 * Return the fully-qualified path of path f resolving the path
611 * through any symlinks or mount point
612 * @param p path to be resolved
613 * @return fully qualified path
614 * @throws FileNotFoundException
615 */
616 public Path resolvePath(final Path p) throws IOException {
617 checkPath(p);
618 return getFileStatus(p).getPath();
619 }
620
621 /**
622 * Opens an FSDataInputStream at the indicated Path.
623 * @param f the file name to open
624 * @param bufferSize the size of the buffer to be used.
625 */
626 public abstract FSDataInputStream open(Path f, int bufferSize)
627 throws IOException;
628
629 /**
630 * Opens an FSDataInputStream at the indicated Path.
631 * @param f the file to open
632 */
633 public FSDataInputStream open(Path f) throws IOException {
634 return open(f, getConf().getInt("io.file.buffer.size", 4096));
635 }
636
637 /**
638 * Create an FSDataOutputStream at the indicated Path.
639 * Files are overwritten by default.
640 * @param f the file to create
641 */
642 public FSDataOutputStream create(Path f) throws IOException {
643 return create(f, true);
644 }
645
646 /**
647 * Create an FSDataOutputStream at the indicated Path.
648 * @param f the file to create
649 * @param overwrite if a file with this name already exists, then if true,
650 * the file will be overwritten, and if false an exception will be thrown.
651 */
652 public FSDataOutputStream create(Path f, boolean overwrite)
653 throws IOException {
654 return create(f, overwrite,
655 getConf().getInt("io.file.buffer.size", 4096),
656 getDefaultReplication(),
657 getDefaultBlockSize());
658 }
659
660 /**
661 * Create an FSDataOutputStream at the indicated Path with write-progress
662 * reporting.
663 * Files are overwritten by default.
664 * @param f the file to create
665 * @param progress to report progress
666 */
667 public FSDataOutputStream create(Path f, Progressable progress)
668 throws IOException {
669 return create(f, true,
670 getConf().getInt("io.file.buffer.size", 4096),
671 getDefaultReplication(),
672 getDefaultBlockSize(), progress);
673 }
674
675 /**
676 * Create an FSDataOutputStream at the indicated Path.
677 * Files are overwritten by default.
678 * @param f the file to create
679 * @param replication the replication factor
680 */
681 public FSDataOutputStream create(Path f, short replication)
682 throws IOException {
683 return create(f, true,
684 getConf().getInt("io.file.buffer.size", 4096),
685 replication,
686 getDefaultBlockSize());
687 }
688
689 /**
690 * Create an FSDataOutputStream at the indicated Path with write-progress
691 * reporting.
692 * Files are overwritten by default.
693 * @param f the file to create
694 * @param replication the replication factor
695 * @param progress to report progress
696 */
697 public FSDataOutputStream create(Path f, short replication,
698 Progressable progress) throws IOException {
699 return create(f, true,
700 getConf().getInt("io.file.buffer.size", 4096),
701 replication,
702 getDefaultBlockSize(), progress);
703 }
704
705
706 /**
707 * Create an FSDataOutputStream at the indicated Path.
708 * @param f the file name to create
709 * @param overwrite if a file with this name already exists, then if true,
710 * the file will be overwritten, and if false an error will be thrown.
711 * @param bufferSize the size of the buffer to be used.
712 */
713 public FSDataOutputStream create(Path f,
714 boolean overwrite,
715 int bufferSize
716 ) throws IOException {
717 return create(f, overwrite, bufferSize,
718 getDefaultReplication(),
719 getDefaultBlockSize());
720 }
721
722 /**
723 * Create an FSDataOutputStream at the indicated Path with write-progress
724 * reporting.
725 * @param f the path of the file to open
726 * @param overwrite if a file with this name already exists, then if true,
727 * the file will be overwritten, and if false an error will be thrown.
728 * @param bufferSize the size of the buffer to be used.
729 */
730 public FSDataOutputStream create(Path f,
731 boolean overwrite,
732 int bufferSize,
733 Progressable progress
734 ) throws IOException {
735 return create(f, overwrite, bufferSize,
736 getDefaultReplication(),
737 getDefaultBlockSize(), progress);
738 }
739
740
741 /**
742 * Create an FSDataOutputStream at the indicated Path.
743 * @param f the file name to open
744 * @param overwrite if a file with this name already exists, then if true,
745 * the file will be overwritten, and if false an error will be thrown.
746 * @param bufferSize the size of the buffer to be used.
747 * @param replication required block replication for the file.
748 */
749 public FSDataOutputStream create(Path f,
750 boolean overwrite,
751 int bufferSize,
752 short replication,
753 long blockSize
754 ) throws IOException {
755 return create(f, overwrite, bufferSize, replication, blockSize, null);
756 }
757
758 /**
759 * Create an FSDataOutputStream at the indicated Path with write-progress
760 * reporting.
761 * @param f the file name to open
762 * @param overwrite if a file with this name already exists, then if true,
763 * the file will be overwritten, and if false an error will be thrown.
764 * @param bufferSize the size of the buffer to be used.
765 * @param replication required block replication for the file.
766 */
767 public FSDataOutputStream create(Path f,
768 boolean overwrite,
769 int bufferSize,
770 short replication,
771 long blockSize,
772 Progressable progress
773 ) throws IOException {
774 return this.create(f, FsPermission.getDefault().applyUMask(
775 FsPermission.getUMask(getConf())), overwrite, bufferSize,
776 replication, blockSize, progress);
777 }
778
779 /**
780 * Create an FSDataOutputStream at the indicated Path with write-progress
781 * reporting.
782 * @param f the file name to open
783 * @param permission
784 * @param overwrite if a file with this name already exists, then if true,
785 * the file will be overwritten, and if false an error will be thrown.
786 * @param bufferSize the size of the buffer to be used.
787 * @param replication required block replication for the file.
788 * @param blockSize
789 * @param progress
790 * @throws IOException
791 * @see #setPermission(Path, FsPermission)
792 */
793 public abstract FSDataOutputStream create(Path f,
794 FsPermission permission,
795 boolean overwrite,
796 int bufferSize,
797 short replication,
798 long blockSize,
799 Progressable progress) throws IOException;
800
801
802 /*.
803 * This create has been added to support the FileContext that processes
804 * the permission
805 * with umask before calling this method.
806 * This a temporary method added to support the transition from FileSystem
807 * to FileContext for user applications.
808 */
809 @Deprecated
810 protected FSDataOutputStream primitiveCreate(Path f,
811 FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
812 short replication, long blockSize, Progressable progress,
813 int bytesPerChecksum) throws IOException {
814
815 boolean pathExists = exists(f);
816 CreateFlag.validate(f, pathExists, flag);
817
818 // Default impl assumes that permissions do not matter and
819 // nor does the bytesPerChecksum hence
820 // calling the regular create is good enough.
821 // FSs that implement permissions should override this.
822
823 if (pathExists && flag.contains(CreateFlag.APPEND)) {
824 return append(f, bufferSize, progress);
825 }
826
827 return this.create(f, absolutePermission,
828 flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
829 blockSize, progress);
830 }
831
832 /**
833 * This version of the mkdirs method assumes that the permission is absolute.
834 * It has been added to support the FileContext that processes the permission
835 * with umask before calling this method.
836 * This a temporary method added to support the transition from FileSystem
837 * to FileContext for user applications.
838 */
839 @Deprecated
840 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
841 throws IOException {
842 // Default impl is to assume that permissions do not matter and hence
843 // calling the regular mkdirs is good enough.
844 // FSs that implement permissions should override this.
845 return this.mkdirs(f, absolutePermission);
846 }
847
848
849 /**
850 * This version of the mkdirs method assumes that the permission is absolute.
851 * It has been added to support the FileContext that processes the permission
852 * with umask before calling this method.
853 * This a temporary method added to support the transition from FileSystem
854 * to FileContext for user applications.
855 */
856 @Deprecated
857 protected void primitiveMkdir(Path f, FsPermission absolutePermission,
858 boolean createParent)
859 throws IOException {
860
861 if (!createParent) { // parent must exist.
862 // since the this.mkdirs makes parent dirs automatically
863 // we must throw exception if parent does not exist.
864 final FileStatus stat = getFileStatus(f.getParent());
865 if (stat == null) {
866 throw new FileNotFoundException("Missing parent:" + f);
867 }
868 if (!stat.isDirectory()) {
869 throw new ParentNotDirectoryException("parent is not a dir");
870 }
871 // parent does exist - go ahead with mkdir of leaf
872 }
873 // Default impl is to assume that permissions do not matter and hence
874 // calling the regular mkdirs is good enough.
875 // FSs that implement permissions should override this.
876 if (!this.mkdirs(f, absolutePermission)) {
877 throw new IOException("mkdir of "+ f + " failed");
878 }
879 }
880
881 /**
882 * Opens an FSDataOutputStream at the indicated Path with write-progress
883 * reporting. Same as create(), except fails if parent directory doesn't
884 * already exist.
885 * @param f the file name to open
886 * @param overwrite if a file with this name already exists, then if true,
887 * the file will be overwritten, and if false an error will be thrown.
888 * @param bufferSize the size of the buffer to be used.
889 * @param replication required block replication for the file.
890 * @param blockSize
891 * @param progress
892 * @throws IOException
893 * @see #setPermission(Path, FsPermission)
894 * @deprecated API only for 0.20-append
895 */
896 @Deprecated
897 public FSDataOutputStream createNonRecursive(Path f,
898 boolean overwrite,
899 int bufferSize, short replication, long blockSize,
900 Progressable progress) throws IOException {
901 return this.createNonRecursive(f, FsPermission.getDefault(),
902 overwrite, bufferSize, replication, blockSize, progress);
903 }
904
905 /**
906 * Opens an FSDataOutputStream at the indicated Path with write-progress
907 * reporting. Same as create(), except fails if parent directory doesn't
908 * already exist.
909 * @param f the file name to open
910 * @param permission
911 * @param overwrite if a file with this name already exists, then if true,
912 * the file will be overwritten, and if false an error will be thrown.
913 * @param bufferSize the size of the buffer to be used.
914 * @param replication required block replication for the file.
915 * @param blockSize
916 * @param progress
917 * @throws IOException
918 * @see #setPermission(Path, FsPermission)
919 * @deprecated API only for 0.20-append
920 */
921 @Deprecated
922 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
923 boolean overwrite, int bufferSize, short replication, long blockSize,
924 Progressable progress) throws IOException {
925 throw new IOException("createNonRecursive unsupported for this filesystem "
926 + this.getClass());
927 }
928
929 /**
930 * Creates the given Path as a brand-new zero-length file. If
931 * create fails, or if it already existed, return false.
932 *
933 * @param f path to use for create
934 */
935 public boolean createNewFile(Path f) throws IOException {
936 if (exists(f)) {
937 return false;
938 } else {
939 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
940 return true;
941 }
942 }
943
944 /**
945 * Append to an existing file (optional operation).
946 * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
947 * @param f the existing file to be appended.
948 * @throws IOException
949 */
950 public FSDataOutputStream append(Path f) throws IOException {
951 return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
952 }
953 /**
954 * Append to an existing file (optional operation).
955 * Same as append(f, bufferSize, null).
956 * @param f the existing file to be appended.
957 * @param bufferSize the size of the buffer to be used.
958 * @throws IOException
959 */
960 public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
961 return append(f, bufferSize, null);
962 }
963
964 /**
965 * Append to an existing file (optional operation).
966 * @param f the existing file to be appended.
967 * @param bufferSize the size of the buffer to be used.
968 * @param progress for reporting progress if it is not null.
969 * @throws IOException
970 */
971 public abstract FSDataOutputStream append(Path f, int bufferSize,
972 Progressable progress) throws IOException;
973
974 /**
975 * Get replication.
976 *
977 * @deprecated Use getFileStatus() instead
978 * @param src file name
979 * @return file replication
980 * @throws IOException
981 */
982 @Deprecated
983 public short getReplication(Path src) throws IOException {
984 return getFileStatus(src).getReplication();
985 }
986
987 /**
988 * Set replication for an existing file.
989 *
990 * @param src file name
991 * @param replication new replication
992 * @throws IOException
993 * @return true if successful;
994 * false if file does not exist or is a directory
995 */
996 public boolean setReplication(Path src, short replication)
997 throws IOException {
998 return true;
999 }
1000
1001 /**
1002 * Renames Path src to Path dst. Can take place on local fs
1003 * or remote DFS.
1004 * @param src path to be renamed
1005 * @param dst new path after rename
1006 * @throws IOException on failure
1007 * @return true if rename is successful
1008 */
1009 public abstract boolean rename(Path src, Path dst) throws IOException;
1010
1011 /**
1012 * Renames Path src to Path dst
1013 * <ul>
1014 * <li
1015 * <li>Fails if src is a file and dst is a directory.
1016 * <li>Fails if src is a directory and dst is a file.
1017 * <li>Fails if the parent of dst does not exist or is a file.
1018 * </ul>
1019 * <p>
1020 * If OVERWRITE option is not passed as an argument, rename fails
1021 * if the dst already exists.
1022 * <p>
1023 * If OVERWRITE option is passed as an argument, rename overwrites
1024 * the dst if it is a file or an empty directory. Rename fails if dst is
1025 * a non-empty directory.
1026 * <p>
1027 * Note that atomicity of rename is dependent on the file system
1028 * implementation. Please refer to the file system documentation for
1029 * details. This default implementation is non atomic.
1030 * <p>
1031 * This method is deprecated since it is a temporary method added to
1032 * support the transition from FileSystem to FileContext for user
1033 * applications.
1034 *
1035 * @param src path to be renamed
1036 * @param dst new path after rename
1037 * @throws IOException on failure
1038 */
1039 @Deprecated
1040 protected void rename(final Path src, final Path dst,
1041 final Rename... options) throws IOException {
1042 // Default implementation
1043 final FileStatus srcStatus = getFileStatus(src);
1044 if (srcStatus == null) {
1045 throw new FileNotFoundException("rename source " + src + " not found.");
1046 }
1047
1048 boolean overwrite = false;
1049 if (null != options) {
1050 for (Rename option : options) {
1051 if (option == Rename.OVERWRITE) {
1052 overwrite = true;
1053 }
1054 }
1055 }
1056
1057 FileStatus dstStatus;
1058 try {
1059 dstStatus = getFileStatus(dst);
1060 } catch (IOException e) {
1061 dstStatus = null;
1062 }
1063 if (dstStatus != null) {
1064 if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
1065 throw new IOException("Source " + src + " Destination " + dst
1066 + " both should be either file or directory");
1067 }
1068 if (!overwrite) {
1069 throw new FileAlreadyExistsException("rename destination " + dst
1070 + " already exists.");
1071 }
1072 // Delete the destination that is a file or an empty directory
1073 if (dstStatus.isDirectory()) {
1074 FileStatus[] list = listStatus(dst);
1075 if (list != null && list.length != 0) {
1076 throw new IOException(
1077 "rename cannot overwrite non empty destination directory " + dst);
1078 }
1079 }
1080 delete(dst, false);
1081 } else {
1082 final Path parent = dst.getParent();
1083 final FileStatus parentStatus = getFileStatus(parent);
1084 if (parentStatus == null) {
1085 throw new FileNotFoundException("rename destination parent " + parent
1086 + " not found.");
1087 }
1088 if (!parentStatus.isDirectory()) {
1089 throw new ParentNotDirectoryException("rename destination parent " + parent
1090 + " is a file.");
1091 }
1092 }
1093 if (!rename(src, dst)) {
1094 throw new IOException("rename from " + src + " to " + dst + " failed.");
1095 }
1096 }
1097
1098 /**
1099 * Delete a file
1100 * @deprecated Use {@link #delete(Path, boolean)} instead.
1101 */
1102 @Deprecated
1103 public boolean delete(Path f) throws IOException {
1104 return delete(f, true);
1105 }
1106
1107 /** Delete a file.
1108 *
1109 * @param f the path to delete.
1110 * @param recursive if path is a directory and set to
1111 * true, the directory is deleted else throws an exception. In
1112 * case of a file the recursive can be set to either true or false.
1113 * @return true if delete is successful else false.
1114 * @throws IOException
1115 */
1116 public abstract boolean delete(Path f, boolean recursive) throws IOException;
1117
1118 /**
1119 * Mark a path to be deleted when FileSystem is closed.
1120 * When the JVM shuts down,
1121 * all FileSystem objects will be closed automatically.
1122 * Then,
1123 * the marked path will be deleted as a result of closing the FileSystem.
1124 *
1125 * The path has to exist in the file system.
1126 *
1127 * @param f the path to delete.
1128 * @return true if deleteOnExit is successful, otherwise false.
1129 * @throws IOException
1130 */
1131 public boolean deleteOnExit(Path f) throws IOException {
1132 if (!exists(f)) {
1133 return false;
1134 }
1135 synchronized (deleteOnExit) {
1136 deleteOnExit.add(f);
1137 }
1138 return true;
1139 }
1140
1141 /**
1142 * Delete all files that were marked as delete-on-exit. This recursively
1143 * deletes all files in the specified paths.
1144 */
1145 protected void processDeleteOnExit() {
1146 synchronized (deleteOnExit) {
1147 for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
1148 Path path = iter.next();
1149 try {
1150 delete(path, true);
1151 }
1152 catch (IOException e) {
1153 LOG.info("Ignoring failure to deleteOnExit for path " + path);
1154 }
1155 iter.remove();
1156 }
1157 }
1158 }
1159
1160 /** Check if exists.
1161 * @param f source file
1162 */
1163 public boolean exists(Path f) throws IOException {
1164 try {
1165 return getFileStatus(f) != null;
1166 } catch (FileNotFoundException e) {
1167 return false;
1168 }
1169 }
1170
1171 /** True iff the named path is a directory.
1172 * Note: Avoid using this method. Instead reuse the FileStatus
1173 * returned by getFileStatus() or listStatus() methods.
1174 * @param f path to check
1175 */
1176 public boolean isDirectory(Path f) throws IOException {
1177 try {
1178 return getFileStatus(f).isDirectory();
1179 } catch (FileNotFoundException e) {
1180 return false; // f does not exist
1181 }
1182 }
1183
1184 /** True iff the named path is a regular file.
1185 * Note: Avoid using this method. Instead reuse the FileStatus
1186 * returned by getFileStatus() or listStatus() methods.
1187 * @param f path to check
1188 */
1189 public boolean isFile(Path f) throws IOException {
1190 try {
1191 return getFileStatus(f).isFile();
1192 } catch (FileNotFoundException e) {
1193 return false; // f does not exist
1194 }
1195 }
1196
1197 /** The number of bytes in a file. */
1198 /** @deprecated Use getFileStatus() instead */
1199 @Deprecated
1200 public long getLength(Path f) throws IOException {
1201 return getFileStatus(f).getLen();
1202 }
1203
1204 /** Return the {@link ContentSummary} of a given {@link Path}.
1205 * @param f path to use
1206 */
1207 public ContentSummary getContentSummary(Path f) throws IOException {
1208 FileStatus status = getFileStatus(f);
1209 if (status.isFile()) {
1210 // f is a file
1211 return new ContentSummary(status.getLen(), 1, 0);
1212 }
1213 // f is a directory
1214 long[] summary = {0, 0, 1};
1215 for(FileStatus s : listStatus(f)) {
1216 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1217 new ContentSummary(s.getLen(), 1, 0);
1218 summary[0] += c.getLength();
1219 summary[1] += c.getFileCount();
1220 summary[2] += c.getDirectoryCount();
1221 }
1222 return new ContentSummary(summary[0], summary[1], summary[2]);
1223 }
1224
1225 final private static PathFilter DEFAULT_FILTER = new PathFilter() {
1226 public boolean accept(Path file) {
1227 return true;
1228 }
1229 };
1230
1231 /**
1232 * List the statuses of the files/directories in the given path if the path is
1233 * a directory.
1234 *
1235 * @param f given path
1236 * @return the statuses of the files/directories in the given patch
1237 * @throws FileNotFoundException when the path does not exist;
1238 * IOException see specific implementation
1239 */
1240 public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException,
1241 IOException;
1242
1243 /*
1244 * Filter files/directories in the given path using the user-supplied path
1245 * filter. Results are added to the given array <code>results</code>.
1246 */
1247 private void listStatus(ArrayList<FileStatus> results, Path f,
1248 PathFilter filter) throws FileNotFoundException, IOException {
1249 FileStatus listing[] = listStatus(f);
1250 if (listing == null) {
1251 throw new IOException("Error accessing " + f);
1252 }
1253
1254 for (int i = 0; i < listing.length; i++) {
1255 if (filter.accept(listing[i].getPath())) {
1256 results.add(listing[i]);
1257 }
1258 }
1259 }
1260
1261 /**
1262 * @return an iterator over the corrupt files under the given path
1263 * (may contain duplicates if a file has more than one corrupt block)
1264 * @throws IOException
1265 */
1266 public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1267 throws IOException {
1268 throw new UnsupportedOperationException(getClass().getCanonicalName() +
1269 " does not support" +
1270 " listCorruptFileBlocks");
1271 }
1272
1273 /**
1274 * Filter files/directories in the given path using the user-supplied path
1275 * filter.
1276 *
1277 * @param f
1278 * a path name
1279 * @param filter
1280 * the user-supplied path filter
1281 * @return an array of FileStatus objects for the files under the given path
1282 * after applying the filter
1283 * @throws FileNotFoundException when the path does not exist;
1284 * IOException see specific implementation
1285 */
1286 public FileStatus[] listStatus(Path f, PathFilter filter)
1287 throws FileNotFoundException, IOException {
1288 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1289 listStatus(results, f, filter);
1290 return results.toArray(new FileStatus[results.size()]);
1291 }
1292
1293 /**
1294 * Filter files/directories in the given list of paths using default
1295 * path filter.
1296 *
1297 * @param files
1298 * a list of paths
1299 * @return a list of statuses for the files under the given paths after
1300 * applying the filter default Path filter
1301 * @throws FileNotFoundException when the path does not exist;
1302 * IOException see specific implementation
1303 */
1304 public FileStatus[] listStatus(Path[] files)
1305 throws FileNotFoundException, IOException {
1306 return listStatus(files, DEFAULT_FILTER);
1307 }
1308
1309 /**
1310 * Filter files/directories in the given list of paths using user-supplied
1311 * path filter.
1312 *
1313 * @param files
1314 * a list of paths
1315 * @param filter
1316 * the user-supplied path filter
1317 * @return a list of statuses for the files under the given paths after
1318 * applying the filter
1319 * @throws FileNotFoundException when the path does not exist;
1320 * IOException see specific implementation
1321 */
1322 public FileStatus[] listStatus(Path[] files, PathFilter filter)
1323 throws FileNotFoundException, IOException {
1324 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1325 for (int i = 0; i < files.length; i++) {
1326 listStatus(results, files[i], filter);
1327 }
1328 return results.toArray(new FileStatus[results.size()]);
1329 }
1330
1331 /**
1332 * <p>Return all the files that match filePattern and are not checksum
1333 * files. Results are sorted by their names.
1334 *
1335 * <p>
1336 * A filename pattern is composed of <i>regular</i> characters and
1337 * <i>special pattern matching</i> characters, which are:
1338 *
1339 * <dl>
1340 * <dd>
1341 * <dl>
1342 * <p>
1343 * <dt> <tt> ? </tt>
1344 * <dd> Matches any single character.
1345 *
1346 * <p>
1347 * <dt> <tt> * </tt>
1348 * <dd> Matches zero or more characters.
1349 *
1350 * <p>
1351 * <dt> <tt> [<i>abc</i>] </tt>
1352 * <dd> Matches a single character from character set
1353 * <tt>{<i>a,b,c</i>}</tt>.
1354 *
1355 * <p>
1356 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1357 * <dd> Matches a single character from the character range
1358 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be
1359 * lexicographically less than or equal to character <tt><i>b</i></tt>.
1360 *
1361 * <p>
1362 * <dt> <tt> [^<i>a</i>] </tt>
1363 * <dd> Matches a single character that is not from character set or range
1364 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
1365 * immediately to the right of the opening bracket.
1366 *
1367 * <p>
1368 * <dt> <tt> \<i>c</i> </tt>
1369 * <dd> Removes (escapes) any special meaning of character <i>c</i>.
1370 *
1371 * <p>
1372 * <dt> <tt> {ab,cd} </tt>
1373 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1374 *
1375 * <p>
1376 * <dt> <tt> {ab,c{de,fh}} </tt>
1377 * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
1378 *
1379 * </dl>
1380 * </dd>
1381 * </dl>
1382 *
1383 * @param pathPattern a regular expression specifying a pth pattern
1384
1385 * @return an array of paths that match the path pattern
1386 * @throws IOException
1387 */
1388 public FileStatus[] globStatus(Path pathPattern) throws IOException {
1389 return globStatus(pathPattern, DEFAULT_FILTER);
1390 }
1391
1392 /**
1393 * Return an array of FileStatus objects whose path names match pathPattern
1394 * and is accepted by the user-supplied path filter. Results are sorted by
1395 * their path names.
1396 * Return null if pathPattern has no glob and the path does not exist.
1397 * Return an empty array if pathPattern has a glob and no path matches it.
1398 *
1399 * @param pathPattern
1400 * a regular expression specifying the path pattern
1401 * @param filter
1402 * a user-supplied path filter
1403 * @return an array of FileStatus objects
1404 * @throws IOException if any I/O error occurs when fetching file status
1405 */
1406 public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
1407 throws IOException {
1408 String filename = pathPattern.toUri().getPath();
1409 List<String> filePatterns = GlobExpander.expand(filename);
1410 if (filePatterns.size() == 1) {
1411 return globStatusInternal(pathPattern, filter);
1412 } else {
1413 List<FileStatus> results = new ArrayList<FileStatus>();
1414 for (String filePattern : filePatterns) {
1415 FileStatus[] files = globStatusInternal(new Path(filePattern), filter);
1416 for (FileStatus file : files) {
1417 results.add(file);
1418 }
1419 }
1420 return results.toArray(new FileStatus[results.size()]);
1421 }
1422 }
1423
1424 private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter)
1425 throws IOException {
1426 Path[] parents = new Path[1];
1427 int level = 0;
1428 String filename = pathPattern.toUri().getPath();
1429
1430 // path has only zero component
1431 if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
1432 return getFileStatus(new Path[]{pathPattern});
1433 }
1434
1435 // path has at least one component
1436 String[] components = filename.split(Path.SEPARATOR);
1437 // get the first component
1438 if (pathPattern.isAbsolute()) {
1439 parents[0] = new Path(Path.SEPARATOR);
1440 level = 1;
1441 } else {
1442 parents[0] = new Path(Path.CUR_DIR);
1443 }
1444
1445 // glob the paths that match the parent path, i.e., [0, components.length-1]
1446 boolean[] hasGlob = new boolean[]{false};
1447 Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob);
1448 FileStatus[] results;
1449 if (parentPaths == null || parentPaths.length == 0) {
1450 results = null;
1451 } else {
1452 // Now work on the last component of the path
1453 GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
1454 if (fp.hasPattern()) { // last component has a pattern
1455 // list parent directories and then glob the results
1456 results = listStatus(parentPaths, fp);
1457 hasGlob[0] = true;
1458 } else { // last component does not have a pattern
1459 // get all the path names
1460 ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
1461 for (int i = 0; i < parentPaths.length; i++) {
1462 parentPaths[i] = new Path(parentPaths[i],
1463 components[components.length - 1]);
1464 if (fp.accept(parentPaths[i])) {
1465 filteredPaths.add(parentPaths[i]);
1466 }
1467 }
1468 // get all their statuses
1469 results = getFileStatus(
1470 filteredPaths.toArray(new Path[filteredPaths.size()]));
1471 }
1472 }
1473
1474 // Decide if the pathPattern contains a glob or not
1475 if (results == null) {
1476 if (hasGlob[0]) {
1477 results = new FileStatus[0];
1478 }
1479 } else {
1480 if (results.length == 0 ) {
1481 if (!hasGlob[0]) {
1482 results = null;
1483 }
1484 } else {
1485 Arrays.sort(results);
1486 }
1487 }
1488 return results;
1489 }
1490
1491 /*
1492 * For a path of N components, return a list of paths that match the
1493 * components [<code>level</code>, <code>N-1</code>].
1494 */
1495 private Path[] globPathsLevel(Path[] parents, String[] filePattern,
1496 int level, boolean[] hasGlob) throws IOException {
1497 if (level == filePattern.length - 1)
1498 return parents;
1499 if (parents == null || parents.length == 0) {
1500 return null;
1501 }
1502 GlobFilter fp = new GlobFilter(filePattern[level]);
1503 if (fp.hasPattern()) {
1504 parents = FileUtil.stat2Paths(listStatus(parents, fp));
1505 hasGlob[0] = true;
1506 } else {
1507 for (int i = 0; i < parents.length; i++) {
1508 parents[i] = new Path(parents[i], filePattern[level]);
1509 }
1510 }
1511 return globPathsLevel(parents, filePattern, level + 1, hasGlob);
1512 }
1513
1514 /**
1515 * List the statuses of the files/directories in the given path if the path is
1516 * a directory.
1517 * Return the file's status and block locations If the path is a file.
1518 *
1519 * If a returned status is a file, it contains the file's block locations.
1520 *
1521 * @param f is the path
1522 *
1523 * @return an iterator that traverses statuses of the files/directories
1524 * in the given path
1525 *
1526 * @throws FileNotFoundException If <code>f</code> does not exist
1527 * @throws IOException If an I/O error occurred
1528 */
1529 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
1530 throws FileNotFoundException, IOException {
1531 return listLocatedStatus(f, DEFAULT_FILTER);
1532 }
1533
1534 /**
1535 * Listing a directory
1536 * The returned results include its block location if it is a file
1537 * The results are filtered by the given path filter
1538 * @param f a path
1539 * @param filter a path filter
1540 * @return an iterator that traverses statuses of the files/directories
1541 * in the given path
1542 * @throws FileNotFoundException if <code>f</code> does not exist
1543 * @throws IOException if any I/O error occurred
1544 */
1545 protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
1546 final PathFilter filter)
1547 throws FileNotFoundException, IOException {
1548 return new RemoteIterator<LocatedFileStatus>() {
1549 private final FileStatus[] stats = listStatus(f, filter);
1550 private int i = 0;
1551
1552 @Override
1553 public boolean hasNext() {
1554 return i<stats.length;
1555 }
1556
1557 @Override
1558 public LocatedFileStatus next() throws IOException {
1559 if (!hasNext()) {
1560 throw new NoSuchElementException("No more entry in " + f);
1561 }
1562 FileStatus result = stats[i++];
1563 BlockLocation[] locs = result.isFile() ?
1564 getFileBlockLocations(result.getPath(), 0, result.getLen()) :
1565 null;
1566 return new LocatedFileStatus(result, locs);
1567 }
1568 };
1569 }
1570
1571 /**
1572 * List the statuses and block locations of the files in the given path.
1573 *
1574 * If the path is a directory,
1575 * if recursive is false, returns files in the directory;
1576 * if recursive is true, return files in the subtree rooted at the path.
1577 * If the path is a file, return the file's status and block locations.
1578 *
1579 * @param f is the path
1580 * @param recursive if the subdirectories need to be traversed recursively
1581 *
1582 * @return an iterator that traverses statuses of the files
1583 *
1584 * @throws FileNotFoundException when the path does not exist;
1585 * IOException see specific implementation
1586 */
1587 public RemoteIterator<LocatedFileStatus> listFiles(
1588 final Path f, final boolean recursive)
1589 throws FileNotFoundException, IOException {
1590 return new RemoteIterator<LocatedFileStatus>() {
1591 private Stack<RemoteIterator<LocatedFileStatus>> itors =
1592 new Stack<RemoteIterator<LocatedFileStatus>>();
1593 private RemoteIterator<LocatedFileStatus> curItor =
1594 listLocatedStatus(f);
1595 private LocatedFileStatus curFile;
1596
1597 @Override
1598 public boolean hasNext() throws IOException {
1599 while (curFile == null) {
1600 if (curItor.hasNext()) {
1601 handleFileStat(curItor.next());
1602 } else if (!itors.empty()) {
1603 curItor = itors.pop();
1604 } else {
1605 return false;
1606 }
1607 }
1608 return true;
1609 }
1610
1611 /**
1612 * Process the input stat.
1613 * If it is a file, return the file stat.
1614 * If it is a directory, traverse the directory if recursive is true;
1615 * ignore it if recursive is false.
1616 * @param stat input status
1617 * @throws IOException if any IO error occurs
1618 */
1619 private void handleFileStat(LocatedFileStatus stat) throws IOException {
1620 if (stat.isFile()) { // file
1621 curFile = stat;
1622 } else if (recursive) { // directory
1623 itors.push(curItor);
1624 curItor = listLocatedStatus(stat.getPath());
1625 }
1626 }
1627
1628 @Override
1629 public LocatedFileStatus next() throws IOException {
1630 if (hasNext()) {
1631 LocatedFileStatus result = curFile;
1632 curFile = null;
1633 return result;
1634 }
1635 throw new java.util.NoSuchElementException("No more entry in " + f);
1636 }
1637 };
1638 }
1639
1640 /** Return the current user's home directory in this filesystem.
1641 * The default implementation returns "/user/$USER/".
1642 */
1643 public Path getHomeDirectory() {
1644 return this.makeQualified(
1645 new Path("/user/"+System.getProperty("user.name")));
1646 }
1647
1648
1649 /**
1650 * Set the current working directory for the given file system. All relative
1651 * paths will be resolved relative to it.
1652 *
1653 * @param new_dir
1654 */
1655 public abstract void setWorkingDirectory(Path new_dir);
1656
1657 /**
1658 * Get the current working directory for the given file system
1659 * @return the directory pathname
1660 */
1661 public abstract Path getWorkingDirectory();
1662
1663
1664 /**
1665 * Note: with the new FilesContext class, getWorkingDirectory()
1666 * will be removed.
1667 * The working directory is implemented in FilesContext.
1668 *
1669 * Some file systems like LocalFileSystem have an initial workingDir
1670 * that we use as the starting workingDir. For other file systems
1671 * like HDFS there is no built in notion of an inital workingDir.
1672 *
1673 * @return if there is built in notion of workingDir then it
1674 * is returned; else a null is returned.
1675 */
1676 protected Path getInitialWorkingDirectory() {
1677 return null;
1678 }
1679
1680 /**
1681 * Call {@link #mkdirs(Path, FsPermission)} with default permission.
1682 */
1683 public boolean mkdirs(Path f) throws IOException {
1684 return mkdirs(f, FsPermission.getDefault());
1685 }
1686
1687 /**
1688 * Make the given file and all non-existent parents into
1689 * directories. Has the semantics of Unix 'mkdir -p'.
1690 * Existence of the directory hierarchy is not an error.
1691 * @param f path to create
1692 * @param permission to apply to f
1693 */
1694 public abstract boolean mkdirs(Path f, FsPermission permission
1695 ) throws IOException;
1696
1697 /**
1698 * The src file is on the local disk. Add it to FS at
1699 * the given dst name and the source is kept intact afterwards
1700 * @param src path
1701 * @param dst path
1702 */
1703 public void copyFromLocalFile(Path src, Path dst)
1704 throws IOException {
1705 copyFromLocalFile(false, src, dst);
1706 }
1707
1708 /**
1709 * The src files is on the local disk. Add it to FS at
1710 * the given dst name, removing the source afterwards.
1711 * @param srcs path
1712 * @param dst path
1713 */
1714 public void moveFromLocalFile(Path[] srcs, Path dst)
1715 throws IOException {
1716 copyFromLocalFile(true, true, srcs, dst);
1717 }
1718
1719 /**
1720 * The src file is on the local disk. Add it to FS at
1721 * the given dst name, removing the source afterwards.
1722 * @param src path
1723 * @param dst path
1724 */
1725 public void moveFromLocalFile(Path src, Path dst)
1726 throws IOException {
1727 copyFromLocalFile(true, src, dst);
1728 }
1729
1730 /**
1731 * The src file is on the local disk. Add it to FS at
1732 * the given dst name.
1733 * delSrc indicates if the source should be removed
1734 * @param delSrc whether to delete the src
1735 * @param src path
1736 * @param dst path
1737 */
1738 public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
1739 throws IOException {
1740 copyFromLocalFile(delSrc, true, src, dst);
1741 }
1742
1743 /**
1744 * The src files are on the local disk. Add it to FS at
1745 * the given dst name.
1746 * delSrc indicates if the source should be removed
1747 * @param delSrc whether to delete the src
1748 * @param overwrite whether to overwrite an existing file
1749 * @param srcs array of paths which are source
1750 * @param dst path
1751 */
1752 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
1753 Path[] srcs, Path dst)
1754 throws IOException {
1755 Configuration conf = getConf();
1756 FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
1757 }
1758
1759 /**
1760 * The src file is on the local disk. Add it to FS at
1761 * the given dst name.
1762 * delSrc indicates if the source should be removed
1763 * @param delSrc whether to delete the src
1764 * @param overwrite whether to overwrite an existing file
1765 * @param src path
1766 * @param dst path
1767 */
1768 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
1769 Path src, Path dst)
1770 throws IOException {
1771 Configuration conf = getConf();
1772 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
1773 }
1774
1775 /**
1776 * The src file is under FS, and the dst is on the local disk.
1777 * Copy it from FS control to the local dst name.
1778 * @param src path
1779 * @param dst path
1780 */
1781 public void copyToLocalFile(Path src, Path dst) throws IOException {
1782 copyToLocalFile(false, src, dst);
1783 }
1784
1785 /**
1786 * The src file is under FS, and the dst is on the local disk.
1787 * Copy it from FS control to the local dst name.
1788 * Remove the source afterwards
1789 * @param src path
1790 * @param dst path
1791 */
1792 public void moveToLocalFile(Path src, Path dst) throws IOException {
1793 copyToLocalFile(true, src, dst);
1794 }
1795
1796 /**
1797 * The src file is under FS, and the dst is on the local disk.
1798 * Copy it from FS control to the local dst name.
1799 * delSrc indicates if the src will be removed or not.
1800 * @param delSrc whether to delete the src
1801 * @param src path
1802 * @param dst path
1803 */
1804 public void copyToLocalFile(boolean delSrc, Path src, Path dst)
1805 throws IOException {
1806 copyToLocalFile(delSrc, src, dst, false);
1807 }
1808
1809 /**
1810 * The src file is under FS, and the dst is on the local disk. Copy it from FS
1811 * control to the local dst name. delSrc indicates if the src will be removed
1812 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
1813 * as local file system or not. RawLocalFileSystem is non crc file system.So,
1814 * It will not create any crc files at local.
1815 *
1816 * @param delSrc
1817 * whether to delete the src
1818 * @param src
1819 * path
1820 * @param dst
1821 * path
1822 * @param useRawLocalFileSystem
1823 * whether to use RawLocalFileSystem as local file system or not.
1824 *
1825 * @throws IOException
1826 * - if any IO error
1827 */
1828 public void copyToLocalFile(boolean delSrc, Path src, Path dst,
1829 boolean useRawLocalFileSystem) throws IOException {
1830 Configuration conf = getConf();
1831 FileSystem local = null;
1832 if (useRawLocalFileSystem) {
1833 local = getLocal(conf).getRawFileSystem();
1834 } else {
1835 local = getLocal(conf);
1836 }
1837 FileUtil.copy(this, src, local, dst, delSrc, conf);
1838 }
1839
1840 /**
1841 * Returns a local File that the user can write output to. The caller
1842 * provides both the eventual FS target name and the local working
1843 * file. If the FS is local, we write directly into the target. If
1844 * the FS is remote, we write into the tmp local area.
1845 * @param fsOutputFile path of output file
1846 * @param tmpLocalFile path of local tmp file
1847 */
1848 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1849 throws IOException {
1850 return tmpLocalFile;
1851 }
1852
1853 /**
1854 * Called when we're all done writing to the target. A local FS will
1855 * do nothing, because we've written to exactly the right place. A remote
1856 * FS will copy the contents of tmpLocalFile to the correct target at
1857 * fsOutputFile.
1858 * @param fsOutputFile path of output file
1859 * @param tmpLocalFile path to local tmp file
1860 */
1861 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1862 throws IOException {
1863 moveFromLocalFile(tmpLocalFile, fsOutputFile);
1864 }
1865
1866 /**
1867 * No more filesystem operations are needed. Will
1868 * release any held locks.
1869 */
1870 public void close() throws IOException {
1871 // delete all files that were marked as delete-on-exit.
1872 processDeleteOnExit();
1873 CACHE.remove(this.key, this);
1874 }
1875
1876 /** Return the total size of all files in the filesystem.*/
1877 public long getUsed() throws IOException{
1878 long used = 0;
1879 FileStatus[] files = listStatus(new Path("/"));
1880 for(FileStatus file:files){
1881 used += file.getLen();
1882 }
1883 return used;
1884 }
1885
1886 /**
1887 * Get the block size for a particular file.
1888 * @param f the filename
1889 * @return the number of bytes in a block
1890 */
1891 /** @deprecated Use getFileStatus() instead */
1892 @Deprecated
1893 public long getBlockSize(Path f) throws IOException {
1894 return getFileStatus(f).getBlockSize();
1895 }
1896
1897 /** Return the number of bytes that large input files should be optimally
1898 * be split into to minimize i/o time. */
1899 public long getDefaultBlockSize() {
1900 // default to 32MB: large enough to minimize the impact of seeks
1901 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
1902 }
1903
1904 /**
1905 * Get the default replication.
1906 */
1907 public short getDefaultReplication() { return 1; }
1908
1909 /**
1910 * Return a file status object that represents the path.
1911 * @param f The path we want information from
1912 * @return a FileStatus object
1913 * @throws FileNotFoundException when the path does not exist;
1914 * IOException see specific implementation
1915 */
1916 public abstract FileStatus getFileStatus(Path f) throws IOException;
1917
1918 /**
1919 * Get the checksum of a file.
1920 *
1921 * @param f The file path
1922 * @return The file checksum. The default return value is null,
1923 * which indicates that no checksum algorithm is implemented
1924 * in the corresponding FileSystem.
1925 */
1926 public FileChecksum getFileChecksum(Path f) throws IOException {
1927 return null;
1928 }
1929
1930 /**
1931 * Set the verify checksum flag. This is only applicable if the
1932 * corresponding FileSystem supports checksum. By default doesn't do anything.
1933 * @param verifyChecksum
1934 */
1935 public void setVerifyChecksum(boolean verifyChecksum) {
1936 //doesn't do anything
1937 }
1938
1939 /**
1940 * Return a list of file status objects that corresponds to the list of paths
1941 * excluding those non-existent paths.
1942 *
1943 * @param paths
1944 * the list of paths we want information from
1945 * @return a list of FileStatus objects
1946 * @throws IOException
1947 * see specific implementation
1948 */
1949 private FileStatus[] getFileStatus(Path[] paths) throws IOException {
1950 if (paths == null) {
1951 return null;
1952 }
1953 ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
1954 for (int i = 0; i < paths.length; i++) {
1955 try {
1956 results.add(getFileStatus(paths[i]));
1957 } catch (FileNotFoundException e) { // do nothing
1958 }
1959 }
1960 return results.toArray(new FileStatus[results.size()]);
1961 }
1962
1963 /**
1964 * Returns a status object describing the use and capacity of the
1965 * file system. If the file system has multiple partitions, the
1966 * use and capacity of the root partition is reflected.
1967 *
1968 * @return a FsStatus object
1969 * @throws IOException
1970 * see specific implementation
1971 */
1972 public FsStatus getStatus() throws IOException {
1973 return getStatus(null);
1974 }
1975
1976 /**
1977 * Returns a status object describing the use and capacity of the
1978 * file system. If the file system has multiple partitions, the
1979 * use and capacity of the partition pointed to by the specified
1980 * path is reflected.
1981 * @param p Path for which status should be obtained. null means
1982 * the default partition.
1983 * @return a FsStatus object
1984 * @throws IOException
1985 * see specific implementation
1986 */
1987 public FsStatus getStatus(Path p) throws IOException {
1988 return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE);
1989 }
1990
1991 /**
1992 * Set permission of a path.
1993 * @param p
1994 * @param permission
1995 */
1996 public void setPermission(Path p, FsPermission permission
1997 ) throws IOException {
1998 }
1999
2000 /**
2001 * Set owner of a path (i.e. a file or a directory).
2002 * The parameters username and groupname cannot both be null.
2003 * @param p The path
2004 * @param username If it is null, the original username remains unchanged.
2005 * @param groupname If it is null, the original groupname remains unchanged.
2006 */
2007 public void setOwner(Path p, String username, String groupname
2008 ) throws IOException {
2009 }
2010
2011 /**
2012 * Set access time of a file
2013 * @param p The path
2014 * @param mtime Set the modification time of this file.
2015 * The number of milliseconds since Jan 1, 1970.
2016 * A value of -1 means that this call should not set modification time.
2017 * @param atime Set the access time of this file.
2018 * The number of milliseconds since Jan 1, 1970.
2019 * A value of -1 means that this call should not set access time.
2020 */
2021 public void setTimes(Path p, long mtime, long atime
2022 ) throws IOException {
2023 }
2024
2025 private static FileSystem createFileSystem(URI uri, Configuration conf
2026 ) throws IOException {
2027 Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
2028 if (clazz == null) {
2029 throw new IOException("No FileSystem for scheme: " + uri.getScheme());
2030 }
2031 FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
2032 fs.initialize(uri, conf);
2033 return fs;
2034 }
2035
2036 /** Caching FileSystem objects */
2037 static class Cache {
2038 private final ClientFinalizer clientFinalizer = new ClientFinalizer();
2039
2040 private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
2041 private final Set<Key> toAutoClose = new HashSet<Key>();
2042
2043 /** A variable that makes all objects in the cache unique */
2044 private static AtomicLong unique = new AtomicLong(1);
2045
2046 FileSystem get(URI uri, Configuration conf) throws IOException{
2047 Key key = new Key(uri, conf);
2048 return getInternal(uri, conf, key);
2049 }
2050
2051 /** The objects inserted into the cache using this method are all unique */
2052 FileSystem getUnique(URI uri, Configuration conf) throws IOException{
2053 Key key = new Key(uri, conf, unique.getAndIncrement());
2054 return getInternal(uri, conf, key);
2055 }
2056
2057 private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
2058 FileSystem fs;
2059 synchronized (this) {
2060 fs = map.get(key);
2061 }
2062 if (fs != null) {
2063 return fs;
2064 }
2065
2066 fs = createFileSystem(uri, conf);
2067 synchronized (this) { // refetch the lock again
2068 FileSystem oldfs = map.get(key);
2069 if (oldfs != null) { // a file system is created while lock is releasing
2070 fs.close(); // close the new file system
2071 return oldfs; // return the old file system
2072 }
2073
2074 // now insert the new file system into the map
2075 if (map.isEmpty() && !clientFinalizer.isAlive()) {
2076 Runtime.getRuntime().addShutdownHook(clientFinalizer);
2077 }
2078 fs.key = key;
2079 map.put(key, fs);
2080 if (conf.getBoolean("fs.automatic.close", true)) {
2081 toAutoClose.add(key);
2082 }
2083 return fs;
2084 }
2085 }
2086
2087 synchronized void remove(Key key, FileSystem fs) {
2088 if (map.containsKey(key) && fs == map.get(key)) {
2089 map.remove(key);
2090 toAutoClose.remove(key);
2091 if (map.isEmpty() && !clientFinalizer.isAlive()) {
2092 if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
2093 LOG.info("Could not cancel cleanup thread, though no " +
2094 "FileSystems are open");
2095 }
2096 }
2097 }
2098 }
2099
2100 synchronized void closeAll() throws IOException {
2101 closeAll(false);
2102 }
2103
2104 /**
2105 * Close all FileSystem instances in the Cache.
2106 * @param onlyAutomatic only close those that are marked for automatic closing
2107 */
2108 synchronized void closeAll(boolean onlyAutomatic) throws IOException {
2109 List<IOException> exceptions = new ArrayList<IOException>();
2110
2111 // Make a copy of the keys in the map since we'll be modifying
2112 // the map while iterating over it, which isn't safe.
2113 List<Key> keys = new ArrayList<Key>();
2114 keys.addAll(map.keySet());
2115
2116 for (Key key : keys) {
2117 final FileSystem fs = map.get(key);
2118
2119 if (onlyAutomatic && !toAutoClose.contains(key)) {
2120 continue;
2121 }
2122
2123 //remove from cache
2124 remove(key, fs);
2125
2126 if (fs != null) {
2127 try {
2128 fs.close();
2129 }
2130 catch(IOException ioe) {
2131 exceptions.add(ioe);
2132 }
2133 }
2134 }
2135
2136 if (!exceptions.isEmpty()) {
2137 throw MultipleIOException.createIOException(exceptions);
2138 }
2139 }
2140
2141 private class ClientFinalizer extends Thread {
2142 public synchronized void run() {
2143 try {
2144 closeAll(true);
2145 } catch (IOException e) {
2146 LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e);
2147 }
2148 }
2149 }
2150
2151 synchronized void closeAll(UserGroupInformation ugi) throws IOException {
2152 List<FileSystem> targetFSList = new ArrayList<FileSystem>();
2153 //Make a pass over the list and collect the filesystems to close
2154 //we cannot close inline since close() removes the entry from the Map
2155 for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
2156 final Key key = entry.getKey();
2157 final FileSystem fs = entry.getValue();
2158 if (ugi.equals(key.ugi) && fs != null) {
2159 targetFSList.add(fs);
2160 }
2161 }
2162 List<IOException> exceptions = new ArrayList<IOException>();
2163 //now make a pass over the target list and close each
2164 for (FileSystem fs : targetFSList) {
2165 try {
2166 fs.close();
2167 }
2168 catch(IOException ioe) {
2169 exceptions.add(ioe);
2170 }
2171 }
2172 if (!exceptions.isEmpty()) {
2173 throw MultipleIOException.createIOException(exceptions);
2174 }
2175 }
2176
2177 /** FileSystem.Cache.Key */
2178 static class Key {
2179 final String scheme;
2180 final String authority;
2181 final UserGroupInformation ugi;
2182 final long unique; // an artificial way to make a key unique
2183
2184 Key(URI uri, Configuration conf) throws IOException {
2185 this(uri, conf, 0);
2186 }
2187
2188 Key(URI uri, Configuration conf, long unique) throws IOException {
2189 scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
2190 authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
2191 this.unique = unique;
2192
2193 this.ugi = UserGroupInformation.getCurrentUser();
2194 }
2195
2196 /** {@inheritDoc} */
2197 public int hashCode() {
2198 return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique;
2199 }
2200
2201 static boolean isEqual(Object a, Object b) {
2202 return a == b || (a != null && a.equals(b));
2203 }
2204
2205 /** {@inheritDoc} */
2206 public boolean equals(Object obj) {
2207 if (obj == this) {
2208 return true;
2209 }
2210 if (obj != null && obj instanceof Key) {
2211 Key that = (Key)obj;
2212 return isEqual(this.scheme, that.scheme)
2213 && isEqual(this.authority, that.authority)
2214 && isEqual(this.ugi, that.ugi)
2215 && (this.unique == that.unique);
2216 }
2217 return false;
2218 }
2219
2220 /** {@inheritDoc} */
2221 public String toString() {
2222 return "("+ugi.toString() + ")@" + scheme + "://" + authority;
2223 }
2224 }
2225 }
2226
2227 public static final class Statistics {
2228 private final String scheme;
2229 private AtomicLong bytesRead = new AtomicLong();
2230 private AtomicLong bytesWritten = new AtomicLong();
2231 private AtomicInteger readOps = new AtomicInteger();
2232 private AtomicInteger largeReadOps = new AtomicInteger();
2233 private AtomicInteger writeOps = new AtomicInteger();
2234
2235 public Statistics(String scheme) {
2236 this.scheme = scheme;
2237 }
2238
2239 /**
2240 * Copy constructor.
2241 *
2242 * @param st
2243 * The input Statistics object which is cloned.
2244 */
2245 public Statistics(Statistics st) {
2246 this.scheme = st.scheme;
2247 this.bytesRead = new AtomicLong(st.bytesRead.longValue());
2248 this.bytesWritten = new AtomicLong(st.bytesWritten.longValue());
2249 }
2250
2251 /**
2252 * Increment the bytes read in the statistics
2253 * @param newBytes the additional bytes read
2254 */
2255 public void incrementBytesRead(long newBytes) {
2256 bytesRead.getAndAdd(newBytes);
2257 }
2258
2259 /**
2260 * Increment the bytes written in the statistics
2261 * @param newBytes the additional bytes written
2262 */
2263 public void incrementBytesWritten(long newBytes) {
2264 bytesWritten.getAndAdd(newBytes);
2265 }
2266
2267 /**
2268 * Increment the number of read operations
2269 * @param count number of read operations
2270 */
2271 public void incrementReadOps(int count) {
2272 readOps.getAndAdd(count);
2273 }
2274
2275 /**
2276 * Increment the number of large read operations
2277 * @param count number of large read operations
2278 */
2279 public void incrementLargeReadOps(int count) {
2280 largeReadOps.getAndAdd(count);
2281 }
2282
2283 /**
2284 * Increment the number of write operations
2285 * @param count number of write operations
2286 */
2287 public void incrementWriteOps(int count) {
2288 writeOps.getAndAdd(count);
2289 }
2290
2291 /**
2292 * Get the total number of bytes read
2293 * @return the number of bytes
2294 */
2295 public long getBytesRead() {
2296 return bytesRead.get();
2297 }
2298
2299 /**
2300 * Get the total number of bytes written
2301 * @return the number of bytes
2302 */
2303 public long getBytesWritten() {
2304 return bytesWritten.get();
2305 }
2306
2307 /**
2308 * Get the number of file system read operations such as list files
2309 * @return number of read operations
2310 */
2311 public int getReadOps() {
2312 return readOps.get() + largeReadOps.get();
2313 }
2314
2315 /**
2316 * Get the number of large file system read operations such as list files
2317 * under a large directory
2318 * @return number of large read operations
2319 */
2320 public int getLargeReadOps() {
2321 return largeReadOps.get();
2322 }
2323
2324 /**
2325 * Get the number of file system write operations such as create, append
2326 * rename etc.
2327 * @return number of write operations
2328 */
2329 public int getWriteOps() {
2330 return writeOps.get();
2331 }
2332
2333 public String toString() {
2334 return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
2335 + readOps + " read ops, " + largeReadOps + " large read ops, "
2336 + writeOps + " write ops";
2337 }
2338
2339 /**
2340 * Reset the counts of bytes to 0.
2341 */
2342 public void reset() {
2343 bytesWritten.set(0);
2344 bytesRead.set(0);
2345 }
2346
2347 /**
2348 * Get the uri scheme associated with this statistics object.
2349 * @return the schema associated with this set of statistics
2350 */
2351 public String getScheme() {
2352 return scheme;
2353 }
2354 }
2355
2356 /**
2357 * Get the Map of Statistics object indexed by URI Scheme.
2358 * @return a Map having a key as URI scheme and value as Statistics object
2359 * @deprecated use {@link #getAllStatistics} instead
2360 */
2361 @Deprecated
2362 public static synchronized Map<String, Statistics> getStatistics() {
2363 Map<String, Statistics> result = new HashMap<String, Statistics>();
2364 for(Statistics stat: statisticsTable.values()) {
2365 result.put(stat.getScheme(), stat);
2366 }
2367 return result;
2368 }
2369
2370 /**
2371 * Return the FileSystem classes that have Statistics
2372 */
2373 public static synchronized List<Statistics> getAllStatistics() {
2374 return new ArrayList<Statistics>(statisticsTable.values());
2375 }
2376
2377 /**
2378 * Get the statistics for a particular file system
2379 * @param cls the class to lookup
2380 * @return a statistics object
2381 */
2382 public static synchronized
2383 Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
2384 Statistics result = statisticsTable.get(cls);
2385 if (result == null) {
2386 result = new Statistics(scheme);
2387 statisticsTable.put(cls, result);
2388 }
2389 return result;
2390 }
2391
2392 /**
2393 * Reset all statistics for all file systems
2394 */
2395 public static synchronized void clearStatistics() {
2396 for(Statistics stat: statisticsTable.values()) {
2397 stat.reset();
2398 }
2399 }
2400
2401 /**
2402 * Print all statistics for all file systems
2403 */
2404 public static synchronized
2405 void printStatistics() throws IOException {
2406 for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
2407 statisticsTable.entrySet()) {
2408 System.out.println(" FileSystem " + pair.getKey().getName() +
2409 ": " + pair.getValue());
2410 }
2411 }
2412 }