001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.fs;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.lang.reflect.Constructor;
023import java.net.URI;
024import java.net.URISyntaxException;
025import java.util.ArrayList;
026import java.util.EnumSet;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.NoSuchElementException;
031import java.util.StringTokenizer;
032import java.util.concurrent.ConcurrentHashMap;
033
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.apache.hadoop.HadoopIllegalArgumentException;
037import org.apache.hadoop.classification.InterfaceAudience;
038import org.apache.hadoop.classification.InterfaceStability;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.fs.FileSystem.Statistics;
041import org.apache.hadoop.fs.Options.ChecksumOpt;
042import org.apache.hadoop.fs.Options.CreateOpts;
043import org.apache.hadoop.fs.Options.Rename;
044import org.apache.hadoop.fs.permission.AclEntry;
045import org.apache.hadoop.fs.permission.AclStatus;
046import org.apache.hadoop.fs.permission.FsPermission;
047import org.apache.hadoop.fs.InvalidPathException;
048import org.apache.hadoop.security.AccessControlException;
049import org.apache.hadoop.security.SecurityUtil;
050import org.apache.hadoop.security.token.Token;
051import org.apache.hadoop.util.Progressable;
052
053/**
054 * This class provides an interface for implementors of a Hadoop file system
055 * (analogous to the VFS of Unix). Applications do not access this class;
056 * instead they access files across all file systems using {@link FileContext}.
057 * 
058 * Pathnames passed to AbstractFileSystem can be fully qualified URI that
059 * matches the "this" file system (ie same scheme and authority) 
060 * or a Slash-relative name that is assumed to be relative
061 * to the root of the "this" file system .
062 */
063@InterfaceAudience.Public
064@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
065public abstract class AbstractFileSystem {
066  static final Log LOG = LogFactory.getLog(AbstractFileSystem.class);
067
068  /** Recording statistics per a file system class. */
069  private static final Map<URI, Statistics> 
070      STATISTICS_TABLE = new HashMap<URI, Statistics>();
071  
072  /** Cache of constructors for each file system class. */
073  private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE = 
074    new ConcurrentHashMap<Class<?>, Constructor<?>>();
075  
076  private static final Class<?>[] URI_CONFIG_ARGS = 
077    new Class[]{URI.class, Configuration.class};
078  
079  /** The statistics for this file system. */
080  protected Statistics statistics;
081  
082  private final URI myUri;
083  
084  public Statistics getStatistics() {
085    return statistics;
086  }
087  
088  /**
089   * Returns true if the specified string is considered valid in the path part
090   * of a URI by this file system.  The default implementation enforces the rules
091   * of HDFS, but subclasses may override this method to implement specific
092   * validation rules for specific file systems.
093   * 
094   * @param src String source filename to check, path part of the URI
095   * @return boolean true if the specified string is considered valid
096   */
097  public boolean isValidName(String src) {
098    // Prohibit ".." "." and anything containing ":"
099    StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
100    while(tokens.hasMoreTokens()) {
101      String element = tokens.nextToken();
102      if (element.equals("..") ||
103          element.equals(".")  ||
104          (element.indexOf(":") >= 0)) {
105        return false;
106      }
107    }
108    return true;
109  }
110  
111  /** 
112   * Create an object for the given class and initialize it from conf.
113   * @param theClass class of which an object is created
114   * @param conf Configuration
115   * @return a new object
116   */
117  @SuppressWarnings("unchecked")
118  static <T> T newInstance(Class<T> theClass,
119    URI uri, Configuration conf) {
120    T result;
121    try {
122      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
123      if (meth == null) {
124        meth = theClass.getDeclaredConstructor(URI_CONFIG_ARGS);
125        meth.setAccessible(true);
126        CONSTRUCTOR_CACHE.put(theClass, meth);
127      }
128      result = meth.newInstance(uri, conf);
129    } catch (Exception e) {
130      throw new RuntimeException(e);
131    }
132    return result;
133  }
134  
135  /**
136   * Create a file system instance for the specified uri using the conf. The
137   * conf is used to find the class name that implements the file system. The
138   * conf is also passed to the file system for its configuration.
139   *
140   * @param uri URI of the file system
141   * @param conf Configuration for the file system
142   * 
143   * @return Returns the file system for the given URI
144   *
145   * @throws UnsupportedFileSystemException file system for <code>uri</code> is
146   *           not found
147   */
148  public static AbstractFileSystem createFileSystem(URI uri, Configuration conf)
149      throws UnsupportedFileSystemException {
150    Class<?> clazz = conf.getClass("fs.AbstractFileSystem." + 
151                                uri.getScheme() + ".impl", null);
152    if (clazz == null) {
153      throw new UnsupportedFileSystemException(
154          "No AbstractFileSystem for scheme: " + uri.getScheme());
155    }
156    return (AbstractFileSystem) newInstance(clazz, uri, conf);
157  }
158
159  /**
160   * Get the statistics for a particular file system.
161   * 
162   * @param uri
163   *          used as key to lookup STATISTICS_TABLE. Only scheme and authority
164   *          part of the uri are used.
165   * @return a statistics object
166   */
167  protected static synchronized Statistics getStatistics(URI uri) {
168    String scheme = uri.getScheme();
169    if (scheme == null) {
170      throw new IllegalArgumentException("Scheme not defined in the uri: "
171          + uri);
172    }
173    URI baseUri = getBaseUri(uri);
174    Statistics result = STATISTICS_TABLE.get(baseUri);
175    if (result == null) {
176      result = new Statistics(scheme);
177      STATISTICS_TABLE.put(baseUri, result);
178    }
179    return result;
180  }
181  
182  private static URI getBaseUri(URI uri) {
183    String scheme = uri.getScheme();
184    String authority = uri.getAuthority();
185    String baseUriString = scheme + "://";
186    if (authority != null) {
187      baseUriString = baseUriString + authority;
188    } else {
189      baseUriString = baseUriString + "/";
190    }
191    return URI.create(baseUriString);
192  }
193  
194  public static synchronized void clearStatistics() {
195    for(Statistics stat: STATISTICS_TABLE.values()) {
196      stat.reset();
197    }
198  }
199
200  /**
201   * Prints statistics for all file systems.
202   */
203  public static synchronized void printStatistics() {
204    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
205      System.out.println("  FileSystem " + pair.getKey().getScheme() + "://"
206          + pair.getKey().getAuthority() + ": " + pair.getValue());
207    }
208  }
209  
210  protected static synchronized Map<URI, Statistics> getAllStatistics() {
211    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
212        STATISTICS_TABLE.size());
213    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
214      URI key = pair.getKey();
215      Statistics value = pair.getValue();
216      Statistics newStatsObj = new Statistics(value);
217      statsMap.put(URI.create(key.toString()), newStatsObj);
218    }
219    return statsMap;
220  }
221
222  /**
223   * The main factory method for creating a file system. Get a file system for
224   * the URI's scheme and authority. The scheme of the <code>uri</code>
225   * determines a configuration property name,
226   * <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
227   * AbstractFileSystem class.
228   * 
229   * The entire URI and conf is passed to the AbstractFileSystem factory method.
230   * 
231   * @param uri for the file system to be created.
232   * @param conf which is passed to the file system impl.
233   * 
234   * @return file system for the given URI.
235   * 
236   * @throws UnsupportedFileSystemException if the file system for
237   *           <code>uri</code> is not supported.
238   */
239  public static AbstractFileSystem get(final URI uri, final Configuration conf)
240      throws UnsupportedFileSystemException {
241    return createFileSystem(uri, conf);
242  }
243
244  /**
245   * Constructor to be called by subclasses.
246   * 
247   * @param uri for this file system.
248   * @param supportedScheme the scheme supported by the implementor
249   * @param authorityNeeded if true then theURI must have authority, if false
250   *          then the URI must have null authority.
251   *
252   * @throws URISyntaxException <code>uri</code> has syntax error
253   */
254  public AbstractFileSystem(final URI uri, final String supportedScheme,
255      final boolean authorityNeeded, final int defaultPort)
256      throws URISyntaxException {
257    myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
258    statistics = getStatistics(uri); 
259  }
260  
261  /**
262   * Check that the Uri's scheme matches
263   * @param uri
264   * @param supportedScheme
265   */
266  public void checkScheme(URI uri, String supportedScheme) {
267    String scheme = uri.getScheme();
268    if (scheme == null) {
269      throw new HadoopIllegalArgumentException("Uri without scheme: " + uri);
270    }
271    if (!scheme.equals(supportedScheme)) {
272      throw new HadoopIllegalArgumentException("Uri scheme " + uri
273          + " does not match the scheme " + supportedScheme);
274    }
275  }
276
277  /**
278   * Get the URI for the file system based on the given URI. The path, query
279   * part of the given URI is stripped out and default file system port is used
280   * to form the URI.
281   * 
282   * @param uri FileSystem URI.
283   * @param authorityNeeded if true authority cannot be null in the URI. If
284   *          false authority must be null.
285   * @param defaultPort default port to use if port is not specified in the URI.
286   * 
287   * @return URI of the file system
288   * 
289   * @throws URISyntaxException <code>uri</code> has syntax error
290   */
291  private URI getUri(URI uri, String supportedScheme,
292      boolean authorityNeeded, int defaultPort) throws URISyntaxException {
293    checkScheme(uri, supportedScheme);
294    // A file system implementation that requires authority must always
295    // specify default port
296    if (defaultPort < 0 && authorityNeeded) {
297      throw new HadoopIllegalArgumentException(
298          "FileSystem implementation error -  default port " + defaultPort
299              + " is not valid");
300    }
301    String authority = uri.getAuthority();
302    if (authority == null) {
303       if (authorityNeeded) {
304         throw new HadoopIllegalArgumentException("Uri without authority: " + uri);
305       } else {
306         return new URI(supportedScheme + ":///");
307       }   
308    }
309    // authority is non null  - AuthorityNeeded may be true or false.
310    int port = uri.getPort();
311    port = (port == -1 ? defaultPort : port);
312    if (port == -1) { // no port supplied and default port is not specified
313      return new URI(supportedScheme, authority, "/", null);
314    }
315    return new URI(supportedScheme + "://" + uri.getHost() + ":" + port);
316  }
317  
318  /**
319   * The default port of this file system.
320   * 
321   * @return default port of this file system's Uri scheme
322   *         A uri with a port of -1 => default port;
323   */
324  public abstract int getUriDefaultPort();
325
326  /**
327   * Returns a URI whose scheme and authority identify this FileSystem.
328   * 
329   * @return the uri of this file system.
330   */
331  public URI getUri() {
332    return myUri;
333  }
334  
335  /**
336   * Check that a Path belongs to this FileSystem.
337   * 
338   * If the path is fully qualified URI, then its scheme and authority
339   * matches that of this file system. Otherwise the path must be 
340   * slash-relative name.
341   * 
342   * @throws InvalidPathException if the path is invalid
343   */
344  public void checkPath(Path path) {
345    URI uri = path.toUri();
346    String thatScheme = uri.getScheme();
347    String thatAuthority = uri.getAuthority();
348    if (thatScheme == null) {
349      if (thatAuthority == null) {
350        if (path.isUriPathAbsolute()) {
351          return;
352        }
353        throw new InvalidPathException("relative paths not allowed:" + 
354            path);
355      } else {
356        throw new InvalidPathException(
357            "Path without scheme with non-null authority:" + path);
358      }
359    }
360    String thisScheme = this.getUri().getScheme();
361    String thisHost = this.getUri().getHost();
362    String thatHost = uri.getHost();
363    
364    // Schemes and hosts must match.
365    // Allow for null Authority for file:///
366    if (!thisScheme.equalsIgnoreCase(thatScheme) ||
367       (thisHost != null && 
368            !thisHost.equalsIgnoreCase(thatHost)) ||
369       (thisHost == null && thatHost != null)) {
370      throw new InvalidPathException("Wrong FS: " + path + ", expected: "
371          + this.getUri());
372    }
373    
374    // Ports must match, unless this FS instance is using the default port, in
375    // which case the port may be omitted from the given URI
376    int thisPort = this.getUri().getPort();
377    int thatPort = uri.getPort();
378    if (thatPort == -1) { // -1 => defaultPort of Uri scheme
379      thatPort = this.getUriDefaultPort();
380    }
381    if (thisPort != thatPort) {
382      throw new InvalidPathException("Wrong FS: " + path + ", expected: "
383          + this.getUri());
384    }
385  }
386  
387  /**
388   * Get the path-part of a pathname. Checks that URI matches this file system
389   * and that the path-part is a valid name.
390   * 
391   * @param p path
392   * 
393   * @return path-part of the Path p
394   */
395  public String getUriPath(final Path p) {
396    checkPath(p);
397    String s = p.toUri().getPath();
398    if (!isValidName(s)) {
399      throw new InvalidPathException("Path part " + s + " from URI " + p
400          + " is not a valid filename.");
401    }
402    return s;
403  }
404  
405  /**
406   * Make the path fully qualified to this file system
407   * @param path
408   * @return the qualified path
409   */
410  public Path makeQualified(Path path) {
411    checkPath(path);
412    return path.makeQualified(this.getUri(), null);
413  }
414  
415  /**
416   * Some file systems like LocalFileSystem have an initial workingDir
417   * that is used as the starting workingDir. For other file systems
418   * like HDFS there is no built in notion of an initial workingDir.
419   * 
420   * @return the initial workingDir if the file system has such a notion
421   *         otherwise return a null.
422   */
423  public Path getInitialWorkingDirectory() {
424    return null;
425  }
426  
427  /** 
428   * Return the current user's home directory in this file system.
429   * The default implementation returns "/user/$USER/".
430   * 
431   * @return current user's home directory.
432   */
433  public Path getHomeDirectory() {
434    return new Path("/user/"+System.getProperty("user.name")).makeQualified(
435                                                                getUri(), null);
436  }
437  
438  /**
439   * Return a set of server default configuration values.
440   * 
441   * @return server default configuration values
442   * 
443   * @throws IOException an I/O error occurred
444   */
445  public abstract FsServerDefaults getServerDefaults() throws IOException; 
446
447  /**
448   * Return the fully-qualified path of path f resolving the path
449   * through any internal symlinks or mount point
450   * @param p path to be resolved
451   * @return fully qualified path 
452   * @throws FileNotFoundException, AccessControlException, IOException
453   *         UnresolvedLinkException if symbolic link on path cannot be resolved
454   *          internally
455   */
456   public Path resolvePath(final Path p) throws FileNotFoundException,
457           UnresolvedLinkException, AccessControlException, IOException {
458     checkPath(p);
459     return getFileStatus(p).getPath(); // default impl is to return the path
460   }
461  
462  /**
463   * The specification of this method matches that of
464   * {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
465   * that the Path f must be fully qualified and the permission is absolute
466   * (i.e. umask has been applied).
467   */
468  public final FSDataOutputStream create(final Path f,
469      final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
470      throws AccessControlException, FileAlreadyExistsException,
471      FileNotFoundException, ParentNotDirectoryException,
472      UnsupportedFileSystemException, UnresolvedLinkException, IOException {
473    checkPath(f);
474    int bufferSize = -1;
475    short replication = -1;
476    long blockSize = -1;
477    int bytesPerChecksum = -1;
478    ChecksumOpt checksumOpt = null;
479    FsPermission permission = null;
480    Progressable progress = null;
481    Boolean createParent = null;
482 
483    for (CreateOpts iOpt : opts) {
484      if (CreateOpts.BlockSize.class.isInstance(iOpt)) {
485        if (blockSize != -1) {
486          throw new HadoopIllegalArgumentException(
487              "BlockSize option is set multiple times");
488        }
489        blockSize = ((CreateOpts.BlockSize) iOpt).getValue();
490      } else if (CreateOpts.BufferSize.class.isInstance(iOpt)) {
491        if (bufferSize != -1) {
492          throw new HadoopIllegalArgumentException(
493              "BufferSize option is set multiple times");
494        }
495        bufferSize = ((CreateOpts.BufferSize) iOpt).getValue();
496      } else if (CreateOpts.ReplicationFactor.class.isInstance(iOpt)) {
497        if (replication != -1) {
498          throw new HadoopIllegalArgumentException(
499              "ReplicationFactor option is set multiple times");
500        }
501        replication = ((CreateOpts.ReplicationFactor) iOpt).getValue();
502      } else if (CreateOpts.BytesPerChecksum.class.isInstance(iOpt)) {
503        if (bytesPerChecksum != -1) {
504          throw new HadoopIllegalArgumentException(
505              "BytesPerChecksum option is set multiple times");
506        }
507        bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
508      } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) {
509        if (checksumOpt != null) {
510          throw new  HadoopIllegalArgumentException(
511              "CreateChecksumType option is set multiple times");
512        }
513        checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue();
514      } else if (CreateOpts.Perms.class.isInstance(iOpt)) {
515        if (permission != null) {
516          throw new HadoopIllegalArgumentException(
517              "Perms option is set multiple times");
518        }
519        permission = ((CreateOpts.Perms) iOpt).getValue();
520      } else if (CreateOpts.Progress.class.isInstance(iOpt)) {
521        if (progress != null) {
522          throw new HadoopIllegalArgumentException(
523              "Progress option is set multiple times");
524        }
525        progress = ((CreateOpts.Progress) iOpt).getValue();
526      } else if (CreateOpts.CreateParent.class.isInstance(iOpt)) {
527        if (createParent != null) {
528          throw new HadoopIllegalArgumentException(
529              "CreateParent option is set multiple times");
530        }
531        createParent = ((CreateOpts.CreateParent) iOpt).getValue();
532      } else {
533        throw new HadoopIllegalArgumentException("Unkown CreateOpts of type " +
534            iOpt.getClass().getName());
535      }
536    }
537    if (permission == null) {
538      throw new HadoopIllegalArgumentException("no permission supplied");
539    }
540
541
542    FsServerDefaults ssDef = getServerDefaults();
543    if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
544      throw new IOException("Internal error: default blockSize is" + 
545          " not a multiple of default bytesPerChecksum ");
546    }
547    
548    if (blockSize == -1) {
549      blockSize = ssDef.getBlockSize();
550    }
551
552    // Create a checksum option honoring user input as much as possible.
553    // If bytesPerChecksum is specified, it will override the one set in
554    // checksumOpt. Any missing value will be filled in using the default.
555    ChecksumOpt defaultOpt = new ChecksumOpt(
556        ssDef.getChecksumType(),
557        ssDef.getBytesPerChecksum());
558    checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt,
559        checksumOpt, bytesPerChecksum);
560
561    if (bufferSize == -1) {
562      bufferSize = ssDef.getFileBufferSize();
563    }
564    if (replication == -1) {
565      replication = ssDef.getReplication();
566    }
567    if (createParent == null) {
568      createParent = false;
569    }
570
571    if (blockSize % bytesPerChecksum != 0) {
572      throw new HadoopIllegalArgumentException(
573             "blockSize should be a multiple of checksumsize");
574    }
575
576    return this.createInternal(f, createFlag, permission, bufferSize,
577      replication, blockSize, progress, checksumOpt, createParent);
578  }
579
580  /**
581   * The specification of this method matches that of
582   * {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
583   * have been declared explicitly.
584   */
585  public abstract FSDataOutputStream createInternal(Path f,
586      EnumSet<CreateFlag> flag, FsPermission absolutePermission,
587      int bufferSize, short replication, long blockSize, Progressable progress,
588      ChecksumOpt checksumOpt, boolean createParent)
589      throws AccessControlException, FileAlreadyExistsException,
590      FileNotFoundException, ParentNotDirectoryException,
591      UnsupportedFileSystemException, UnresolvedLinkException, IOException;
592
593  /**
594   * The specification of this method matches that of
595   * {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
596   * f must be fully qualified and the permission is absolute (i.e. 
597   * umask has been applied).
598   */
599  public abstract void mkdir(final Path dir, final FsPermission permission,
600      final boolean createParent) throws AccessControlException,
601      FileAlreadyExistsException, FileNotFoundException,
602      UnresolvedLinkException, IOException;
603
604  /**
605   * The specification of this method matches that of
606   * {@link FileContext#delete(Path, boolean)} except that Path f must be for
607   * this file system.
608   */
609  public abstract boolean delete(final Path f, final boolean recursive)
610      throws AccessControlException, FileNotFoundException,
611      UnresolvedLinkException, IOException;
612
613  /**
614   * The specification of this method matches that of
615   * {@link FileContext#open(Path)} except that Path f must be for this
616   * file system.
617   */
618  public FSDataInputStream open(final Path f) throws AccessControlException,
619      FileNotFoundException, UnresolvedLinkException, IOException {
620    return open(f, getServerDefaults().getFileBufferSize());
621  }
622
623  /**
624   * The specification of this method matches that of
625   * {@link FileContext#open(Path, int)} except that Path f must be for this
626   * file system.
627   */
628  public abstract FSDataInputStream open(final Path f, int bufferSize)
629      throws AccessControlException, FileNotFoundException,
630      UnresolvedLinkException, IOException;
631
632  /**
633   * The specification of this method matches that of
634   * {@link FileContext#setReplication(Path, short)} except that Path f must be
635   * for this file system.
636   */
637  public abstract boolean setReplication(final Path f,
638      final short replication) throws AccessControlException,
639      FileNotFoundException, UnresolvedLinkException, IOException;
640
641  /**
642   * The specification of this method matches that of
643   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
644   * f must be for this file system.
645   */
646  public final void rename(final Path src, final Path dst,
647      final Options.Rename... options) throws AccessControlException,
648      FileAlreadyExistsException, FileNotFoundException,
649      ParentNotDirectoryException, UnresolvedLinkException, IOException {
650    boolean overwrite = false;
651    if (null != options) {
652      for (Rename option : options) {
653        if (option == Rename.OVERWRITE) {
654          overwrite = true;
655        }
656      }
657    }
658    renameInternal(src, dst, overwrite);
659  }
660  
661  /**
662   * The specification of this method matches that of
663   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
664   * f must be for this file system and NO OVERWRITE is performed.
665   * 
666   * File systems that do not have a built in overwrite need implement only this
667   * method and can take advantage of the default impl of the other
668   * {@link #renameInternal(Path, Path, boolean)}
669   */
670  public abstract void renameInternal(final Path src, final Path dst)
671      throws AccessControlException, FileAlreadyExistsException,
672      FileNotFoundException, ParentNotDirectoryException,
673      UnresolvedLinkException, IOException;
674  
675  /**
676   * The specification of this method matches that of
677   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
678   * f must be for this file system.
679   */
680  public void renameInternal(final Path src, final Path dst,
681      boolean overwrite) throws AccessControlException,
682      FileAlreadyExistsException, FileNotFoundException,
683      ParentNotDirectoryException, UnresolvedLinkException, IOException {
684    // Default implementation deals with overwrite in a non-atomic way
685    final FileStatus srcStatus = getFileLinkStatus(src);
686
687    FileStatus dstStatus;
688    try {
689      dstStatus = getFileLinkStatus(dst);
690    } catch (IOException e) {
691      dstStatus = null;
692    }
693    if (dstStatus != null) {
694      if (dst.equals(src)) {
695        throw new FileAlreadyExistsException(
696            "The source "+src+" and destination "+dst+" are the same");
697      }
698      if (srcStatus.isSymlink() && dst.equals(srcStatus.getSymlink())) {
699        throw new FileAlreadyExistsException(
700            "Cannot rename symlink "+src+" to its target "+dst);
701      }
702      // It's OK to rename a file to a symlink and vice versa
703      if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
704        throw new IOException("Source " + src + " and destination " + dst
705            + " must both be directories");
706      }
707      if (!overwrite) {
708        throw new FileAlreadyExistsException("Rename destination " + dst
709            + " already exists.");
710      }
711      // Delete the destination that is a file or an empty directory
712      if (dstStatus.isDirectory()) {
713        RemoteIterator<FileStatus> list = listStatusIterator(dst);
714        if (list != null && list.hasNext()) {
715          throw new IOException(
716              "Rename cannot overwrite non empty destination directory " + dst);
717        }
718      }
719      delete(dst, false);
720    } else {
721      final Path parent = dst.getParent();
722      final FileStatus parentStatus = getFileStatus(parent);
723      if (parentStatus.isFile()) {
724        throw new ParentNotDirectoryException("Rename destination parent "
725            + parent + " is a file.");
726      }
727    }
728    renameInternal(src, dst);
729  }
730  
731  /**
732   * Returns true if the file system supports symlinks, false otherwise.
733   * @return true if filesystem supports symlinks
734   */
735  public boolean supportsSymlinks() {
736    return false;
737  }
738  
739  /**
740   * The specification of this method matches that of  
741   * {@link FileContext#createSymlink(Path, Path, boolean)};
742   */
743  public void createSymlink(final Path target, final Path link,
744      final boolean createParent) throws IOException, UnresolvedLinkException {
745    throw new IOException("File system does not support symlinks");    
746  }
747
748  /**
749   * Partially resolves the path. This is used during symlink resolution in
750   * {@link FSLinkResolver}, and differs from the similarly named method
751   * {@link FileContext#getLinkTarget(Path)}.
752   * @throws IOException subclass implementations may throw IOException 
753   */
754  public Path getLinkTarget(final Path f) throws IOException {
755    throw new AssertionError("Implementation Error: " + getClass()
756        + " that threw an UnresolvedLinkException, causing this method to be"
757        + " called, needs to override this method.");
758  }
759    
760  /**
761   * The specification of this method matches that of
762   * {@link FileContext#setPermission(Path, FsPermission)} except that Path f
763   * must be for this file system.
764   */
765  public abstract void setPermission(final Path f,
766      final FsPermission permission) throws AccessControlException,
767      FileNotFoundException, UnresolvedLinkException, IOException;
768
769  /**
770   * The specification of this method matches that of
771   * {@link FileContext#setOwner(Path, String, String)} except that Path f must
772   * be for this file system.
773   */
774  public abstract void setOwner(final Path f, final String username,
775      final String groupname) throws AccessControlException,
776      FileNotFoundException, UnresolvedLinkException, IOException;
777
778  /**
779   * The specification of this method matches that of
780   * {@link FileContext#setTimes(Path, long, long)} except that Path f must be
781   * for this file system.
782   */
783  public abstract void setTimes(final Path f, final long mtime,
784    final long atime) throws AccessControlException, FileNotFoundException,
785      UnresolvedLinkException, IOException;
786
787  /**
788   * The specification of this method matches that of
789   * {@link FileContext#getFileChecksum(Path)} except that Path f must be for
790   * this file system.
791   */
792  public abstract FileChecksum getFileChecksum(final Path f)
793      throws AccessControlException, FileNotFoundException,
794      UnresolvedLinkException, IOException;
795  
796  /**
797   * The specification of this method matches that of
798   * {@link FileContext#getFileStatus(Path)} 
799   * except that an UnresolvedLinkException may be thrown if a symlink is 
800   * encountered in the path.
801   */
802  public abstract FileStatus getFileStatus(final Path f)
803      throws AccessControlException, FileNotFoundException,
804      UnresolvedLinkException, IOException;
805
806  /**
807   * The specification of this method matches that of
808   * {@link FileContext#getFileLinkStatus(Path)}
809   * except that an UnresolvedLinkException may be thrown if a symlink is  
810   * encountered in the path leading up to the final path component.
811   * If the file system does not support symlinks then the behavior is
812   * equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
813   */
814  public FileStatus getFileLinkStatus(final Path f)
815      throws AccessControlException, FileNotFoundException,
816      UnsupportedFileSystemException, IOException {
817    return getFileStatus(f);
818  }
819
820  /**
821   * The specification of this method matches that of
822   * {@link FileContext#getFileBlockLocations(Path, long, long)} except that
823   * Path f must be for this file system.
824   */
825  public abstract BlockLocation[] getFileBlockLocations(final Path f,
826      final long start, final long len) throws AccessControlException,
827      FileNotFoundException, UnresolvedLinkException, IOException;
828
829  /**
830   * The specification of this method matches that of
831   * {@link FileContext#getFsStatus(Path)} except that Path f must be for this
832   * file system.
833   */
834  public FsStatus getFsStatus(final Path f) throws AccessControlException,
835      FileNotFoundException, UnresolvedLinkException, IOException {
836    // default impl gets FsStatus of root
837    return getFsStatus();
838  }
839  
840  /**
841   * The specification of this method matches that of
842   * {@link FileContext#getFsStatus(Path)}.
843   */
844  public abstract FsStatus getFsStatus() throws AccessControlException,
845      FileNotFoundException, IOException;
846
847  /**
848   * The specification of this method matches that of
849   * {@link FileContext#listStatus(Path)} except that Path f must be for this
850   * file system.
851   */
852  public RemoteIterator<FileStatus> listStatusIterator(final Path f)
853      throws AccessControlException, FileNotFoundException,
854      UnresolvedLinkException, IOException {
855    return new RemoteIterator<FileStatus>() {
856      private int i = 0;
857      private FileStatus[] statusList = listStatus(f);
858      
859      @Override
860      public boolean hasNext() {
861        return i < statusList.length;
862      }
863      
864      @Override
865      public FileStatus next() {
866        if (!hasNext()) {
867          throw new NoSuchElementException();
868        }
869        return statusList[i++];
870      }
871    };
872  }
873
874  /**
875   * The specification of this method matches that of
876   * {@link FileContext#listLocatedStatus(Path)} except that Path f 
877   * must be for this file system.
878   */
879  public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
880      throws AccessControlException, FileNotFoundException,
881      UnresolvedLinkException, IOException {
882    return new RemoteIterator<LocatedFileStatus>() {
883      private RemoteIterator<FileStatus> itor = listStatusIterator(f);
884      
885      @Override
886      public boolean hasNext() throws IOException {
887        return itor.hasNext();
888      }
889      
890      @Override
891      public LocatedFileStatus next() throws IOException {
892        if (!hasNext()) {
893          throw new NoSuchElementException("No more entry in " + f);
894        }
895        FileStatus result = itor.next();
896        BlockLocation[] locs = null;
897        if (result.isFile()) {
898          locs = getFileBlockLocations(
899              result.getPath(), 0, result.getLen());
900        }
901        return new LocatedFileStatus(result, locs);
902      }
903    };
904  }
905
906  /**
907   * The specification of this method matches that of
908   * {@link FileContext.Util#listStatus(Path)} except that Path f must be 
909   * for this file system.
910   */
911  public abstract FileStatus[] listStatus(final Path f)
912      throws AccessControlException, FileNotFoundException,
913      UnresolvedLinkException, IOException;
914
915  /**
916   * @return an iterator over the corrupt files under the given path
917   * (may contain duplicates if a file has more than one corrupt block)
918   * @throws IOException
919   */
920  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
921    throws IOException {
922    throw new UnsupportedOperationException(getClass().getCanonicalName() +
923                                            " does not support" +
924                                            " listCorruptFileBlocks");
925  }
926
927  /**
928   * The specification of this method matches that of
929   * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
930   * must be for this file system.
931   */
932  public abstract void setVerifyChecksum(final boolean verifyChecksum)
933      throws AccessControlException, IOException;
934  
935  /**
936   * Get a canonical name for this file system.
937   * @return a URI string that uniquely identifies this file system
938   */
939  public String getCanonicalServiceName() {
940    return SecurityUtil.buildDTServiceName(getUri(), getUriDefaultPort());
941  }
942  
943  /**
944   * Get one or more delegation tokens associated with the filesystem. Normally
945   * a file system returns a single delegation token. A file system that manages
946   * multiple file systems underneath, could return set of delegation tokens for
947   * all the file systems it manages
948   * 
949   * @param renewer the account name that is allowed to renew the token.
950   * @return List of delegation tokens.
951   *   If delegation tokens not supported then return a list of size zero.
952   * @throws IOException
953   */
954  @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
955  public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
956    return new ArrayList<Token<?>>(0);
957  }
958
959  /**
960   * Modifies ACL entries of files and directories.  This method can add new ACL
961   * entries or modify the permissions on existing ACL entries.  All existing
962   * ACL entries that are not specified in this call are retained without
963   * changes.  (Modifications are merged into the current ACL.)
964   *
965   * @param path Path to modify
966   * @param aclSpec List<AclEntry> describing modifications
967   * @throws IOException if an ACL could not be modified
968   */
969  public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
970      throws IOException {
971    throw new UnsupportedOperationException(getClass().getSimpleName()
972        + " doesn't support modifyAclEntries");
973  }
974
975  /**
976   * Removes ACL entries from files and directories.  Other ACL entries are
977   * retained.
978   *
979   * @param path Path to modify
980   * @param aclSpec List<AclEntry> describing entries to remove
981   * @throws IOException if an ACL could not be modified
982   */
983  public void removeAclEntries(Path path, List<AclEntry> aclSpec)
984      throws IOException {
985    throw new UnsupportedOperationException(getClass().getSimpleName()
986        + " doesn't support removeAclEntries");
987  }
988
989  /**
990   * Removes all default ACL entries from files and directories.
991   *
992   * @param path Path to modify
993   * @throws IOException if an ACL could not be modified
994   */
995  public void removeDefaultAcl(Path path)
996      throws IOException {
997    throw new UnsupportedOperationException(getClass().getSimpleName()
998        + " doesn't support removeDefaultAcl");
999  }
1000
1001  /**
1002   * Removes all but the base ACL entries of files and directories.  The entries
1003   * for user, group, and others are retained for compatibility with permission
1004   * bits.
1005   *
1006   * @param path Path to modify
1007   * @throws IOException if an ACL could not be removed
1008   */
1009  public void removeAcl(Path path)
1010      throws IOException {
1011    throw new UnsupportedOperationException(getClass().getSimpleName()
1012        + " doesn't support removeAcl");
1013  }
1014
1015  /**
1016   * Fully replaces ACL of files and directories, discarding all existing
1017   * entries.
1018   *
1019   * @param path Path to modify
1020   * @param aclSpec List<AclEntry> describing modifications, must include entries
1021   *   for user, group, and others for compatibility with permission bits.
1022   * @throws IOException if an ACL could not be modified
1023   */
1024  public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
1025    throw new UnsupportedOperationException(getClass().getSimpleName()
1026        + " doesn't support setAcl");
1027  }
1028
1029  /**
1030   * Gets the ACLs of files and directories.
1031   *
1032   * @param path Path to get
1033   * @return RemoteIterator<AclStatus> which returns each AclStatus
1034   * @throws IOException if an ACL could not be read
1035   */
1036  public AclStatus getAclStatus(Path path) throws IOException {
1037    throw new UnsupportedOperationException(getClass().getSimpleName()
1038        + " doesn't support getAclStatus");
1039  }
1040
1041  /**
1042   * Set an xattr of a file or directory.
1043   * The name must be prefixed with user/trusted/security/system and
1044   * followed by ".". For example, "user.attr".
1045   * <p/>
1046   * A regular user can only set an xattr for the "user" namespace.
1047   * The super user can set an xattr of either the "user" or "trusted" namespaces.
1048   * The xattrs of the "security" and "system" namespaces are only used/exposed 
1049   * internally by/to the FS impl.
1050   * <p/>
1051   * The access permissions of an xattr in the "user" namespace are
1052   * defined by the file and directory permission bits.
1053   * An xattr can only be set when the logged-in user has the correct permissions.
1054   * If the xattr exists, it will be replaced.
1055   * <p/>
1056   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1057   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1058   *
1059   * @param path Path to modify
1060   * @param name xattr name.
1061   * @param value xattr value.
1062   * @throws IOException
1063   */
1064  public void setXAttr(Path path, String name, byte[] value)
1065      throws IOException {
1066    setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE,
1067        XAttrSetFlag.REPLACE));
1068  }
1069
1070  /**
1071   * Set an xattr of a file or directory.
1072   * The name must be prefixed with user/trusted/security/system and
1073   * followed by ".". For example, "user.attr".
1074   * <p/>
1075   * A regular user can only set an xattr for the "user" namespace.
1076   * The super user can set an xattr of either the "user" or "trusted" namespaces.
1077   * The xattrs of the "security" and "system" namespaces are only used/exposed 
1078   * internally by/to the FS impl.
1079   * <p/>
1080   * The access permissions of an xattr in the "user" namespace are
1081   * defined by the file and directory permission bits.
1082   * An xattr can only be set when the logged-in user has the correct permissions.
1083   * If the xattr exists, it will be replaced.
1084   * <p/>
1085   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1086   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1087   *
1088   * @param path Path to modify
1089   * @param name xattr name.
1090   * @param value xattr value.
1091   * @param flag xattr set flag
1092   * @throws IOException
1093   */
1094  public void setXAttr(Path path, String name, byte[] value,
1095      EnumSet<XAttrSetFlag> flag) throws IOException {
1096    throw new UnsupportedOperationException(getClass().getSimpleName()
1097        + " doesn't support setXAttr");
1098  }
1099
1100  /**
1101   * Get an xattr for a file or directory.
1102   * The name must be prefixed with user/trusted/security/system and
1103   * followed by ".". For example, "user.attr".
1104   * <p/>
1105   * A regular user can only get an xattr for the "user" namespace.
1106   * The super user can get an xattr of either the "user" or "trusted" namespaces.
1107   * The xattrs of the "security" and "system" namespaces are only used/exposed 
1108   * internally by/to the FS impl.
1109   * <p/>
1110   * An xattr will only be returned when the logged-in user has the correct permissions.
1111   * <p/>
1112   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1113   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1114   *
1115   * @param path Path to get extended attribute
1116   * @param name xattr name.
1117   * @return byte[] xattr value.
1118   * @throws IOException
1119   */
1120  public byte[] getXAttr(Path path, String name) throws IOException {
1121    throw new UnsupportedOperationException(getClass().getSimpleName()
1122        + " doesn't support getXAttr");
1123  }
1124
1125  /**
1126   * Get all of the xattrs for a file or directory.
1127   * Only those xattrs for which the logged-in user has permissions to view
1128   * are returned.
1129   * <p/>
1130   * A regular user can only get xattrs for the "user" namespace.
1131   * The super user can only get xattrs for "user" and "trusted" namespaces.
1132   * The xattr of "security" and "system" namespaces are only used/exposed 
1133   * internally by/to the FS impl.
1134   * <p/>
1135   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1136   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1137   *
1138   * @param path Path to get extended attributes
1139   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1140   * @throws IOException
1141   */
1142  public Map<String, byte[]> getXAttrs(Path path) throws IOException {
1143    throw new UnsupportedOperationException(getClass().getSimpleName()
1144        + " doesn't support getXAttrs");
1145  }
1146
1147  /**
1148   * Get all of the xattrs for a file or directory.
1149   * Only those xattrs for which the logged-in user has permissions to view
1150   * are returned.
1151   * <p/>
1152   * A regular user can only get xattrs for the "user" namespace.
1153   * The super user can only get xattrs for "user" and "trusted" namespaces.
1154   * The xattr of "security" and "system" namespaces are only used/exposed 
1155   * internally by/to the FS impl.
1156   * <p/>
1157   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1158   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1159   *
1160   * @param path Path to get extended attributes
1161   * @param names XAttr names.
1162   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1163   * @throws IOException
1164   */
1165  public Map<String, byte[]> getXAttrs(Path path, List<String> names)
1166      throws IOException {
1167    throw new UnsupportedOperationException(getClass().getSimpleName()
1168        + " doesn't support getXAttrs");
1169  }
1170
1171  /**
1172   * Get all of the xattr names for a file or directory.
1173   * Only the xattr names for which the logged-in user has permissions to view
1174   * are returned.
1175   * <p/>
1176   * A regular user can only get xattr names for the "user" namespace.
1177   * The super user can only get xattr names for the "user" and "trusted"
1178   * namespaces.
1179   * The xattr names in the "security" and "system" namespaces are only
1180   * used/exposed internally by/to the FS impl.
1181   * <p/>
1182   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1183   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1184   *
1185   * @param path Path to get extended attributes
1186   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1187   * @throws IOException
1188   */
1189  public List<String> listXAttrs(Path path)
1190          throws IOException {
1191    throw new UnsupportedOperationException(getClass().getSimpleName()
1192            + " doesn't support listXAttrs");
1193  }
1194
1195  /**
1196   * Remove an xattr of a file or directory.
1197   * The name must be prefixed with user/trusted/security/system and
1198   * followed by ".". For example, "user.attr".
1199   * <p/>
1200   * A regular user can only remove an xattr for the "user" namespace.
1201   * The super user can remove an xattr of either the "user" or "trusted" namespaces.
1202   * The xattrs of the "security" and "system" namespaces are only used/exposed 
1203   * internally by/to the FS impl.
1204   * <p/>
1205   * The access permissions of an xattr in the "user" namespace are
1206   * defined by the file and directory permission bits.
1207   * An xattr can only be set when the logged-in user has the correct permissions.
1208   * If the xattr exists, it will be replaced.
1209   * <p/>
1210   * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1211   * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1212   *
1213   * @param path Path to remove extended attribute
1214   * @param name xattr name
1215   * @throws IOException
1216   */
1217  public void removeXAttr(Path path, String name) throws IOException {
1218    throw new UnsupportedOperationException(getClass().getSimpleName()
1219        + " doesn't support removeXAttr");
1220  }
1221
1222  @Override //Object
1223  public int hashCode() {
1224    return myUri.hashCode();
1225  }
1226  
1227  @Override //Object
1228  public boolean equals(Object other) {
1229    if (other == null || !(other instanceof AbstractFileSystem)) {
1230      return false;
1231    }
1232    return myUri.equals(((AbstractFileSystem) other).myUri);
1233  }
1234}