001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.BufferedOutputStream;
022 import java.io.DataOutput;
023 import java.io.File;
024 import java.io.FileInputStream;
025 import java.io.FileNotFoundException;
026 import java.io.FileOutputStream;
027 import java.io.IOException;
028 import java.io.OutputStream;
029 import java.io.FileDescriptor;
030 import java.net.URI;
031 import java.nio.ByteBuffer;
032 import java.util.Arrays;
033 import java.util.EnumSet;
034 import java.util.StringTokenizer;
035
036 import org.apache.hadoop.classification.InterfaceAudience;
037 import org.apache.hadoop.classification.InterfaceStability;
038 import org.apache.hadoop.conf.Configuration;
039 import org.apache.hadoop.fs.permission.FsPermission;
040 import org.apache.hadoop.io.nativeio.NativeIO;
041 import org.apache.hadoop.util.Progressable;
042 import org.apache.hadoop.util.Shell;
043 import org.apache.hadoop.util.StringUtils;
044
045 /****************************************************************
046 * Implement the FileSystem API for the raw local filesystem.
047 *
048 *****************************************************************/
049 @InterfaceAudience.Public
050 @InterfaceStability.Stable
051 public class RawLocalFileSystem extends FileSystem {
052 static final URI NAME = URI.create("file:///");
053 private Path workingDir;
054
055 public RawLocalFileSystem() {
056 workingDir = getInitialWorkingDirectory();
057 }
058
059 private Path makeAbsolute(Path f) {
060 if (f.isAbsolute()) {
061 return f;
062 } else {
063 return new Path(workingDir, f);
064 }
065 }
066
067 /** Convert a path to a File. */
068 public File pathToFile(Path path) {
069 checkPath(path);
070 if (!path.isAbsolute()) {
071 path = new Path(getWorkingDirectory(), path);
072 }
073 return new File(path.toUri().getPath());
074 }
075
076 @Override
077 public URI getUri() { return NAME; }
078
079 @Override
080 public void initialize(URI uri, Configuration conf) throws IOException {
081 super.initialize(uri, conf);
082 setConf(conf);
083 }
084
085 class TrackingFileInputStream extends FileInputStream {
086 public TrackingFileInputStream(File f) throws IOException {
087 super(f);
088 }
089
090 @Override
091 public int read() throws IOException {
092 int result = super.read();
093 if (result != -1) {
094 statistics.incrementBytesRead(1);
095 }
096 return result;
097 }
098
099 @Override
100 public int read(byte[] data) throws IOException {
101 int result = super.read(data);
102 if (result != -1) {
103 statistics.incrementBytesRead(result);
104 }
105 return result;
106 }
107
108 @Override
109 public int read(byte[] data, int offset, int length) throws IOException {
110 int result = super.read(data, offset, length);
111 if (result != -1) {
112 statistics.incrementBytesRead(result);
113 }
114 return result;
115 }
116 }
117
118 /*******************************************************
119 * For open()'s FSInputStream.
120 *******************************************************/
121 class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor {
122 private FileInputStream fis;
123 private long position;
124
125 public LocalFSFileInputStream(Path f) throws IOException {
126 this.fis = new TrackingFileInputStream(pathToFile(f));
127 }
128
129 @Override
130 public void seek(long pos) throws IOException {
131 fis.getChannel().position(pos);
132 this.position = pos;
133 }
134
135 @Override
136 public long getPos() throws IOException {
137 return this.position;
138 }
139
140 @Override
141 public boolean seekToNewSource(long targetPos) throws IOException {
142 return false;
143 }
144
145 /*
146 * Just forward to the fis
147 */
148 @Override
149 public int available() throws IOException { return fis.available(); }
150 @Override
151 public void close() throws IOException { fis.close(); }
152 @Override
153 public boolean markSupported() { return false; }
154
155 @Override
156 public int read() throws IOException {
157 try {
158 int value = fis.read();
159 if (value >= 0) {
160 this.position++;
161 }
162 return value;
163 } catch (IOException e) { // unexpected exception
164 throw new FSError(e); // assume native fs error
165 }
166 }
167
168 @Override
169 public int read(byte[] b, int off, int len) throws IOException {
170 try {
171 int value = fis.read(b, off, len);
172 if (value > 0) {
173 this.position += value;
174 }
175 return value;
176 } catch (IOException e) { // unexpected exception
177 throw new FSError(e); // assume native fs error
178 }
179 }
180
181 @Override
182 public int read(long position, byte[] b, int off, int len)
183 throws IOException {
184 ByteBuffer bb = ByteBuffer.wrap(b, off, len);
185 try {
186 return fis.getChannel().read(bb, position);
187 } catch (IOException e) {
188 throw new FSError(e);
189 }
190 }
191
192 @Override
193 public long skip(long n) throws IOException {
194 long value = fis.skip(n);
195 if (value > 0) {
196 this.position += value;
197 }
198 return value;
199 }
200
201 @Override
202 public FileDescriptor getFileDescriptor() throws IOException {
203 return fis.getFD();
204 }
205 }
206
207 @Override
208 public FSDataInputStream open(Path f, int bufferSize) throws IOException {
209 if (!exists(f)) {
210 throw new FileNotFoundException(f.toString());
211 }
212 return new FSDataInputStream(new BufferedFSInputStream(
213 new LocalFSFileInputStream(f), bufferSize));
214 }
215
216 /*********************************************************
217 * For create()'s FSOutputStream.
218 *********************************************************/
219 class LocalFSFileOutputStream extends OutputStream {
220 private FileOutputStream fos;
221
222 private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
223 this.fos = new FileOutputStream(pathToFile(f), append);
224 }
225
226 /*
227 * Just forward to the fos
228 */
229 @Override
230 public void close() throws IOException { fos.close(); }
231 @Override
232 public void flush() throws IOException { fos.flush(); }
233 @Override
234 public void write(byte[] b, int off, int len) throws IOException {
235 try {
236 fos.write(b, off, len);
237 } catch (IOException e) { // unexpected exception
238 throw new FSError(e); // assume native fs error
239 }
240 }
241
242 @Override
243 public void write(int b) throws IOException {
244 try {
245 fos.write(b);
246 } catch (IOException e) { // unexpected exception
247 throw new FSError(e); // assume native fs error
248 }
249 }
250 }
251
252 @Override
253 public FSDataOutputStream append(Path f, int bufferSize,
254 Progressable progress) throws IOException {
255 if (!exists(f)) {
256 throw new FileNotFoundException("File " + f + " not found");
257 }
258 if (getFileStatus(f).isDirectory()) {
259 throw new IOException("Cannot append to a diretory (=" + f + " )");
260 }
261 return new FSDataOutputStream(new BufferedOutputStream(
262 new LocalFSFileOutputStream(f, true), bufferSize), statistics);
263 }
264
265 @Override
266 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
267 short replication, long blockSize, Progressable progress)
268 throws IOException {
269 return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
270 }
271
272 private FSDataOutputStream create(Path f, boolean overwrite,
273 boolean createParent, int bufferSize, short replication, long blockSize,
274 Progressable progress) throws IOException {
275 if (exists(f) && !overwrite) {
276 throw new IOException("File already exists: "+f);
277 }
278 Path parent = f.getParent();
279 if (parent != null && !mkdirs(parent)) {
280 throw new IOException("Mkdirs failed to create " + parent.toString());
281 }
282 return new FSDataOutputStream(new BufferedOutputStream(
283 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
284 }
285
286 @Override
287 @Deprecated
288 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
289 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
290 Progressable progress) throws IOException {
291 if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
292 throw new IOException("File already exists: "+f);
293 }
294 return new FSDataOutputStream(new BufferedOutputStream(
295 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
296 }
297
298 @Override
299 public FSDataOutputStream create(Path f, FsPermission permission,
300 boolean overwrite, int bufferSize, short replication, long blockSize,
301 Progressable progress) throws IOException {
302
303 FSDataOutputStream out = create(f,
304 overwrite, bufferSize, replication, blockSize, progress);
305 setPermission(f, permission);
306 return out;
307 }
308
309 @Override
310 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
311 boolean overwrite,
312 int bufferSize, short replication, long blockSize,
313 Progressable progress) throws IOException {
314 FSDataOutputStream out = create(f,
315 overwrite, false, bufferSize, replication, blockSize, progress);
316 setPermission(f, permission);
317 return out;
318 }
319
320 @Override
321 public boolean rename(Path src, Path dst) throws IOException {
322 // Attempt rename using Java API.
323 File srcFile = pathToFile(src);
324 File dstFile = pathToFile(dst);
325 if (srcFile.renameTo(dstFile)) {
326 return true;
327 }
328
329 // Enforce POSIX rename behavior that a source directory replaces an existing
330 // destination if the destination is an empty directory. On most platforms,
331 // this is already handled by the Java API call above. Some platforms
332 // (notably Windows) do not provide this behavior, so the Java API call above
333 // fails. Delete destination and attempt rename again.
334 if (this.exists(dst)) {
335 FileStatus sdst = this.getFileStatus(dst);
336 if (sdst.isDirectory() && dstFile.list().length == 0) {
337 if (LOG.isDebugEnabled()) {
338 LOG.debug("Deleting empty destination and renaming " + src + " to " +
339 dst);
340 }
341 if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
342 return true;
343 }
344 }
345 }
346
347 // The fallback behavior accomplishes the rename by a full copy.
348 if (LOG.isDebugEnabled()) {
349 LOG.debug("Falling through to a copy of " + src + " to " + dst);
350 }
351 return FileUtil.copy(this, src, this, dst, true, getConf());
352 }
353
354 /**
355 * Delete the given path to a file or directory.
356 * @param p the path to delete
357 * @param recursive to delete sub-directories
358 * @return true if the file or directory and all its contents were deleted
359 * @throws IOException if p is non-empty and recursive is false
360 */
361 @Override
362 public boolean delete(Path p, boolean recursive) throws IOException {
363 File f = pathToFile(p);
364 if (f.isFile()) {
365 return f.delete();
366 } else if (!recursive && f.isDirectory() &&
367 (FileUtil.listFiles(f).length != 0)) {
368 throw new IOException("Directory " + f.toString() + " is not empty");
369 }
370 return FileUtil.fullyDelete(f);
371 }
372
373 @Override
374 public FileStatus[] listStatus(Path f) throws IOException {
375 File localf = pathToFile(f);
376 FileStatus[] results;
377
378 if (!localf.exists()) {
379 throw new FileNotFoundException("File " + f + " does not exist");
380 }
381 if (localf.isFile()) {
382 return new FileStatus[] {
383 new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
384 }
385
386 File[] names = localf.listFiles();
387 if (names == null) {
388 return null;
389 }
390 results = new FileStatus[names.length];
391 int j = 0;
392 for (int i = 0; i < names.length; i++) {
393 try {
394 results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
395 j++;
396 } catch (FileNotFoundException e) {
397 // ignore the files not found since the dir list may have have changed
398 // since the names[] list was generated.
399 }
400 }
401 if (j == names.length) {
402 return results;
403 }
404 return Arrays.copyOf(results, j);
405 }
406
407 /**
408 * Creates the specified directory hierarchy. Does not
409 * treat existence as an error.
410 */
411 @Override
412 public boolean mkdirs(Path f) throws IOException {
413 if(f == null) {
414 throw new IllegalArgumentException("mkdirs path arg is null");
415 }
416 Path parent = f.getParent();
417 File p2f = pathToFile(f);
418 if(parent != null) {
419 File parent2f = pathToFile(parent);
420 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
421 throw new FileAlreadyExistsException("Parent path is not a directory: "
422 + parent);
423 }
424 }
425 return (parent == null || mkdirs(parent)) &&
426 (p2f.mkdir() || p2f.isDirectory());
427 }
428
429 @Override
430 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
431 boolean b = mkdirs(f);
432 if(b) {
433 setPermission(f, permission);
434 }
435 return b;
436 }
437
438
439 @Override
440 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
441 throws IOException {
442 boolean b = mkdirs(f);
443 setPermission(f, absolutePermission);
444 return b;
445 }
446
447
448 @Override
449 public Path getHomeDirectory() {
450 return this.makeQualified(new Path(System.getProperty("user.home")));
451 }
452
453 /**
454 * Set the working directory to the given directory.
455 */
456 @Override
457 public void setWorkingDirectory(Path newDir) {
458 workingDir = makeAbsolute(newDir);
459 checkPath(workingDir);
460
461 }
462
463 @Override
464 public Path getWorkingDirectory() {
465 return workingDir;
466 }
467
468 @Override
469 protected Path getInitialWorkingDirectory() {
470 return this.makeQualified(new Path(System.getProperty("user.dir")));
471 }
472
473 @Override
474 public FsStatus getStatus(Path p) throws IOException {
475 File partition = pathToFile(p == null ? new Path("/") : p);
476 //File provides getUsableSpace() and getFreeSpace()
477 //File provides no API to obtain used space, assume used = total - free
478 return new FsStatus(partition.getTotalSpace(),
479 partition.getTotalSpace() - partition.getFreeSpace(),
480 partition.getFreeSpace());
481 }
482
483 // In the case of the local filesystem, we can just rename the file.
484 @Override
485 public void moveFromLocalFile(Path src, Path dst) throws IOException {
486 rename(src, dst);
487 }
488
489 // We can write output directly to the final location
490 @Override
491 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
492 throws IOException {
493 return fsOutputFile;
494 }
495
496 // It's in the right place - nothing to do.
497 @Override
498 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
499 throws IOException {
500 }
501
502 @Override
503 public void close() throws IOException {
504 super.close();
505 }
506
507 @Override
508 public String toString() {
509 return "LocalFS";
510 }
511
512 @Override
513 public FileStatus getFileStatus(Path f) throws IOException {
514 File path = pathToFile(f);
515 if (path.exists()) {
516 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
517 } else {
518 throw new FileNotFoundException("File " + f + " does not exist");
519 }
520 }
521
522 static class RawLocalFileStatus extends FileStatus {
523 /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
524 * We recognize if the information is already loaded by check if
525 * onwer.equals("").
526 */
527 private boolean isPermissionLoaded() {
528 return !super.getOwner().equals("");
529 }
530
531 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
532 super(f.length(), f.isDirectory(), 1, defaultBlockSize,
533 f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
534 fs.getWorkingDirectory()));
535 }
536
537 @Override
538 public FsPermission getPermission() {
539 if (!isPermissionLoaded()) {
540 loadPermissionInfo();
541 }
542 return super.getPermission();
543 }
544
545 @Override
546 public String getOwner() {
547 if (!isPermissionLoaded()) {
548 loadPermissionInfo();
549 }
550 return super.getOwner();
551 }
552
553 @Override
554 public String getGroup() {
555 if (!isPermissionLoaded()) {
556 loadPermissionInfo();
557 }
558 return super.getGroup();
559 }
560
561 /// loads permissions, owner, and group from `ls -ld`
562 private void loadPermissionInfo() {
563 IOException e = null;
564 try {
565 String output = FileUtil.execCommand(new File(getPath().toUri()),
566 Shell.getGetPermissionCommand());
567 StringTokenizer t =
568 new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
569 //expected format
570 //-rw------- 1 username groupname ...
571 String permission = t.nextToken();
572 if (permission.length() > 10) { //files with ACLs might have a '+'
573 permission = permission.substring(0, 10);
574 }
575 setPermission(FsPermission.valueOf(permission));
576 t.nextToken();
577
578 String owner = t.nextToken();
579 // If on windows domain, token format is DOMAIN\\user and we want to
580 // extract only the user name
581 if (Shell.WINDOWS) {
582 int i = owner.indexOf('\\');
583 if (i != -1)
584 owner = owner.substring(i + 1);
585 }
586 setOwner(owner);
587
588 setGroup(t.nextToken());
589 } catch (Shell.ExitCodeException ioe) {
590 if (ioe.getExitCode() != 1) {
591 e = ioe;
592 } else {
593 setPermission(null);
594 setOwner(null);
595 setGroup(null);
596 }
597 } catch (IOException ioe) {
598 e = ioe;
599 } finally {
600 if (e != null) {
601 throw new RuntimeException("Error while running command to get " +
602 "file permissions : " +
603 StringUtils.stringifyException(e));
604 }
605 }
606 }
607
608 @Override
609 public void write(DataOutput out) throws IOException {
610 if (!isPermissionLoaded()) {
611 loadPermissionInfo();
612 }
613 super.write(out);
614 }
615 }
616
617 /**
618 * Use the command chown to set owner.
619 */
620 @Override
621 public void setOwner(Path p, String username, String groupname)
622 throws IOException {
623 FileUtil.setOwner(pathToFile(p), username, groupname);
624 }
625
626 /**
627 * Use the command chmod to set permission.
628 */
629 @Override
630 public void setPermission(Path p, FsPermission permission)
631 throws IOException {
632 if (NativeIO.isAvailable()) {
633 NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
634 permission.toShort());
635 } else {
636 String perm = String.format("%04o", permission.toShort());
637 Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
638 FileUtil.makeShellPath(pathToFile(p), true)));
639 }
640 }
641
642 /**
643 * Sets the {@link Path}'s last modified time <em>only</em> to the given
644 * valid time.
645 *
646 * @param mtime the modification time to set (only if greater than zero).
647 * @param atime currently ignored.
648 * @throws IOException if setting the last modified time fails.
649 */
650 @Override
651 public void setTimes(Path p, long mtime, long atime) throws IOException {
652 File f = pathToFile(p);
653 if(mtime >= 0) {
654 if(!f.setLastModified(mtime)) {
655 throw new IOException(
656 "couldn't set last-modified time to " +
657 mtime +
658 " for " +
659 f.getAbsolutePath());
660 }
661 }
662 }
663
664 private static String execCommand(File f, String... cmd) throws IOException {
665 String[] args = new String[cmd.length + 1];
666 System.arraycopy(cmd, 0, args, 0, cmd.length);
667 args[cmd.length] = FileUtil.makeShellPath(f, true);
668 String output = Shell.execCommand(args);
669 return output;
670 }
671
672 }