001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.BufferedOutputStream;
022 import java.io.DataOutput;
023 import java.io.File;
024 import java.io.FileInputStream;
025 import java.io.FileNotFoundException;
026 import java.io.FileOutputStream;
027 import java.io.IOException;
028 import java.io.OutputStream;
029 import java.io.FileDescriptor;
030 import java.net.URI;
031 import java.nio.ByteBuffer;
032 import java.util.Arrays;
033 import java.util.EnumSet;
034 import java.util.StringTokenizer;
035
036 import org.apache.hadoop.classification.InterfaceAudience;
037 import org.apache.hadoop.classification.InterfaceStability;
038 import org.apache.hadoop.conf.Configuration;
039 import org.apache.hadoop.fs.permission.FsPermission;
040 import org.apache.hadoop.io.nativeio.NativeIO;
041 import org.apache.hadoop.util.Progressable;
042 import org.apache.hadoop.util.Shell;
043 import org.apache.hadoop.util.StringUtils;
044
045 /****************************************************************
046 * Implement the FileSystem API for the raw local filesystem.
047 *
048 *****************************************************************/
049 @InterfaceAudience.Public
050 @InterfaceStability.Stable
051 public class RawLocalFileSystem extends FileSystem {
052 static final URI NAME = URI.create("file:///");
053 private Path workingDir;
054
055 public RawLocalFileSystem() {
056 workingDir = getInitialWorkingDirectory();
057 }
058
059 private Path makeAbsolute(Path f) {
060 if (f.isAbsolute()) {
061 return f;
062 } else {
063 return new Path(workingDir, f);
064 }
065 }
066
067 /** Convert a path to a File. */
068 public File pathToFile(Path path) {
069 checkPath(path);
070 if (!path.isAbsolute()) {
071 path = new Path(getWorkingDirectory(), path);
072 }
073 return new File(path.toUri().getPath());
074 }
075
076 @Override
077 public URI getUri() { return NAME; }
078
079 @Override
080 public void initialize(URI uri, Configuration conf) throws IOException {
081 super.initialize(uri, conf);
082 setConf(conf);
083 }
084
085 class TrackingFileInputStream extends FileInputStream {
086 public TrackingFileInputStream(File f) throws IOException {
087 super(f);
088 }
089
090 @Override
091 public int read() throws IOException {
092 int result = super.read();
093 if (result != -1) {
094 statistics.incrementBytesRead(1);
095 }
096 return result;
097 }
098
099 @Override
100 public int read(byte[] data) throws IOException {
101 int result = super.read(data);
102 if (result != -1) {
103 statistics.incrementBytesRead(result);
104 }
105 return result;
106 }
107
108 @Override
109 public int read(byte[] data, int offset, int length) throws IOException {
110 int result = super.read(data, offset, length);
111 if (result != -1) {
112 statistics.incrementBytesRead(result);
113 }
114 return result;
115 }
116 }
117
118 /*******************************************************
119 * For open()'s FSInputStream.
120 *******************************************************/
121 class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor {
122 private FileInputStream fis;
123 private long position;
124
125 public LocalFSFileInputStream(Path f) throws IOException {
126 this.fis = new TrackingFileInputStream(pathToFile(f));
127 }
128
129 @Override
130 public void seek(long pos) throws IOException {
131 fis.getChannel().position(pos);
132 this.position = pos;
133 }
134
135 @Override
136 public long getPos() throws IOException {
137 return this.position;
138 }
139
140 @Override
141 public boolean seekToNewSource(long targetPos) throws IOException {
142 return false;
143 }
144
145 /*
146 * Just forward to the fis
147 */
148 @Override
149 public int available() throws IOException { return fis.available(); }
150 @Override
151 public void close() throws IOException { fis.close(); }
152 @Override
153 public boolean markSupported() { return false; }
154
155 @Override
156 public int read() throws IOException {
157 try {
158 int value = fis.read();
159 if (value >= 0) {
160 this.position++;
161 }
162 return value;
163 } catch (IOException e) { // unexpected exception
164 throw new FSError(e); // assume native fs error
165 }
166 }
167
168 @Override
169 public int read(byte[] b, int off, int len) throws IOException {
170 try {
171 int value = fis.read(b, off, len);
172 if (value > 0) {
173 this.position += value;
174 }
175 return value;
176 } catch (IOException e) { // unexpected exception
177 throw new FSError(e); // assume native fs error
178 }
179 }
180
181 @Override
182 public int read(long position, byte[] b, int off, int len)
183 throws IOException {
184 ByteBuffer bb = ByteBuffer.wrap(b, off, len);
185 try {
186 return fis.getChannel().read(bb, position);
187 } catch (IOException e) {
188 throw new FSError(e);
189 }
190 }
191
192 @Override
193 public long skip(long n) throws IOException {
194 long value = fis.skip(n);
195 if (value > 0) {
196 this.position += value;
197 }
198 return value;
199 }
200
201 @Override
202 public FileDescriptor getFileDescriptor() throws IOException {
203 return fis.getFD();
204 }
205 }
206
207 @Override
208 public FSDataInputStream open(Path f, int bufferSize) throws IOException {
209 if (!exists(f)) {
210 throw new FileNotFoundException(f.toString());
211 }
212 return new FSDataInputStream(new BufferedFSInputStream(
213 new LocalFSFileInputStream(f), bufferSize));
214 }
215
216 /*********************************************************
217 * For create()'s FSOutputStream.
218 *********************************************************/
219 class LocalFSFileOutputStream extends OutputStream {
220 private FileOutputStream fos;
221
222 private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
223 this.fos = new FileOutputStream(pathToFile(f), append);
224 }
225
226 /*
227 * Just forward to the fos
228 */
229 @Override
230 public void close() throws IOException { fos.close(); }
231 @Override
232 public void flush() throws IOException { fos.flush(); }
233 @Override
234 public void write(byte[] b, int off, int len) throws IOException {
235 try {
236 fos.write(b, off, len);
237 } catch (IOException e) { // unexpected exception
238 throw new FSError(e); // assume native fs error
239 }
240 }
241
242 @Override
243 public void write(int b) throws IOException {
244 try {
245 fos.write(b);
246 } catch (IOException e) { // unexpected exception
247 throw new FSError(e); // assume native fs error
248 }
249 }
250 }
251
252 @Override
253 public FSDataOutputStream append(Path f, int bufferSize,
254 Progressable progress) throws IOException {
255 if (!exists(f)) {
256 throw new FileNotFoundException("File " + f + " not found");
257 }
258 if (getFileStatus(f).isDirectory()) {
259 throw new IOException("Cannot append to a diretory (=" + f + " )");
260 }
261 return new FSDataOutputStream(new BufferedOutputStream(
262 new LocalFSFileOutputStream(f, true), bufferSize), statistics);
263 }
264
265 @Override
266 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
267 short replication, long blockSize, Progressable progress)
268 throws IOException {
269 return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
270 }
271
272 private FSDataOutputStream create(Path f, boolean overwrite,
273 boolean createParent, int bufferSize, short replication, long blockSize,
274 Progressable progress) throws IOException {
275 if (exists(f) && !overwrite) {
276 throw new IOException("File already exists: "+f);
277 }
278 Path parent = f.getParent();
279 if (parent != null && !mkdirs(parent)) {
280 throw new IOException("Mkdirs failed to create " + parent.toString());
281 }
282 return new FSDataOutputStream(new BufferedOutputStream(
283 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
284 }
285
286 @Override
287 @Deprecated
288 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
289 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
290 Progressable progress) throws IOException {
291 if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
292 throw new IOException("File already exists: "+f);
293 }
294 return new FSDataOutputStream(new BufferedOutputStream(
295 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
296 }
297
298 @Override
299 public FSDataOutputStream create(Path f, FsPermission permission,
300 boolean overwrite, int bufferSize, short replication, long blockSize,
301 Progressable progress) throws IOException {
302
303 FSDataOutputStream out = create(f,
304 overwrite, bufferSize, replication, blockSize, progress);
305 setPermission(f, permission);
306 return out;
307 }
308
309 @Override
310 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
311 boolean overwrite,
312 int bufferSize, short replication, long blockSize,
313 Progressable progress) throws IOException {
314 FSDataOutputStream out = create(f,
315 overwrite, false, bufferSize, replication, blockSize, progress);
316 setPermission(f, permission);
317 return out;
318 }
319
320 @Override
321 public boolean rename(Path src, Path dst) throws IOException {
322 // Attempt rename using Java API.
323 File srcFile = pathToFile(src);
324 File dstFile = pathToFile(dst);
325 if (srcFile.renameTo(dstFile)) {
326 return true;
327 }
328
329 // Enforce POSIX rename behavior that a source directory replaces an existing
330 // destination if the destination is an empty directory. On most platforms,
331 // this is already handled by the Java API call above. Some platforms
332 // (notably Windows) do not provide this behavior, so the Java API call above
333 // fails. Delete destination and attempt rename again.
334 if (this.exists(dst)) {
335 FileStatus sdst = this.getFileStatus(dst);
336 if (sdst.isDirectory() && dstFile.list().length == 0) {
337 if (LOG.isDebugEnabled()) {
338 LOG.debug("Deleting empty destination and renaming " + src + " to " +
339 dst);
340 }
341 if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
342 return true;
343 }
344 }
345 }
346
347 // The fallback behavior accomplishes the rename by a full copy.
348 if (LOG.isDebugEnabled()) {
349 LOG.debug("Falling through to a copy of " + src + " to " + dst);
350 }
351 return FileUtil.copy(this, src, this, dst, true, getConf());
352 }
353
354 /**
355 * Delete the given path to a file or directory.
356 * @param p the path to delete
357 * @param recursive to delete sub-directories
358 * @return true if the file or directory and all its contents were deleted
359 * @throws IOException if p is non-empty and recursive is false
360 */
361 @Override
362 public boolean delete(Path p, boolean recursive) throws IOException {
363 File f = pathToFile(p);
364 if (f.isFile()) {
365 return f.delete();
366 } else if (!recursive && f.isDirectory() &&
367 (FileUtil.listFiles(f).length != 0)) {
368 throw new IOException("Directory " + f.toString() + " is not empty");
369 }
370 return FileUtil.fullyDelete(f);
371 }
372
373 @Override
374 public FileStatus[] listStatus(Path f) throws IOException {
375 File localf = pathToFile(f);
376 FileStatus[] results;
377
378 if (!localf.exists()) {
379 throw new FileNotFoundException("File " + f + " does not exist");
380 }
381 if (localf.isFile()) {
382 return new FileStatus[] {
383 new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
384 }
385
386 String[] names = localf.list();
387 if (names == null) {
388 return null;
389 }
390 results = new FileStatus[names.length];
391 int j = 0;
392 for (int i = 0; i < names.length; i++) {
393 try {
394 // Assemble the path using the Path 3 arg constructor to make sure
395 // paths with colon are properly resolved on Linux
396 results[j] = getFileStatus(new Path(f, new Path(null, null, names[i])));
397 j++;
398 } catch (FileNotFoundException e) {
399 // ignore the files not found since the dir list may have have changed
400 // since the names[] list was generated.
401 }
402 }
403 if (j == names.length) {
404 return results;
405 }
406 return Arrays.copyOf(results, j);
407 }
408
409 /**
410 * Creates the specified directory hierarchy. Does not
411 * treat existence as an error.
412 */
413 @Override
414 public boolean mkdirs(Path f) throws IOException {
415 if(f == null) {
416 throw new IllegalArgumentException("mkdirs path arg is null");
417 }
418 Path parent = f.getParent();
419 File p2f = pathToFile(f);
420 if(parent != null) {
421 File parent2f = pathToFile(parent);
422 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
423 throw new FileAlreadyExistsException("Parent path is not a directory: "
424 + parent);
425 }
426 }
427 return (parent == null || mkdirs(parent)) &&
428 (p2f.mkdir() || p2f.isDirectory());
429 }
430
431 @Override
432 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
433 boolean b = mkdirs(f);
434 if(b) {
435 setPermission(f, permission);
436 }
437 return b;
438 }
439
440
441 @Override
442 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
443 throws IOException {
444 boolean b = mkdirs(f);
445 setPermission(f, absolutePermission);
446 return b;
447 }
448
449
450 @Override
451 public Path getHomeDirectory() {
452 return this.makeQualified(new Path(System.getProperty("user.home")));
453 }
454
455 /**
456 * Set the working directory to the given directory.
457 */
458 @Override
459 public void setWorkingDirectory(Path newDir) {
460 workingDir = makeAbsolute(newDir);
461 checkPath(workingDir);
462
463 }
464
465 @Override
466 public Path getWorkingDirectory() {
467 return workingDir;
468 }
469
470 @Override
471 protected Path getInitialWorkingDirectory() {
472 return this.makeQualified(new Path(System.getProperty("user.dir")));
473 }
474
475 @Override
476 public FsStatus getStatus(Path p) throws IOException {
477 File partition = pathToFile(p == null ? new Path("/") : p);
478 //File provides getUsableSpace() and getFreeSpace()
479 //File provides no API to obtain used space, assume used = total - free
480 return new FsStatus(partition.getTotalSpace(),
481 partition.getTotalSpace() - partition.getFreeSpace(),
482 partition.getFreeSpace());
483 }
484
485 // In the case of the local filesystem, we can just rename the file.
486 @Override
487 public void moveFromLocalFile(Path src, Path dst) throws IOException {
488 rename(src, dst);
489 }
490
491 // We can write output directly to the final location
492 @Override
493 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
494 throws IOException {
495 return fsOutputFile;
496 }
497
498 // It's in the right place - nothing to do.
499 @Override
500 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
501 throws IOException {
502 }
503
504 @Override
505 public void close() throws IOException {
506 super.close();
507 }
508
509 @Override
510 public String toString() {
511 return "LocalFS";
512 }
513
514 @Override
515 public FileStatus getFileStatus(Path f) throws IOException {
516 File path = pathToFile(f);
517 if (path.exists()) {
518 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
519 } else {
520 throw new FileNotFoundException("File " + f + " does not exist");
521 }
522 }
523
524 static class RawLocalFileStatus extends FileStatus {
525 /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
526 * We recognize if the information is already loaded by check if
527 * onwer.equals("").
528 */
529 private boolean isPermissionLoaded() {
530 return !super.getOwner().isEmpty();
531 }
532
533 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
534 super(f.length(), f.isDirectory(), 1, defaultBlockSize,
535 f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
536 fs.getWorkingDirectory()));
537 }
538
539 @Override
540 public FsPermission getPermission() {
541 if (!isPermissionLoaded()) {
542 loadPermissionInfo();
543 }
544 return super.getPermission();
545 }
546
547 @Override
548 public String getOwner() {
549 if (!isPermissionLoaded()) {
550 loadPermissionInfo();
551 }
552 return super.getOwner();
553 }
554
555 @Override
556 public String getGroup() {
557 if (!isPermissionLoaded()) {
558 loadPermissionInfo();
559 }
560 return super.getGroup();
561 }
562
563 /// loads permissions, owner, and group from `ls -ld`
564 private void loadPermissionInfo() {
565 IOException e = null;
566 try {
567 String output = FileUtil.execCommand(new File(getPath().toUri()),
568 Shell.getGetPermissionCommand());
569 StringTokenizer t =
570 new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
571 //expected format
572 //-rw------- 1 username groupname ...
573 String permission = t.nextToken();
574 if (permission.length() > 10) { //files with ACLs might have a '+'
575 permission = permission.substring(0, 10);
576 }
577 setPermission(FsPermission.valueOf(permission));
578 t.nextToken();
579
580 String owner = t.nextToken();
581 // If on windows domain, token format is DOMAIN\\user and we want to
582 // extract only the user name
583 if (Shell.WINDOWS) {
584 int i = owner.indexOf('\\');
585 if (i != -1)
586 owner = owner.substring(i + 1);
587 }
588 setOwner(owner);
589
590 setGroup(t.nextToken());
591 } catch (Shell.ExitCodeException ioe) {
592 if (ioe.getExitCode() != 1) {
593 e = ioe;
594 } else {
595 setPermission(null);
596 setOwner(null);
597 setGroup(null);
598 }
599 } catch (IOException ioe) {
600 e = ioe;
601 } finally {
602 if (e != null) {
603 throw new RuntimeException("Error while running command to get " +
604 "file permissions : " +
605 StringUtils.stringifyException(e));
606 }
607 }
608 }
609
610 @Override
611 public void write(DataOutput out) throws IOException {
612 if (!isPermissionLoaded()) {
613 loadPermissionInfo();
614 }
615 super.write(out);
616 }
617 }
618
619 /**
620 * Use the command chown to set owner.
621 */
622 @Override
623 public void setOwner(Path p, String username, String groupname)
624 throws IOException {
625 FileUtil.setOwner(pathToFile(p), username, groupname);
626 }
627
628 /**
629 * Use the command chmod to set permission.
630 */
631 @Override
632 public void setPermission(Path p, FsPermission permission)
633 throws IOException {
634 if (NativeIO.isAvailable()) {
635 NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
636 permission.toShort());
637 } else {
638 String perm = String.format("%04o", permission.toShort());
639 Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
640 FileUtil.makeShellPath(pathToFile(p), true)));
641 }
642 }
643
644 /**
645 * Sets the {@link Path}'s last modified time <em>only</em> to the given
646 * valid time.
647 *
648 * @param mtime the modification time to set (only if greater than zero).
649 * @param atime currently ignored.
650 * @throws IOException if setting the last modified time fails.
651 */
652 @Override
653 public void setTimes(Path p, long mtime, long atime) throws IOException {
654 File f = pathToFile(p);
655 if(mtime >= 0) {
656 if(!f.setLastModified(mtime)) {
657 throw new IOException(
658 "couldn't set last-modified time to " +
659 mtime +
660 " for " +
661 f.getAbsolutePath());
662 }
663 }
664 }
665
666 }