001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.BufferedOutputStream;
022 import java.io.DataOutput;
023 import java.io.File;
024 import java.io.FileInputStream;
025 import java.io.FileNotFoundException;
026 import java.io.FileOutputStream;
027 import java.io.IOException;
028 import java.io.OutputStream;
029 import java.io.FileDescriptor;
030 import java.net.URI;
031 import java.nio.ByteBuffer;
032 import java.util.Arrays;
033 import java.util.StringTokenizer;
034
035 import org.apache.hadoop.classification.InterfaceAudience;
036 import org.apache.hadoop.classification.InterfaceStability;
037 import org.apache.hadoop.conf.Configuration;
038 import org.apache.hadoop.fs.permission.FsPermission;
039 import org.apache.hadoop.io.nativeio.NativeIO;
040 import org.apache.hadoop.util.Progressable;
041 import org.apache.hadoop.util.Shell;
042 import org.apache.hadoop.util.StringUtils;
043
044 /****************************************************************
045 * Implement the FileSystem API for the raw local filesystem.
046 *
047 *****************************************************************/
048 @InterfaceAudience.Public
049 @InterfaceStability.Stable
050 public class RawLocalFileSystem extends FileSystem {
051 static final URI NAME = URI.create("file:///");
052 private Path workingDir;
053
054 public RawLocalFileSystem() {
055 workingDir = getInitialWorkingDirectory();
056 }
057
058 private Path makeAbsolute(Path f) {
059 if (f.isAbsolute()) {
060 return f;
061 } else {
062 return new Path(workingDir, f);
063 }
064 }
065
066 /** Convert a path to a File. */
067 public File pathToFile(Path path) {
068 checkPath(path);
069 if (!path.isAbsolute()) {
070 path = new Path(getWorkingDirectory(), path);
071 }
072 return new File(path.toUri().getPath());
073 }
074
075 @Override
076 public URI getUri() { return NAME; }
077
078 @Override
079 public void initialize(URI uri, Configuration conf) throws IOException {
080 super.initialize(uri, conf);
081 setConf(conf);
082 }
083
084 class TrackingFileInputStream extends FileInputStream {
085 public TrackingFileInputStream(File f) throws IOException {
086 super(f);
087 }
088
089 @Override
090 public int read() throws IOException {
091 int result = super.read();
092 if (result != -1) {
093 statistics.incrementBytesRead(1);
094 }
095 return result;
096 }
097
098 @Override
099 public int read(byte[] data) throws IOException {
100 int result = super.read(data);
101 if (result != -1) {
102 statistics.incrementBytesRead(result);
103 }
104 return result;
105 }
106
107 @Override
108 public int read(byte[] data, int offset, int length) throws IOException {
109 int result = super.read(data, offset, length);
110 if (result != -1) {
111 statistics.incrementBytesRead(result);
112 }
113 return result;
114 }
115 }
116
117 /*******************************************************
118 * For open()'s FSInputStream.
119 *******************************************************/
120 class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor {
121 private FileInputStream fis;
122 private long position;
123
124 public LocalFSFileInputStream(Path f) throws IOException {
125 this.fis = new TrackingFileInputStream(pathToFile(f));
126 }
127
128 @Override
129 public void seek(long pos) throws IOException {
130 fis.getChannel().position(pos);
131 this.position = pos;
132 }
133
134 @Override
135 public long getPos() throws IOException {
136 return this.position;
137 }
138
139 @Override
140 public boolean seekToNewSource(long targetPos) throws IOException {
141 return false;
142 }
143
144 /*
145 * Just forward to the fis
146 */
147 @Override
148 public int available() throws IOException { return fis.available(); }
149 @Override
150 public void close() throws IOException { fis.close(); }
151 @Override
152 public boolean markSupported() { return false; }
153
154 @Override
155 public int read() throws IOException {
156 try {
157 int value = fis.read();
158 if (value >= 0) {
159 this.position++;
160 }
161 return value;
162 } catch (IOException e) { // unexpected exception
163 throw new FSError(e); // assume native fs error
164 }
165 }
166
167 @Override
168 public int read(byte[] b, int off, int len) throws IOException {
169 try {
170 int value = fis.read(b, off, len);
171 if (value > 0) {
172 this.position += value;
173 }
174 return value;
175 } catch (IOException e) { // unexpected exception
176 throw new FSError(e); // assume native fs error
177 }
178 }
179
180 @Override
181 public int read(long position, byte[] b, int off, int len)
182 throws IOException {
183 ByteBuffer bb = ByteBuffer.wrap(b, off, len);
184 try {
185 return fis.getChannel().read(bb, position);
186 } catch (IOException e) {
187 throw new FSError(e);
188 }
189 }
190
191 @Override
192 public long skip(long n) throws IOException {
193 long value = fis.skip(n);
194 if (value > 0) {
195 this.position += value;
196 }
197 return value;
198 }
199
200 @Override
201 public FileDescriptor getFileDescriptor() throws IOException {
202 return fis.getFD();
203 }
204 }
205
206 @Override
207 public FSDataInputStream open(Path f, int bufferSize) throws IOException {
208 if (!exists(f)) {
209 throw new FileNotFoundException(f.toString());
210 }
211 return new FSDataInputStream(new BufferedFSInputStream(
212 new LocalFSFileInputStream(f), bufferSize));
213 }
214
215 /*********************************************************
216 * For create()'s FSOutputStream.
217 *********************************************************/
218 class LocalFSFileOutputStream extends OutputStream {
219 private FileOutputStream fos;
220
221 private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
222 this.fos = new FileOutputStream(pathToFile(f), append);
223 }
224
225 /*
226 * Just forward to the fos
227 */
228 @Override
229 public void close() throws IOException { fos.close(); }
230 @Override
231 public void flush() throws IOException { fos.flush(); }
232 @Override
233 public void write(byte[] b, int off, int len) throws IOException {
234 try {
235 fos.write(b, off, len);
236 } catch (IOException e) { // unexpected exception
237 throw new FSError(e); // assume native fs error
238 }
239 }
240
241 @Override
242 public void write(int b) throws IOException {
243 try {
244 fos.write(b);
245 } catch (IOException e) { // unexpected exception
246 throw new FSError(e); // assume native fs error
247 }
248 }
249 }
250
251 @Override
252 public FSDataOutputStream append(Path f, int bufferSize,
253 Progressable progress) throws IOException {
254 if (!exists(f)) {
255 throw new FileNotFoundException("File " + f + " not found");
256 }
257 if (getFileStatus(f).isDirectory()) {
258 throw new IOException("Cannot append to a diretory (=" + f + " )");
259 }
260 return new FSDataOutputStream(new BufferedOutputStream(
261 new LocalFSFileOutputStream(f, true), bufferSize), statistics);
262 }
263
264 @Override
265 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
266 short replication, long blockSize, Progressable progress)
267 throws IOException {
268 return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
269 }
270
271 private FSDataOutputStream create(Path f, boolean overwrite,
272 boolean createParent, int bufferSize, short replication, long blockSize,
273 Progressable progress) throws IOException {
274 if (exists(f) && !overwrite) {
275 throw new IOException("File already exists: "+f);
276 }
277 Path parent = f.getParent();
278 if (parent != null && !mkdirs(parent)) {
279 throw new IOException("Mkdirs failed to create " + parent.toString());
280 }
281 return new FSDataOutputStream(new BufferedOutputStream(
282 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
283 }
284
285 @Override
286 public FSDataOutputStream create(Path f, FsPermission permission,
287 boolean overwrite, int bufferSize, short replication, long blockSize,
288 Progressable progress) throws IOException {
289
290 FSDataOutputStream out = create(f,
291 overwrite, bufferSize, replication, blockSize, progress);
292 setPermission(f, permission);
293 return out;
294 }
295
296 @Override
297 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
298 boolean overwrite,
299 int bufferSize, short replication, long blockSize,
300 Progressable progress) throws IOException {
301 FSDataOutputStream out = create(f,
302 overwrite, false, bufferSize, replication, blockSize, progress);
303 setPermission(f, permission);
304 return out;
305 }
306
307 @Override
308 public boolean rename(Path src, Path dst) throws IOException {
309 if (pathToFile(src).renameTo(pathToFile(dst))) {
310 return true;
311 }
312 return FileUtil.copy(this, src, this, dst, true, getConf());
313 }
314
315 /**
316 * Delete the given path to a file or directory.
317 * @param p the path to delete
318 * @param recursive to delete sub-directories
319 * @return true if the file or directory and all its contents were deleted
320 * @throws IOException if p is non-empty and recursive is false
321 */
322 @Override
323 public boolean delete(Path p, boolean recursive) throws IOException {
324 File f = pathToFile(p);
325 if (f.isFile()) {
326 return f.delete();
327 } else if (!recursive && f.isDirectory() &&
328 (FileUtil.listFiles(f).length != 0)) {
329 throw new IOException("Directory " + f.toString() + " is not empty");
330 }
331 return FileUtil.fullyDelete(f);
332 }
333
334 @Override
335 public FileStatus[] listStatus(Path f) throws IOException {
336 File localf = pathToFile(f);
337 FileStatus[] results;
338
339 if (!localf.exists()) {
340 throw new FileNotFoundException("File " + f + " does not exist");
341 }
342 if (localf.isFile()) {
343 return new FileStatus[] {
344 new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
345 }
346
347 String[] names = localf.list();
348 if (names == null) {
349 return null;
350 }
351 results = new FileStatus[names.length];
352 int j = 0;
353 for (int i = 0; i < names.length; i++) {
354 try {
355 results[j] = getFileStatus(new Path(f, names[i]));
356 j++;
357 } catch (FileNotFoundException e) {
358 // ignore the files not found since the dir list may have have changed
359 // since the names[] list was generated.
360 }
361 }
362 if (j == names.length) {
363 return results;
364 }
365 return Arrays.copyOf(results, j);
366 }
367
368 /**
369 * Creates the specified directory hierarchy. Does not
370 * treat existence as an error.
371 */
372 @Override
373 public boolean mkdirs(Path f) throws IOException {
374 if(f == null) {
375 throw new IllegalArgumentException("mkdirs path arg is null");
376 }
377 Path parent = f.getParent();
378 File p2f = pathToFile(f);
379 if(parent != null) {
380 File parent2f = pathToFile(parent);
381 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
382 throw new FileAlreadyExistsException("Parent path is not a directory: "
383 + parent);
384 }
385 }
386 return (parent == null || mkdirs(parent)) &&
387 (p2f.mkdir() || p2f.isDirectory());
388 }
389
390 @Override
391 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
392 boolean b = mkdirs(f);
393 if(b) {
394 setPermission(f, permission);
395 }
396 return b;
397 }
398
399
400 @Override
401 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
402 throws IOException {
403 boolean b = mkdirs(f);
404 setPermission(f, absolutePermission);
405 return b;
406 }
407
408
409 @Override
410 public Path getHomeDirectory() {
411 return this.makeQualified(new Path(System.getProperty("user.home")));
412 }
413
414 /**
415 * Set the working directory to the given directory.
416 */
417 @Override
418 public void setWorkingDirectory(Path newDir) {
419 workingDir = makeAbsolute(newDir);
420 checkPath(workingDir);
421
422 }
423
424 @Override
425 public Path getWorkingDirectory() {
426 return workingDir;
427 }
428
429 @Override
430 protected Path getInitialWorkingDirectory() {
431 return this.makeQualified(new Path(System.getProperty("user.dir")));
432 }
433
434 @Override
435 public FsStatus getStatus(Path p) throws IOException {
436 File partition = pathToFile(p == null ? new Path("/") : p);
437 //File provides getUsableSpace() and getFreeSpace()
438 //File provides no API to obtain used space, assume used = total - free
439 return new FsStatus(partition.getTotalSpace(),
440 partition.getTotalSpace() - partition.getFreeSpace(),
441 partition.getFreeSpace());
442 }
443
444 // In the case of the local filesystem, we can just rename the file.
445 @Override
446 public void moveFromLocalFile(Path src, Path dst) throws IOException {
447 rename(src, dst);
448 }
449
450 // We can write output directly to the final location
451 @Override
452 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
453 throws IOException {
454 return fsOutputFile;
455 }
456
457 // It's in the right place - nothing to do.
458 @Override
459 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
460 throws IOException {
461 }
462
463 @Override
464 public void close() throws IOException {
465 super.close();
466 }
467
468 @Override
469 public String toString() {
470 return "LocalFS";
471 }
472
473 @Override
474 public FileStatus getFileStatus(Path f) throws IOException {
475 File path = pathToFile(f);
476 if (path.exists()) {
477 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
478 } else {
479 throw new FileNotFoundException("File " + f + " does not exist");
480 }
481 }
482
483 static class RawLocalFileStatus extends FileStatus {
484 /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
485 * We recognize if the information is already loaded by check if
486 * onwer.equals("").
487 */
488 private boolean isPermissionLoaded() {
489 return !super.getOwner().equals("");
490 }
491
492 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
493 super(f.length(), f.isDirectory(), 1, defaultBlockSize,
494 f.lastModified(), fs.makeQualified(new Path(f.getPath())));
495 }
496
497 @Override
498 public FsPermission getPermission() {
499 if (!isPermissionLoaded()) {
500 loadPermissionInfo();
501 }
502 return super.getPermission();
503 }
504
505 @Override
506 public String getOwner() {
507 if (!isPermissionLoaded()) {
508 loadPermissionInfo();
509 }
510 return super.getOwner();
511 }
512
513 @Override
514 public String getGroup() {
515 if (!isPermissionLoaded()) {
516 loadPermissionInfo();
517 }
518 return super.getGroup();
519 }
520
521 /// loads permissions, owner, and group from `ls -ld`
522 private void loadPermissionInfo() {
523 IOException e = null;
524 try {
525 StringTokenizer t = new StringTokenizer(
526 execCommand(new File(getPath().toUri()),
527 Shell.getGET_PERMISSION_COMMAND()));
528 //expected format
529 //-rw------- 1 username groupname ...
530 String permission = t.nextToken();
531 if (permission.length() > 10) { //files with ACLs might have a '+'
532 permission = permission.substring(0, 10);
533 }
534 setPermission(FsPermission.valueOf(permission));
535 t.nextToken();
536 setOwner(t.nextToken());
537 setGroup(t.nextToken());
538 } catch (Shell.ExitCodeException ioe) {
539 if (ioe.getExitCode() != 1) {
540 e = ioe;
541 } else {
542 setPermission(null);
543 setOwner(null);
544 setGroup(null);
545 }
546 } catch (IOException ioe) {
547 e = ioe;
548 } finally {
549 if (e != null) {
550 throw new RuntimeException("Error while running command to get " +
551 "file permissions : " +
552 StringUtils.stringifyException(e));
553 }
554 }
555 }
556
557 @Override
558 public void write(DataOutput out) throws IOException {
559 if (!isPermissionLoaded()) {
560 loadPermissionInfo();
561 }
562 super.write(out);
563 }
564 }
565
566 /**
567 * Use the command chown to set owner.
568 */
569 @Override
570 public void setOwner(Path p, String username, String groupname)
571 throws IOException {
572 if (username == null && groupname == null) {
573 throw new IOException("username == null && groupname == null");
574 }
575
576 if (username == null) {
577 execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname);
578 } else {
579 //OWNER[:[GROUP]]
580 String s = username + (groupname == null? "": ":" + groupname);
581 execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
582 }
583 }
584
585 /**
586 * Use the command chmod to set permission.
587 */
588 @Override
589 public void setPermission(Path p, FsPermission permission)
590 throws IOException {
591 if (NativeIO.isAvailable()) {
592 NativeIO.chmod(pathToFile(p).getCanonicalPath(),
593 permission.toShort());
594 } else {
595 execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
596 String.format("%05o", permission.toShort()));
597 }
598 }
599
600 private static String execCommand(File f, String... cmd) throws IOException {
601 String[] args = new String[cmd.length + 1];
602 System.arraycopy(cmd, 0, args, 0, cmd.length);
603 args[cmd.length] = FileUtil.makeShellPath(f, true);
604 String output = Shell.execCommand(args);
605 return output;
606 }
607
608 }