SparkHadoopUtil

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def addCredentials(conf: JobConf): Unit

   

   Add any user credentials to the job conf which are necessary for running on a secure Hadoop cluster.

5. def addCurrentUserCredentials(creds: Credentials): Unit

   

6. def addSecretKeyToUserCredentials(key: String, secret: String): Unit

   

7. def appendS3AndSparkHadoopConfigurations(conf: SparkConf, hadoopConf: Configuration): Unit

   

   Appends S3-specific, spark.hadoop.*, and spark.buffer.size configurations to a Hadoop configuration.

8. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

9. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

10. val conf: Configuration

    

11. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

12. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

13. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

14. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

15. def getCurrentUserCredentials(): Credentials

    

16. def getSecretKeyFromUserCredentials(key: String): Array[Byte]

    

17. def globPath(pattern: Path): Seq[Path]

    

18. def globPathIfNecessary(pattern: Path): Seq[Path]

    

19. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

20. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

21. def isGlobPath(pattern: Path): Boolean

    

22. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

23. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

24. def isYarnMode(): Boolean

    

25. def listFilesSorted(remoteFs: FileSystem, dir: Path, prefix: String, exclusionSuffix: String): Array[FileStatus]

    

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix.

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix. The returned {{FileStatus}} instances are sorted by the modification times of the respective files.

26. def listLeafDirStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

27. def listLeafDirStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

28. def listLeafStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

29. def listLeafStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

30. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

31. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

32. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

33. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

34. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

35. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

36. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

37. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

38. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

39. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

40. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

41. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

42. def loginUserFromKeytab(principalName: String, keytabFilename: String): Unit

    

43. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

44. def newConfiguration(conf: SparkConf): Configuration

    

    Return an appropriate (subclass) of Configuration.

    Return an appropriate (subclass) of Configuration. Creating config can initializes some Hadoop subsystems.

45. final def notify(): Unit

    

    Definition Classes

    AnyRef

46. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

47. def runAsSparkUser(func: () ⇒ Unit): Unit

    

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    IMPORTANT NOTE: If this function is going to be called repeated in the same process you need to look https://issues.apache.org/jira/browse/HDFS-3545 and possibly do a FileSystem.closeAllForUGI in order to avoid leaking Filesystems

48. def substituteHadoopVariables(text: String, hadoopConf: Configuration): String

    

    Substitute variables by looking them up in Hadoop configs.

    Substitute variables by looking them up in Hadoop configs. Only variables that match the ${hadoopconf- .. } pattern are substituted.

49. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

50. def toString(): String

    

    Definition Classes

    AnyRef → Any

51. def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation): Unit

    

52. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

53. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

54. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )