SparkHadoopUtil

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def addCredentials(conf: JobConf): Unit

   

   Add any user credentials to the job conf which are necessary for running on a secure Hadoop cluster.

5. def addCurrentUserCredentials(creds: Credentials): Unit

   

6. def addSecretKeyToUserCredentials(key: String, secret: String): Unit

   

7. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

8. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

9. val conf: Configuration

   

10. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

11. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

12. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

13. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

14. def getConfigurationFromJobContext(context: JobContext): Configuration

    

    Using reflection to get the Configuration from JobContext/TaskAttemptContext.

    Using reflection to get the Configuration from JobContext/TaskAttemptContext. If we directly call JobContext/TaskAttemptContext.getConfiguration, it will generate different byte codes for Hadoop 1.+ and Hadoop 2.+ because JobContext/TaskAttemptContext is class in Hadoop 1.+ while it's interface in Hadoop 2.+.

15. def getCurrentUserCredentials(): Credentials

    

16. def getSecretKeyFromUserCredentials(key: String): Array[Byte]

    

17. def getTimeFromNowToRenewal(sparkConf: SparkConf, fraction: Double, credentials: Credentials): Long

    

    How much time is remaining (in millis) from now to (fraction * renewal time for the token that is valid the latest)? This will return -ve (or 0) value if the fraction of validity has already expired.

18. def globPath(pattern: Path): Seq[Path]

    

19. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

22. def isYarnMode(): Boolean

    

23. def listFilesSorted(remoteFs: FileSystem, dir: Path, prefix: String, exclusionSuffix: String): Array[FileStatus]

    

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix.

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix. The returned {{FileStatus}} instances are sorted by the modification times of the respective files.

24. def listLeafDirStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

25. def listLeafDirStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

26. def listLeafStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

27. def listLeafStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

28. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

29. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

30. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

31. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

32. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

33. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

34. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

35. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

36. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

37. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

38. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

39. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

40. def loginUserFromKeytab(principalName: String, keytabFilename: String): Unit

    

41. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

42. def newConfiguration(conf: SparkConf): Configuration

    

    Return an appropriate (subclass) of Configuration.

    Return an appropriate (subclass) of Configuration. Creating config can initializes some Hadoop subsystems.

43. def newConfiguration(): Configuration

    

    Annotations

    @Deprecated

44. final def notify(): Unit

    

    Definition Classes

    AnyRef

45. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

46. def runAsSparkUser(func: () ⇒ Unit): Unit

    

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    IMPORTANT NOTE: If this function is going to be called repeated in the same process you need to look https://issues.apache.org/jira/browse/HDFS-3545 and possibly do a FileSystem.closeAllForUGI in order to avoid leaking Filesystems

47. def substituteHadoopVariables(text: String, hadoopConf: Configuration): String

    

    Substitute variables by looking them up in Hadoop configs.

    Substitute variables by looking them up in Hadoop configs. Only variables that match the ${hadoopconf- .. } pattern are substituted.

48. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

49. def toString(): String

    

    Definition Classes

    AnyRef → Any

50. def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation): Unit

    

51. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

52. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

53. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )