SparkHadoopUtil

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def addCredentials(conf: JobConf): Unit

   

   Add any user credentials to the job conf which are necessary for running on a secure Hadoop cluster.

5. def addCurrentUserCredentials(creds: Credentials): Unit

   

6. def addSecretKeyToUserCredentials(key: String, secret: String): Unit

   

7. def appendS3AndSparkHadoopConfigurations(conf: SparkConf, hadoopConf: Configuration): Unit

   

   Appends S3-specific, spark.hadoop.*, and spark.buffer.size configurations to a Hadoop configuration.

8. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

9. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

10. val conf: Configuration

    

11. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

12. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

13. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

14. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

15. def getCurrentUserCredentials(): Credentials

    

16. def getSecretKeyFromUserCredentials(key: String): Array[Byte]

    

17. def getTimeFromNowToRenewal(sparkConf: SparkConf, fraction: Double, credentials: Credentials): Long

    

    How much time is remaining (in millis) from now to (fraction * renewal time for the token that is valid the latest)? This will return -ve (or 0) value if the fraction of validity has already expired.

18. def globPath(pattern: Path): Seq[Path]

    

19. def globPathIfNecessary(pattern: Path): Seq[Path]

    

20. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

21. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

22. final def isDebugEnabled: Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

23. def isGlobPath(pattern: Path): Boolean

    

24. final def isInfoEnabled: Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

25. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

26. final def isTraceEnabled: Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

27. def isYarnMode(): Boolean

    

28. def listFilesSorted(remoteFs: FileSystem, dir: Path, prefix: String, exclusionSuffix: String): Array[FileStatus]

    

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix.

    Lists all the files in a directory with the specified prefix, and does not end with the given suffix. The returned {{FileStatus}} instances are sorted by the modification times of the respective files.

29. def listLeafDirStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

30. def listLeafDirStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

31. def listLeafStatuses(fs: FileSystem, baseStatus: FileStatus): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

32. def listLeafStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus]

    

    Get FileStatus objects for all leaf children (files) under the given base path.

    Get FileStatus objects for all leaf children (files) under the given base path. If the given path points to a file, return a single-element collection containing FileStatus of that file.

33. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

34. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

35. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

36. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

37. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

38. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

39. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

40. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

41. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

42. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

43. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

44. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

45. def loginUserFromKeytab(principalName: String, keytabFilename: String): Unit

    

46. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

47. def newConfiguration(conf: SparkConf): Configuration

    

    Return an appropriate (subclass) of Configuration.

    Return an appropriate (subclass) of Configuration. Creating config can initializes some Hadoop subsystems.

48. final def notify(): Unit

    

    Definition Classes

    AnyRef

49. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

50. def runAsSparkUser(func: () ⇒ Unit): Unit

    

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    Runs the given function with a Hadoop UserGroupInformation as a thread local variable (distributed to child threads), used for authenticating HDFS and YARN calls.

    IMPORTANT NOTE: If this function is going to be called repeated in the same process you need to look https://issues.apache.org/jira/browse/HDFS-3545 and possibly do a FileSystem.closeAllForUGI in order to avoid leaking Filesystems

51. def substituteHadoopVariables(text: String, hadoopConf: Configuration): String

    

    Substitute variables by looking them up in Hadoop configs.

    Substitute variables by looking them up in Hadoop configs. Only variables that match the ${hadoopconf- .. } pattern are substituted.

52. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

53. def toString(): String

    

    Definition Classes

    AnyRef → Any

54. def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation): Unit

    

55. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

56. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

57. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )