org
.
apache
.
spark
.
deploy
.
yarn
YarnSparkHadoopUtil
Related Docs:
object YarnSparkHadoopUtil
|
package yarn
class
YarnSparkHadoopUtil
extends
SparkHadoopUtil
Contains util methods to interact with Hadoop from spark.
Linear Supertypes
SparkHadoopUtil
,
Logging
,
AnyRef
,
Any
Ordering
Alphabetic
By Inheritance
Inherited
YarnSparkHadoopUtil
SparkHadoopUtil
Logging
AnyRef
Any
Hide All
Show All
Visibility
Public
All
Instance Constructors
new
YarnSparkHadoopUtil
()
Value Members
final
def
!=
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
final
def
##
()
:
Int
Definition Classes
AnyRef → Any
final
def
==
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
def
addCredentials
(
conf:
JobConf
)
:
Unit
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
addCurrentUserCredentials
(
creds:
Credentials
)
:
Unit
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
addSecretKeyToUserCredentials
(
key:
String
,
secret:
String
)
:
Unit
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
appendS3AndSparkHadoopConfigurations
(
conf:
SparkConf
,
hadoopConf:
Configuration
)
:
Unit
Definition Classes
SparkHadoopUtil
final
def
asInstanceOf
[
T0
]
:
T0
Definition Classes
Any
def
clone
()
:
AnyRef
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
...
)
val
conf
:
Configuration
Definition Classes
SparkHadoopUtil
final
def
eq
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
def
equals
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
def
finalize
()
:
Unit
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
classOf[java.lang.Throwable]
)
final
def
getClass
()
:
Class
[_]
Definition Classes
AnyRef → Any
def
getCurrentUserCredentials
()
:
Credentials
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
getSecretKeyFromUserCredentials
(
key:
String
)
:
Array
[
Byte
]
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
globPath
(
fs:
FileSystem
,
pattern:
Path
)
:
Seq
[
Path
]
Definition Classes
SparkHadoopUtil
def
globPath
(
pattern:
Path
)
:
Seq
[
Path
]
Definition Classes
SparkHadoopUtil
def
globPathIfNecessary
(
fs:
FileSystem
,
pattern:
Path
)
:
Seq
[
Path
]
Definition Classes
SparkHadoopUtil
def
globPathIfNecessary
(
pattern:
Path
)
:
Seq
[
Path
]
Definition Classes
SparkHadoopUtil
def
hashCode
()
:
Int
Definition Classes
AnyRef → Any
def
initializeLogIfNecessary
(
isInterpreter:
Boolean
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
isGlobPath
(
pattern:
Path
)
:
Boolean
Definition Classes
SparkHadoopUtil
final
def
isInstanceOf
[
T0
]
:
Boolean
Definition Classes
Any
def
isTraceEnabled
()
:
Boolean
Attributes
protected
Definition Classes
Logging
def
isYarnMode
()
:
Boolean
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
def
listFilesSorted
(
remoteFs:
FileSystem
,
dir:
Path
,
prefix:
String
,
exclusionSuffix:
String
)
:
Array
[
FileStatus
]
Definition Classes
SparkHadoopUtil
def
listLeafDirStatuses
(
fs:
FileSystem
,
baseStatus:
FileStatus
)
:
Seq
[
FileStatus
]
Definition Classes
SparkHadoopUtil
def
listLeafDirStatuses
(
fs:
FileSystem
,
basePath:
Path
)
:
Seq
[
FileStatus
]
Definition Classes
SparkHadoopUtil
def
listLeafStatuses
(
fs:
FileSystem
,
baseStatus:
FileStatus
)
:
Seq
[
FileStatus
]
Definition Classes
SparkHadoopUtil
def
listLeafStatuses
(
fs:
FileSystem
,
basePath:
Path
)
:
Seq
[
FileStatus
]
Definition Classes
SparkHadoopUtil
def
log
:
Logger
Attributes
protected
Definition Classes
Logging
def
logDebug
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logDebug
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logError
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logError
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logInfo
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logInfo
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logName
:
String
Attributes
protected
Definition Classes
Logging
def
logTrace
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logTrace
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logWarning
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logWarning
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
loginUserFromKeytab
(
principalName:
String
,
keytabFilename:
String
)
:
Unit
Definition Classes
SparkHadoopUtil
final
def
ne
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
def
newConfiguration
(
conf:
SparkConf
)
:
Configuration
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
final
def
notify
()
:
Unit
Definition Classes
AnyRef
final
def
notifyAll
()
:
Unit
Definition Classes
AnyRef
def
runAsSparkUser
(
func: () ⇒
Unit
)
:
Unit
Definition Classes
SparkHadoopUtil
def
substituteHadoopVariables
(
text:
String
,
hadoopConf:
Configuration
)
:
String
Definition Classes
SparkHadoopUtil
final
def
synchronized
[
T0
]
(
arg0: ⇒
T0
)
:
T0
Definition Classes
AnyRef
def
toString
()
:
String
Definition Classes
AnyRef → Any
def
transferCredentials
(
source:
UserGroupInformation
,
dest:
UserGroupInformation
)
:
Unit
Definition Classes
YarnSparkHadoopUtil
→ SparkHadoopUtil
final
def
wait
()
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
,
arg1:
Int
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
Inherited from
SparkHadoopUtil
Inherited from
Logging
Inherited from
AnyRef
Inherited from
Any
Ungrouped
Contains util methods to interact with Hadoop from spark.