MesosSchedulerUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def checkPorts(conf: SparkConf, ports: List[(Long, Long)]): Boolean

Checks executor ports if they are within some range of the offered list of ports ranges,
Checks executor ports if they are within some range of the offered list of ports ranges,
conf
the Spark Config
ports
the list of ports to check
returns
true if ports are within range false otherwise

Attributes
protected
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createResource(resourceName: String, quantity: Double): Resource

Build a Mesos resource protobuf object
Build a Mesos resource protobuf object

Attributes
protected
def createResource(name: String, amount: Double, role: Option[String] = None): Resource
def createSchedulerDriver(masterUrl: String, scheduler: Scheduler, sparkUser: String, appName: String, conf: SparkConf, webuiUrl: Option[String] = None, checkpoint: Option[Boolean] = None, failoverTimeout: Option[Double] = None, frameworkId: Option[String] = None): SchedulerDriver

Creates a new MesosSchedulerDriver that communicates to the Mesos master.
Creates a new MesosSchedulerDriver that communicates to the Mesos master.
masterUrl
The url to connect to Mesos master
scheduler
the scheduler class to receive scheduler callbacks
sparkUser
User to impersonate with when running tasks
appName
The framework name to display on the Mesos UI
conf
Spark configuration
webuiUrl
The WebUI url to link from Mesos UI
checkpoint
Option to checkpoint tasks for failover
failoverTimeout
Duration Mesos master expect scheduler to reconnect on disconnect
frameworkId
The id of the new framework

Attributes
protected
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def executorMemory(sc: SparkContext): Int

Return the amount of memory to allocate to each executor, taking into account container overheads.
Return the amount of memory to allocate to each executor, taking into account container overheads.
sc
SparkContext to use to get spark.mesos.executor.memoryOverhead value
returns
memory requirement as (0.1 * <memoryOverhead>) or MEMORY_OVERHEAD_MINIMUM (whichever is larger)
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getAttribute(attr: Attribute): (String, Set[String])

Helper method to get the key,value-set pair for a Mesos Attribute protobuf
Helper method to get the key,value-set pair for a Mesos Attribute protobuf

Attributes
protected
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getRangeResource(res: List[Resource], name: String): List[(Long, Long)]

Transforms a range resource to a list of ranges
Transforms a range resource to a list of ranges
res
the mesos resource list
name
the name of the resource
returns
the list of ranges returned

Attributes
protected
def getRejectOfferDurationForReachedMaxCores(sc: SparkContext): Long

Attributes
protected
def getRejectOfferDurationForUnmetConstraints(sc: SparkContext): Long

Attributes
protected
def getResource(res: List[Resource], name: String): Double
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
val managedPortNames: List[String]
def markErr(): Unit

Attributes
protected
def markRegistered(): Unit

Signal that the scheduler has registered with Mesos.
Signal that the scheduler has registered with Mesos.

Attributes
protected
def matchesAttributeRequirements(slaveOfferConstraints: Map[String, Set[String]], offerAttributes: Map[String, GeneratedMessage]): Boolean

Match the requirements (if any) to the offer attributes.
Match the requirements (if any) to the offer attributes. if attribute requirements are not specified - return true else if attribute is defined and no values are given, simple attribute presence is performed else if attribute name and value is specified, subset match is performed on slave attributes
var mesosDriver: SchedulerDriver

Attributes
protected
def mesosToTaskState(state: mesos.Protos.TaskState): TaskState.TaskState
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def nonZeroPortValuesFromConfig(conf: SparkConf): List[Long]

The values of the non-zero ports to be used by the executor process.
The values of the non-zero ports to be used by the executor process.
conf
the spark config to use
returns
the ono-zero values of the ports
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def parseConstraintString(constraintsVal: String): Map[String, Set[String]]

Parses the attributes constraints provided to spark and build a matching data struct: Map[<attribute-name>, Set[values-to-match]] The constraints are specified as ';' separated key-value pairs where keys and values are separated by ':'.
Parses the attributes constraints provided to spark and build a matching data struct: Map[<attribute-name>, Set[values-to-match]] The constraints are specified as ';' separated key-value pairs where keys and values are separated by ':'. The ':' implies equality (for singular values) and "is one of" for multiple values (comma separated). For example:
```
parseConstraintString("os:centos7;zone:us-east-1a,us-east-1b")
// would result in
<code>
Map(
  "os" -> Set("centos7"),
  "zone":   -> Set("us-east-1a", "us-east-1b")
)
```
Mesos documentation: http://mesos.apache.org/documentation/attributes-resources/ https://github.com/apache/mesos/blob/master/src/common/values.cpp https://github.com/apache/mesos/blob/master/src/common/attributes.cpp
constraintsVal
constaints string consisting of ';' separated key-value pairs (separated by ':')
returns
Map of constraints to match resources offers.
def partitionPortResources(requestedPorts: List[Long], offeredResources: List[Resource]): (List[Resource], List[Resource])

Partitions port resources.
Partitions port resources.
requestedPorts
non-zero ports to assign
offeredResources
the resources offered
returns
resources left, port resources to be used.
def partitionResources(resources: List[Resource], resourceName: String, amountToUse: Double): (List[Resource], List[Resource])

Partition the existing set of resources into two groups, those remaining to be scheduled and those requested to be used for a new task.
Partition the existing set of resources into two groups, those remaining to be scheduled and those requested to be used for a new task.
resources
The full list of available resources
resourceName
The name of the resource to take from the available resources
amountToUse
The amount of resources to take from the available resources
returns
The remaining resources list and the used resources list.
def setupUris(uris: String, builder: Builder, useFetcherCache: Boolean = false): Unit
def startScheduler(newDriver: SchedulerDriver): Unit

Starts the MesosSchedulerDriver and stores the current running driver to this new instance.
Starts the MesosSchedulerDriver and stores the current running driver to this new instance. This driver is expected to not be running. This method returns only after the scheduler has registered with Mesos.
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def taskStateToMesos(state: TaskState.TaskState): mesos.Protos.TaskState
def toAttributeMap(offerAttributes: List[Attribute]): Map[String, GeneratedMessage]

Converts the attributes from the resource offer into a Map of name -> Attribute Value The attribute values are the mesos attribute types and they are
Converts the attributes from the resource offer into a Map of name -> Attribute Value The attribute values are the mesos attribute types and they are
offerAttributes
the attributes offered

Attributes
protected
def toString(): String

Definition Classes
AnyRef → Any
def unsetFrameworkID(sc: SparkContext): Unit

spark.mesos.driver.frameworkId is set by the cluster dispatcher to correlate driver submissions with frameworkIDs.
spark.mesos.driver.frameworkId is set by the cluster dispatcher to correlate driver submissions with frameworkIDs. However, this causes issues when a driver process launches more than one framework (more than one SparkContext(, because they all try to register with the same frameworkID. To enforce that only the first driver registers with the configured framework ID, the driver calls this method after the first registration.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package mesos

trait MesosSchedulerUtils extends Logging

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def checkPorts(conf: SparkConf, ports: List[(Long, Long)]): Boolean

def clone(): AnyRef

def createResource(resourceName: String, quantity: Double): Resource

def createResource(name: String, amount: Double, role: Option[String] = None): Resource

def createSchedulerDriver(masterUrl: String, scheduler: Scheduler, sparkUser: String, appName: String, conf: SparkConf, webuiUrl: Option[String] = None, checkpoint: Option[Boolean] = None, failoverTimeout: Option[Double] = None, frameworkId: Option[String] = None): SchedulerDriver

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executorMemory(sc: SparkContext): Int

def finalize(): Unit

def getAttribute(attr: Attribute): (String, Set[String])

final def getClass(): Class[_]

def getRangeResource(res: List[Resource], name: String): List[(Long, Long)]

def getRejectOfferDurationForReachedMaxCores(sc: SparkContext): Long

def getRejectOfferDurationForUnmetConstraints(sc: SparkContext): Long

def getResource(res: List[Resource], name: String): Double

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

val managedPortNames: List[String]

def markErr(): Unit

def markRegistered(): Unit

def matchesAttributeRequirements(slaveOfferConstraints: Map[String, Set[String]], offerAttributes: Map[String, GeneratedMessage]): Boolean

var mesosDriver: SchedulerDriver

def mesosToTaskState(state: mesos.Protos.TaskState): TaskState.TaskState

final def ne(arg0: AnyRef): Boolean

def nonZeroPortValuesFromConfig(conf: SparkConf): List[Long]

final def notify(): Unit

final def notifyAll(): Unit

def parseConstraintString(constraintsVal: String): Map[String, Set[String]]

def partitionPortResources(requestedPorts: List[Long], offeredResources: List[Resource]): (List[Resource], List[Resource])

def partitionResources(resources: List[Resource], resourceName: String, amountToUse: Double): (List[Resource], List[Resource])

def setupUris(uris: String, builder: Builder, useFetcherCache: Boolean = false): Unit

def startScheduler(newDriver: SchedulerDriver): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def taskStateToMesos(state: TaskState.TaskState): mesos.Protos.TaskState

def toAttributeMap(offerAttributes: List[Attribute]): Map[String, GeneratedMessage]

def toString(): String

def unsetFrameworkID(sc: SparkContext): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped