Package

org.apache.spark.sql

executionmetrics

Permalink

package executionmetrics

data schema is in snake case spark doesn't gives us a native converter from snake case to camel case, we have to do that in this class.

Linear Supertypes
LazyLogging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. executionmetrics
  2. LazyLogging
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. case class ComponentRuns(uid: String, component_uri: String, pipeline_uri: String, pipeline_run_uid: String, fabric_uid: String, component_name: String, interim_component_name: String, component_type: String, interim_subgraph_name: String, interim_process_id: String, interim_out_port: String, created_at: Option[Timestamp] = None, created_by: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, expired: Option[Boolean] = Some(false), run_type: Option[String], job_uri: Option[String], branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  2. case class ComponentRunsWithRunDates(componentRunsWithStatusAndInterims: Option[List[ComponentRunsWithStatusAndInterims]] = None, runDates: Option[List[RunDates]] = None) extends Product with Serializable

    Permalink

  3. case class ComponentRunsWithStatus(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], job_uri: Option[String] = None, run_type: Option[String] = None, branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  4. case class ComponentRunsWithStatusAndInterims(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], interim: Option[String] = None, job_uri: Option[String] = None, run_type: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  5. case class DatasetRunsResponseCamelCase(uid: String, datasetUID: String, datasetName: Option[String] = None, pipelineUID: String, pipelineName: Option[String] = None, fabric: Option[Fabric], submissionTime: Option[Timestamp] = None, pipelineRun: PipelineRun, componentName: String, componentType: String, recordsProcessed: Option[Long] = None, status: Option[String] = None, interims: Option[String] = None, runDates: Option[Seq[RunDates]] = None, runType: Option[String] = None, jobUID: Option[String] = None, jobName: Option[String] = None, bytes: Option[Long], partition: Option[Long], branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  6. trait ExecutionMetricsDAO[A <: ExecutionMetricsEntity] extends AnyRef

    Permalink

  7. sealed trait ExecutionMetricsEntity extends AnyRef

    Permalink

    TODO Atm we hopefully should have something related to spark which helps in expressionEncoder to keep cases in camelCase rather than snake_case.

  8. case class Fabric(uid: String, name: Option[String] = None) extends Product with Serializable

    Permalink

  9. case class FileContent(path: String, content: String) extends Product with Serializable

    Permalink

  10. case class HistoricalViewCodeResponse(pipelineId: String, pipelineRun: PipelineRunsResponseCamelCase) extends Product with Serializable

    Permalink

  11. case class InMemoryStore(spark: SparkSession, uuid: String) extends LazyLogging with Product with Serializable

    Permalink

  12. case class InterimPath(subgraph: String, component: String, port: String) extends Product with Serializable

    Permalink

  13. case class InterimResponse(uid: String, interim_component_name: String, interim_out_port: String, interim_process_id: String, interim: String) extends Product with Serializable

    Permalink

  14. case class InterimResponseCamelCase(uid: String, interimComponentName: String, interimOutPort: String, interimProcessId: String, interim: String, submissionTime: Option[Timestamp] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  15. implicit class LoggingSparkSession extends LazyLogging

    Permalink

  16. case class NextFilters(lastSubmissionTimeInMs: Timestamp, lastUid: String) extends Product with Serializable

    Permalink

  17. case class PipelineRun(uid: String) extends Product with Serializable

    Permalink

  18. case class PipelineRuns(uid: String, pipeline_uri: String, job_uri: Option[String] = None, job_run_uid: String, task_run_uid: String, status: String, fabric_uid: String, time_taken: Option[Long] = None, rows_read: Option[Long] = None, rows_written: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, run_type: String, input_datasets: Option[Array[String]] = None, output_datasets: Option[Array[String]] = None, workflow_code: Option[Map[String, String]] = None, expired: Option[Boolean] = Some(false), branch: Option[String] = None, pipeline_config: Option[String] = None, user_config: Option[String] = None, expected_interims: Option[Int] = None, actual_interims: Option[Int] = None, logs: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  19. case class PipelineRunsResponseCamelCase(pipelineRunId: String, pipelineUID: String, jobUID: Option[String] = None, jobName: Option[String] = None, fabric: Option[Fabric] = None, fabricId: Option[Long] = None, submissionTime: Option[Timestamp], status: String, timeTaken: Option[Long] = None, rowsRead: Option[Long] = None, rowsWritten: Option[Long] = None, runType: String, code: Option[Map[String, String]] = None, branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  20. sealed trait ResponseCamelCase extends AnyRef

    Permalink

  21. implicit class ResponseWrapperAsList[T <: ResponseCamelCase] extends AnyRef

    Permalink

  22. case class ResponsesAsList[T <: ResponseCamelCase](rows: List[T], limit: Int, next_filters: Option[NextFilters]) extends Product with Serializable

    Permalink

  23. case class RunDates(uid: String, runId: String, submissionTime: Timestamp) extends Product with Serializable

    Permalink

  24. implicit class SchemaEvolvingDataFrame extends AnyRef

    Permalink

Value Members

  1. val ActualInterims: String

    Permalink

  2. val Branch: String

    Permalink

  3. val Bytes: String

    Permalink

  4. val ComponentName: String

    Permalink

  5. object ComponentRuns extends Serializable

    Permalink

  6. object ComponentRunsWithStatus extends Serializable

    Permalink

  7. object ComponentRunsWithStatusAndInterims extends Serializable

    Permalink

  8. val ComponentType: String

    Permalink

  9. val ComponentUri: String

    Permalink

  10. val CreatedAt: String

    Permalink

  11. val CreatedBy: String

    Permalink

  12. object DatasetRunsResponseCamelCase extends Serializable

    Permalink

  13. val ExpectedInterims: String

    Permalink

  14. val Expired: String

    Permalink

  15. object Fabric extends Serializable

    Permalink

  16. val FabricUID: String

    Permalink

  17. object FileContent extends Serializable

    Permalink

  18. object HistoricalViewCodeResponse extends Serializable

    Permalink

  19. object InMemoryStore extends Serializable

    Permalink

  20. val InputDatasets: String

    Permalink

  21. val Interim: String

    Permalink

  22. val InterimComponentName: String

    Permalink

  23. val InterimOutPort: String

    Permalink

  24. object InterimPath extends Serializable

    Permalink

  25. val InterimProcessId: String

    Permalink

  26. object InterimResponse extends Serializable

    Permalink

  27. object InterimResponseCamelCase extends Serializable

    Permalink

  28. val InterimSubgraphName: String

    Permalink

  29. val JobRunUid: String

    Permalink

  30. val JobUri: String

    Permalink

  31. val Logs: String

    Permalink

  32. object NextFilters extends Serializable

    Permalink

  33. val OutputDatasets: String

    Permalink

  34. val Partitions: String

    Permalink

  35. val PipelineConfig: String

    Permalink

  36. object PipelineRun extends Serializable

    Permalink

  37. val PipelineRunUid: String

    Permalink

  38. object PipelineRunsResponseCamelCase extends Serializable

    Permalink

  39. val PipelineUri: String

    Permalink

  40. val Records: String

    Permalink

  41. object ResponseCamelCase

    Permalink

  42. object ResponsesAsList extends Serializable

    Permalink

  43. val RowsRead: String

    Permalink

  44. val RowsWritten: String

    Permalink

  45. object RunDates extends Serializable

    Permalink

  46. val RunTypeColumn: String

    Permalink

  47. val Status: String

    Permalink

  48. val TaskRunUid: String

    Permalink

  49. val TimeTaken: String

    Permalink

  50. val UID: String

    Permalink

  51. val UserConfig: String

    Permalink

  52. val WorkflowCode: String

    Permalink

  53. object ZipFileExtractor extends LazyLogging

    Permalink

  54. def checkExpiredRowPf[T <: ExecutionMetricsEntity](uid: String): PartialFunction[T, T]

    Permalink

  55. package componentruns

    Permalink

  56. def error(msg: String, cause: Option[Throwable] = None): Nothing

    Permalink

  57. package evolutions

    Permalink

  58. package interims

    Permalink

  59. def isMetricsOffloadSupported(spark: SparkSession): Boolean

    Permalink

  60. lazy val logger: Logger

    Permalink
    Attributes
    protected
    Definition Classes
    LazyLogging

  61. def now(): Timestamp

    Permalink

  62. def onFail(uid: String): Nothing

    Permalink

  63. package pipelineruns

    Permalink

  64. implicit lazy val timestampFormat: Format[Timestamp]

    Permalink

Inherited from LazyLogging

Inherited from AnyRef

Inherited from Any

Ungrouped