Package

org.apache.spark.sql

executionmetrics

Permalink

package executionmetrics

data schema is in snake case spark doesn't gives us a native converter from snake case to camel case, we have to do that in this class.

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. executionmetrics
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. case class ComponentRuns(uid: String, component_uri: String, pipeline_uri: String, pipeline_run_uid: String, fabric_uid: String, component_name: String, interim_component_name: String, component_type: String, interim_subgraph_name: String, interim_process_id: String, interim_out_port: String, created_at: Option[Timestamp] = None, created_by: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, expired: Option[Boolean] = Some(false), run_type: Option[String], job_uri: Option[String], branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  2. case class ComponentRunsWithRunDates(componentRunsWithStatusAndInterims: Option[List[ComponentRunsWithStatusAndInterims]] = None, runDates: Option[List[RunDates]] = None) extends Product with Serializable

    Permalink

  3. case class ComponentRunsWithStatus(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], job_uri: Option[String] = None, run_type: Option[String] = None, branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  4. case class ComponentRunsWithStatusAndInterims(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], interim: Option[String] = None, job_uri: Option[String] = None, run_type: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  5. case class DatasetRunsResponseCamelCase(uid: String, datasetUID: String, datasetName: Option[String] = None, pipelineUID: String, pipelineName: Option[String] = None, fabric: Option[Fabric], submissionTime: Option[Timestamp] = None, pipelineRun: PipelineRun, componentName: String, componentType: String, recordsProcessed: Option[Long] = None, status: Option[String] = None, interims: Option[String] = None, runDates: Option[Seq[RunDates]] = None, runType: Option[String] = None, jobUID: Option[String] = None, jobName: Option[String] = None, bytes: Option[Long], partition: Option[Long], branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  6. trait ExecutionMetricsDAO[A <: ExecutionMetricsEntity] extends AnyRef

    Permalink

  7. sealed trait ExecutionMetricsEntity extends AnyRef

    Permalink

    TODO Atm we hopefully should have something related to spark which helps in expressionEncoder to keep cases in camelCase rather than snake_case.

  8. case class Fabric(uid: String, name: Option[String] = None) extends Product with Serializable

    Permalink

  9. case class FileContent(path: String, content: String) extends Product with Serializable

    Permalink

  10. case class HistoricalViewCodeResponse(pipelineId: String, pipelineRun: PipelineRunsResponseCamelCase) extends Product with Serializable

    Permalink

  11. case class InMemoryStore(uuid: String, sparkSession: SparkSession)(implicit ec: ExecutionContext) extends LazyLogging with Product with Serializable

    Permalink

  12. case class InterimResponse(uid: String, interim_component_name: String, interim_out_port: String, interim_process_id: String, interim: String) extends Product with Serializable

    Permalink

  13. case class InterimResponseCamelCase(uid: String, interimComponentName: String, interimOutPort: String, interimProcessId: String, interim: String, submissionTime: Option[Timestamp] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  14. implicit class LoggingSparkSession extends LazyLogging

    Permalink

  15. case class NextFilters(lastSubmissionTimeInMs: Timestamp, lastUid: String) extends Product with Serializable

    Permalink

  16. case class PipelineRun(uid: String) extends Product with Serializable

    Permalink

  17. case class PipelineRuns(uid: String, pipeline_uri: String, job_uri: Option[String] = None, job_run_uid: String, task_run_uid: String, status: String, fabric_uid: String, time_taken: Option[Long] = None, rows_read: Option[Long] = None, rows_written: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, run_type: String, input_datasets: Option[Array[String]] = None, output_datasets: Option[Array[String]] = None, workflow_code: Option[Map[String, String]] = None, expired: Option[Boolean] = Some(false), branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable

    Permalink

  18. case class PipelineRunsResponseCamelCase(pipelineRunId: String, pipelineUID: String, jobUID: Option[String] = None, jobName: Option[String] = None, fabric: Option[Fabric] = None, fabricId: Option[Long] = None, submissionTime: Option[Timestamp], status: String, timeTaken: Option[Long] = None, rowsRead: Option[Long] = None, rowsWritten: Option[Long] = None, runType: String, code: Option[Map[String, String]] = None, branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable

    Permalink

  19. sealed trait ResponseCamelCase extends AnyRef

    Permalink

  20. implicit class ResponseWrapperAsList[T <: ResponseCamelCase] extends AnyRef

    Permalink

  21. case class ResponsesAsList[T <: ResponseCamelCase](rows: List[T], limit: Int, next_filters: Option[NextFilters]) extends Product with Serializable

    Permalink

  22. case class RunDates(uid: String, runId: String, submissionTime: Timestamp) extends Product with Serializable

    Permalink

  23. sealed trait RunningMode extends EnumEntry

    Permalink

Value Members

  1. val Branch: String

    Permalink

  2. val Bytes: String

    Permalink

  3. val ComponentName: String

    Permalink

  4. object ComponentRuns extends Serializable

    Permalink

  5. object ComponentRunsWithStatus extends Serializable

    Permalink

  6. object ComponentRunsWithStatusAndInterims extends Serializable

    Permalink

  7. val ComponentType: String

    Permalink

  8. val ComponentUri: String

    Permalink

  9. val CreatedAt: String

    Permalink

  10. val CreatedBy: String

    Permalink

  11. object DatasetRunsResponseCamelCase extends Serializable

    Permalink

  12. val Expired: String

    Permalink

  13. object Fabric extends Serializable

    Permalink

  14. val FabricUID: String

    Permalink

  15. object FileContent extends Serializable

    Permalink

  16. object HistoricalViewCodeResponse extends Serializable

    Permalink

  17. val InputDatasets: String

    Permalink

  18. val Interim: String

    Permalink

  19. val InterimComponentName: String

    Permalink

  20. val InterimOutPort: String

    Permalink

  21. val InterimProcessId: String

    Permalink

  22. object InterimResponse extends Serializable

    Permalink

  23. object InterimResponseCamelCase extends Serializable

    Permalink

  24. val InterimSubgraphName: String

    Permalink

  25. val JobRunUid: String

    Permalink

  26. val JobUri: String

    Permalink

  27. object NextFilters extends Serializable

    Permalink

  28. val OutputDatasets: String

    Permalink

  29. val Partitions: String

    Permalink

  30. object PipelineRun extends Serializable

    Permalink

  31. val PipelineRunUid: String

    Permalink

  32. object PipelineRuns extends Serializable

    Permalink

  33. object PipelineRunsResponseCamelCase extends Serializable

    Permalink

  34. val PipelineUri: String

    Permalink

  35. val Records: String

    Permalink

  36. object ResponseCamelCase

    Permalink

  37. object ResponsesAsList extends Serializable

    Permalink

  38. val RowsRead: String

    Permalink

  39. val RowsWritten: String

    Permalink

  40. object RunDates extends Serializable

    Permalink

  41. val RunTypeColumn: String

    Permalink

  42. object RunningMode extends Enum[RunningMode] with PlayJsonEnum[RunningMode]

    Permalink

  43. val Status: String

    Permalink

  44. val TaskRunUid: String

    Permalink

  45. val TimeTaken: String

    Permalink

  46. val UID: String

    Permalink

  47. val WorkflowCode: String

    Permalink

  48. val WorkflowJson: String

    Permalink

  49. object ZipFileExtractor extends LazyLogging

    Permalink

  50. def checkExpiredRowPf[T <: ExecutionMetricsEntity](uid: String): PartialFunction[T, T]

    Permalink

  51. package componentruns

    Permalink

  52. def error(msg: String, cause: Option[Throwable] = None): Nothing

    Permalink

  53. package evolutions

    Permalink

  54. package interims

    Permalink

  55. def now(): Timestamp

    Permalink

  56. def onFail(uid: String): Nothing

    Permalink

  57. package pipelineruns

    Permalink

  58. implicit lazy val timestampFormat: Format[Timestamp]

    Permalink

Inherited from AnyRef

Inherited from Any

Ungrouped