dataquality

Type Members

1. sealed abstract class AlertImportance extends AnyRef

   

2. trait BaseEmailQualityAlert extends DataQualityAlertHandler

   

3. implicit class DataQualityActionImplicits extends AnyRef

   

4. case class DataQualityAlert(alertMessage: String, importance: AlertImportance) extends Product with Serializable

   

5. class DataQualityAlertException extends RuntimeException

   

6. trait DataQualityAlertHandler extends AnyRef

   

   Knows how to handle data quality alerts

7. trait DataQualityAlertHandlerService extends AnyRef

   

   A service for registering data quality alert handlers

8. trait DataQualityCheck[Self <: DataQualityCheck[Self]] extends AnyRef

   

   Defines a data quality check

9. trait DataQualityConfigurationExtension extends DataFlowConfigurationExtension[SparkDataFlow]

   

10. case class DataQualityMeta[CheckType <: DataQualityCheck[CheckType]](label: String, alertHandlers: Seq[DataQualityAlertHandler], check: CheckType) extends Product with Serializable

    

    Defines a data quality check to be performed on a label, along with the alert handlers to be used

    Defines a data quality check to be performed on a label, along with the alert handlers to be used

    CheckType

    the type of the data quality check

    label

    the label of the dataset on which the check should be performed

    alertHandlers

    the alert handlers to be used

    check

    the data quality check to be performed

11. case class DataQualityMetadataExtension[CheckType <: DataQualityCheck[CheckType]](meta: Seq[DataQualityMeta[CheckType]]) extends DataFlowMetadataExtension[SparkDataFlow] with Logging with Product with Serializable

    

12. case class DataQualityMetadataExtensionIdentifier[CheckType <: DataQualityCheck[CheckType]]() extends DataFlowMetadataExtensionIdentifier with Product with Serializable

    

13. trait DatasetCheck[T] extends AnyRef

    

14. case class DatasetChecks(checks: Seq[DatasetCheck[_]]) extends DataQualityCheck[DatasetChecks] with Product with Serializable

    

    A simple implementation of DataQualityCheck which doesn't perform any optimisation on check concatenation.

    A simple implementation of DataQualityCheck which doesn't perform any optimisation on check concatenation. Only use this if it's really necessary, otherwise use the Deequ implementation in the waimak-deequ module.

15. case class EmailQualityAlert(settings: EmailSettings, alertOn: List[AlertImportance] = List.empty) extends BaseEmailQualityAlert with Product with Serializable

    

    Sends alerts via email

    Sends alerts via email

    settings

    the email settings to use

    alertOn

    If specified, the list of alert importance levels to alert on. If unspecified or empty, every level will be alerted on.

16. class EmailQualityAlertService extends DataQualityAlertHandlerService

    

17. trait EmailSettings extends AnyRef

    

    Email settings used to configure an EmailQualityAlert.

    Email settings used to configure an EmailQualityAlert. Used to build a specific implementation of a settings object, such as SMTPEmailSettings

18. case class ExceptionQualityAlert(alertOn: List[AlertImportance] = List.empty) extends DataQualityAlertHandler with Product with Serializable

    

    Handles alerts by causing an exception to be thrown.

    Handles alerts by causing an exception to be thrown.

    alertOn

    If specified, the list of alert importance levels to alert on. If unspecified or empty, every level will be alerted on.

19. class ExceptionQualityAlertService extends DataQualityAlertHandlerService

    

20. case class SMTPEmailSettings(to: List[String] = List.empty, cc: List[String] = List.empty, bcc: List[String] = List.empty, from: Option[String] = None, host: String, port: Int = 25, auth: Boolean = false, starttls: Boolean = true, ssl: Boolean = false, user: Option[String] = None, pass: Option[String] = None, connectionTimeout: Long = 1000, timeout: Long = 1000, debug: Boolean = false) extends EmailSettings with Product with Serializable

    

    Email settings used to configure an EmailQualityAlert for SMTP emails

    Email settings used to configure an EmailQualityAlert for SMTP emails

    to

    (Optional) comma-separated list of 'to' destination addresses

    cc

    (Optional) comma-separated list of 'cc' destination addresses

    bcc

    (Optional) comma-separated list of 'bcc' destination addresses

    from

    (Optional) from address in email message

    host

    (Mandatory) hostname/address of email server

    port

    (Optional) port of email server, default 25

    auth

    (Optional) whether to use authentication to email server, default false

    starttls

    (Optional) whether to enable starttls when communicating with email server, default true

    ssl

    (Optional) whether to force connections only over SSL, default false

    user

    (Optional) username to use if authentication enabled

    pass

    (Optional) password to use if authentication enabled

    connectionTimeout

    (Optional) Socket connection timeout in MS, default 1000ms

    timeout

    (Optional) Socket I/O connection timeout in MS, default 1000ms

    debug

    (Optional) Whether to enable debugging on the session object, default true

21. class SimpleDatasetCheck[T] extends DatasetCheck[T]

    

22. case class SlackAttachment(title: Option[String] = None, title_link: Option[String] = None, color: Option[SlackColor] = None, ts: Option[String] = None, footer: Option[String] = None, fields: Option[Seq[SlackField]] = None) extends Product with Serializable

    

23. sealed abstract class SlackColor extends AnyRef

    

24. case class SlackField(title: String, value: String) extends Product with Serializable

    

25. case class SlackMessage(text: Option[String] = None, attachments: Option[Seq[SlackAttachment]] = None) extends Product with Serializable

    

26. case class SlackQualityAlert(token: String, alertOn: List[AlertImportance] = List.empty) extends DataQualityAlertHandler with Product with Serializable

    

    Sends alerts to a Slack channel

    Sends alerts to a Slack channel

    token

    the Slack token for the channel

    alertOn

    If specified, the list of alert importance levels to alert on. If unspecified or empty, every level will be alerted on.

27. class SlackQualityAlertService extends DataQualityAlertHandlerService