ingest

Type Members

1. case class AuditLog(jobid: String, paths: String, domain: String, schema: String, success: Boolean, count: Long, countAccepted: Long, countRejected: Long, timestamp: Timestamp, duration: Long, message: String, step: String) extends Product with Serializable

   

2. case class ContinuousMetricRecord(domain: String, schema: String, attribute: String, min: Option[Double], max: Option[Double], mean: Option[Double], missingValues: Option[Long], standardDev: Option[Double], variance: Option[Double], sum: Option[Double], skewness: Option[Double], kurtosis: Option[Double], percentile25: Option[Double], median: Option[Double], percentile75: Option[Double], count: Long, cometTime: Long, cometStage: String, cometMetric: String, jobId: String) extends Product with Serializable

   

3. case class DiscreteMetricRecord(domain: String, schema: String, attribute: String, missingValuesDiscrete: Long, countDistinct: Long, count: Long, cometTime: Long, cometStage: String, cometMetric: String, jobId: String) extends Product with Serializable

   

4. class DsvIngestionJob extends IngestionJob

   

   Main class to ingest delimiter separated values file

5. trait DsvValidator extends AnyRef

   

6. case class FrequencyMetricRecord(domain: String, schema: String, attribute: String, category: String, frequency: Long, count: Long, cometTime: Long, cometStage: String, jobId: String) extends Product with Serializable

   

7. trait IngestionJob extends SparkJob

   

8. class JsonIngestionJob extends IngestionJob

   

   Main class to complex json delimiter separated values file If your json contains only one level simple attribute aka.

   Main class to complex json delimiter separated values file If your json contains only one level simple attribute aka. kind of dsv but in json format please use SIMPLE_JSON instead. It's way faster

9. class KafkaIngestionJob extends JsonIngestionJob

   

   Main class to ingest JSON messages from Kafka

10. case class LoadConfig(domain: String = "", schema: String = "", paths: List[Path] = Nil, options: Map[String, String] = Map.empty) extends Product with Serializable

    
    domain

    domain name of the dataset

    schema

    schema name of the dataset

    paths

    Absolute path of the file to ingest (present in the ingesting area of the domain)

11. class PositionIngestionJob extends DsvIngestionJob

    

    Main class to ingest delimiter separated values file

12. case class RejectedRecord(jobid: String, timestamp: Timestamp, domain: String, schema: String, error: String, path: String) extends Product with Serializable

    

13. class SimpleJsonIngestionJob extends DsvIngestionJob

    

    Parse a simple one level json file.

    Parse a simple one level json file. Complex types such as arrays & maps are not supported. Use JsonIngestionJob instead. This class is for simple json only that makes it way faster.

14. sealed case class Step(value: String) extends Product with Serializable

    

15. class XmlIngestionJob extends IngestionJob

    

    Main class to complex json delimiter separated values file If your json contains only one level simple attribute aka.

    Main class to complex json delimiter separated values file If your json contains only one level simple attribute aka. kind of dsv but in json format please use SIMPLE_JSON instead. It's way faster