Object/Class

com.spotify.ratatool.diffy

BigDiffy

Related Docs: class BigDiffy | package diffy

Permalink

object BigDiffy extends Command

Big diff between two data sets given a primary key.

Linear Supertypes
Command, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. BigDiffy
  2. Command
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. case class DeltaBigQuery(field: String, left: String, right: String, delta: DeltaValueBigQuery) extends Product with Serializable

    Permalink

  2. type DeltaSCollection = SCollection[(MultiKey, (Seq[Delta], DiffType.Value))]

    Permalink

  3. case class DeltaStatsBigQuery(deltaType: String, min: Double, max: Double, count: Long, mean: Double, variance: Double, stddev: Double, skewness: Double, kurtosis: Double) extends Product with Serializable

    Permalink

  4. case class DeltaValueBigQuery(deltaType: String, deltaValue: Option[Double]) extends Product with Serializable

    Permalink

  5. case class FieldStatsBigQuery(field: String @com.spotify.scio.bigquery.types.BigQueryTag, count: Long @com.spotify.scio.bigquery.types.BigQueryTag, fraction: Double @com.spotify.scio.bigquery.types.BigQueryTag, deltaStats: Option[DeltaStatsBigQuery]) extends HasAnnotation with Product with Serializable

    Permalink
    Annotations
    @BigQueryTag()

  6. case class GlobalStatsBigQuery(numTotal: Long @com.spotify.scio.bigquery.types.BigQueryTag, numSame: Long @com.spotify.scio.bigquery.types.BigQueryTag, numDiff: Long @com.spotify.scio.bigquery.types.BigQueryTag, numMissingLhs: Long @com.spotify.scio.bigquery.types.BigQueryTag, numMissingRhs: Long @com.spotify.scio.bigquery.types.BigQueryTag) extends HasAnnotation with Product with Serializable

    Permalink
    Annotations
    @BigQueryTag()

  7. case class KeyStatsBigQuery(key: String @com.spotify.scio.bigquery.types.BigQueryTag, diffType: String @com.spotify.scio.bigquery.types.BigQueryTag, delta: Option[DeltaBigQuery]) extends HasAnnotation with Product with Serializable

    Permalink
    Annotations
    @BigQueryTag()

  8. implicit class TextFileHeader extends AnyRef

    Permalink

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any

  3. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  4. object FieldStatsBigQuery extends HasSchema[FieldStatsBigQuery] with (String, Long, Double, Option[DeltaStatsBigQuery]) ⇒ FieldStatsBigQuery with Serializable

    Permalink

  5. object GlobalStatsBigQuery extends HasSchema[GlobalStatsBigQuery] with (Long, Long, Long, Long, Long) ⇒ GlobalStatsBigQuery with Serializable

    Permalink

  6. object KeyStatsBigQuery extends HasSchema[KeyStatsBigQuery] with (String, String, Option[DeltaBigQuery]) ⇒ KeyStatsBigQuery with Serializable

    Permalink

  7. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any

  8. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  9. val command: String

    Permalink
    Definition Classes
    BigDiffy → Command

  10. def diff[T](lhs: SCollection[T], rhs: SCollection[T], d: Diffy[T], keyFn: (T) ⇒ MultiKey, ignoreNan: Boolean = false)(implicit arg0: ClassTag[T], arg1: Coder[T]): BigDiffy[T]

    Permalink

    Diff two data sets.

  11. def diffAvro[T <: GenericRecord](sc: ScioContext, lhs: String, rhs: String, keyFn: (T) ⇒ MultiKey, diffy: AvroDiffy[T], schema: Schema = null, ignoreNan: Boolean = false)(implicit arg0: ClassTag[T], arg1: Coder[T]): BigDiffy[T]

    Permalink

    Diff two Avro data sets.

  12. def diffProtoBuf[T <: AbstractMessage](sc: ScioContext, lhs: String, rhs: String, keyFn: (T) ⇒ MultiKey, diffy: ProtoBufDiffy[T])(implicit arg0: ClassTag[T]): BigDiffy[T]

    Permalink

    Diff two ProtoBuf data sets.

  13. def diffTableRow(sc: ScioContext, lhs: String, rhs: String, keyFn: (TableRow) ⇒ MultiKey, diffy: TableRowDiffy, ignoreNan: Boolean = false): BigDiffy[TableRow]

    Permalink

    Diff two TableRow data sets.

  14. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  15. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  16. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  17. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any

  18. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any

  19. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any

  20. def main(cmdlineArgs: Array[String]): Unit

    Permalink

    for easier running via sbt

  21. def mergeTableSchema(x: TableSchema, y: TableSchema): TableSchema

    Permalink

    Merge two BigQuery TableSchemas.

  22. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  23. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef

  24. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef

  25. def pathWithShards(path: String): String

    Permalink

  26. def run(cmdlineArgs: Array[String]): Unit

    Permalink

    Scio pipeline for BigDiffy.

  27. def saveStats[T](bigDiffy: BigDiffy[T], output: String, withHeader: Boolean = false, outputMode: OutputMode = GCS): Unit

    Permalink

    saves stats to either GCS as text, or BigQuery

  28. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef

  29. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any

  30. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  31. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  32. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Inherited from Command

Inherited from AnyRef

Inherited from Any

Ungrouped