testing

Type Members

case class AvroIO[T](path: String) extends TestIO[Nothing] with Product with Serializable
case class BigQueryIO(tableSpecOrQuery: String) extends TestIO[TableRow] with Product with Serializable
case class CustomIO[T](name: String) extends TestIO[Nothing] with Product with Serializable
case class DatastoreIO(projectId: String, query: Query = null, namespace: String = null) extends TestIO[Entity] with Product with Serializable
case class DistCacheIO[T](uri: String) extends Product with Serializable
case class ObjectFileIO[T](path: String) extends TestIO[Nothing] with Product with Serializable
case class ProtobufIO[T](path: String) extends TestIO[Nothing] with Product with Serializable
case class PubsubIO(topic: String) extends TestIO[String] with Product with Serializable
case class TFRecordIO(path: String) extends TestIO[Array[Byte]] with Product with Serializable
case class TableRowJsonIO(path: String) extends TestIO[TableRow] with Product with Serializable
class TestIO[+T] extends AnyRef
case class TextIO(path: String) extends TestIO[String] with Product with Serializable

Value Members

object DistCacheIO extends Serializable

object JobTest

Set up a Scio job for end-to-end unit testing.

Set up a Scio job for end-to-end unit testing. To be used in a PipelineSpec. For example:

import com.spotify.scio.testing._

class WordCountTest extends PipelineSpec {

  // Mock input data, mock distributed cache and expected result
  val inData = Seq("a b c d e", "a b a b")
  val distCache = Map(1 -> "Jan", 2 -> "Feb", 3 -> "Mar")
  val expected = Seq("a: 3", "b: 3", "c: 1", "d: 1", "e: 1")

  // Test specification
  "WordCount" should "work" in {
    JobTest("com.spotify.scio.examples.WordCount")
    // Or the type safe version
    // JobTest[com.spotify.scio.examples.WordCount.type]

      // Command line arguments
      .args("--input=in.txt", "--output=out.txt")

      // Mock input data
      .input(TextIO("in.txt"), inData)

      // Mock distributed cache
      .distCache(DistCacheIO("gs://dataflow-samples/samples/misc/months.txt", distCache)

      // Verify output
      .output(TextIO("out.txt")) { actual => actual should containInAnyOrder (expected) }

      // Run job test
      .run()
  }
}

package testing

Type Members

case class AvroIO[T](path: String) extends TestIO[Nothing] with Product with Serializable

case class BigQueryIO(tableSpecOrQuery: String) extends TestIO[TableRow] with Product with Serializable

case class CustomIO[T](name: String) extends TestIO[Nothing] with Product with Serializable

case class DatastoreIO(projectId: String, query: Query = null, namespace: String = null) extends TestIO[Entity] with Product with Serializable

case class DistCacheIO[T](uri: String) extends Product with Serializable

case class ObjectFileIO[T](path: String) extends TestIO[Nothing] with Product with Serializable

case class ProtobufIO[T](path: String) extends TestIO[Nothing] with Product with Serializable

case class PubsubIO(topic: String) extends TestIO[String] with Product with Serializable

case class TFRecordIO(path: String) extends TestIO[Array[Byte]] with Product with Serializable

case class TableRowJsonIO(path: String) extends TestIO[TableRow] with Product with Serializable

class TestIO[+T] extends AnyRef

case class TextIO(path: String) extends TestIO[String] with Product with Serializable

Value Members

object DistCacheIO extends Serializable

object JobTest

Ungrouped