JobTest

Set up a Scio job for end-to-end unit testing. To be used in a PipelineSpec. For example:

import com.spotify.scio.testing._

class WordCountTest extends PipelineSpec {

  // Mock input data, mock distributed cache and expected result
  val inData = Seq("a b c d e", "a b a b")
  val distCache = Map(1 -> "Jan", 2 -> "Feb", 3 -> "Mar")
  val expected = Seq("a: 3", "b: 3", "c: 1", "d: 1", "e: 1")

  // Test specification
  "WordCount" should "work" in {
    JobTest("com.spotify.scio.examples.WordCount")
    // Or the type safe version
    // JobTest[com.spotify.scio.examples.WordCount.type]

      // Command line arguments
      .args("--input=in.txt", "--output=out.txt")

      // Mock input data
      .input(TextIO("in.txt"), inData)

      // Mock distributed cache
      .distCache(DistCacheIO("gs://dataflow-samples/samples/misc/months.txt", distCache)

      // Verify output
      .output(TextIO("out.txt")) { actual => actual should containInAnyOrder (expected) }

      // Run job test
      .run()
  }
}

Linear Supertypes

AnyRef, Any

Type Members

case class Builder(className: String, cmdlineArgs: Array[String], inputs: Map[TestIO[_], Iterable[_]], outputs: Map[TestIO[_], (SCollection[_]) ⇒ Unit], distCaches: Map[DistCacheIO[_], _]) extends Product with Serializable

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def apply[T](implicit arg0: ClassTag[T]): Builder

Create a new JobTest.Builder instance.
def apply(className: String): Builder

Create a new JobTest.Builder instance.
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTestId(appName: String): Boolean
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newTestId(className: String = "TestClass"): String
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package testing

object JobTest

Type Members

case class Builder(className: String, cmdlineArgs: Array[String], inputs: Map[TestIO[_], Iterable[_]], outputs: Map[TestIO[_], (SCollection[_]) ⇒ Unit], distCaches: Map[DistCacheIO[_], _]) extends Product with Serializable

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def apply[T](implicit arg0: ClassTag[T]): Builder

def apply(className: String): Builder

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def isTestId(appName: String): Boolean

final def ne(arg0: AnyRef): Boolean

def newTestId(className: String = "TestClass"): String

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped