ScioContext

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def addArtifacts(extraLocalArtifacts: List[String]): Unit

   

   Add artifact to stage in workers.

   Add artifact to stage in workers. Artifact can be jar/text-files etc. NOTE: currently one can only add artifacts before pipeline object is created.

5. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

6. def avroFile[T](path: String, schema: Schema = null)(implicit arg0: ClassTag[T]): SCollection[T]

   

   Get an SCollection for an Avro file.

7. def bigQuerySelect(sqlQuery: String, flattenResults: Boolean = false): SCollection[TableRow]

   

   Get an SCollection for a BigQuery SELECT query.

8. def bigQueryTable(tableSpec: String): SCollection[TableRow]

   

   Get an SCollection for a BigQuery table.

9. def bigQueryTable(table: TableReference): SCollection[TableRow]

   

   Get an SCollection for a BigQuery table.

10. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

11. def close(): ScioResult

    

    Close the context.

    Close the context. No operation can be performed once the context is closed.

12. def customInput[T](transform: PTransform[PInput, PCollection[T]])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection with a custom input transform.

    Get an SCollection with a custom input transform. The transform should have a unique name.

13. def datastore(projectId: String, query: Query, namespace: String = null): SCollection[Entity]

    

    Get an SCollection for a Datastore query.

14. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

15. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

16. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

17. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

18. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

19. def isClosed: Boolean

    

    Whether the context is closed.

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. def isTest: Boolean

    

    Whether this is a test context.

22. def maxAccumulator[T](n: String)(implicit at: AccumulatorType[T]): Accumulator[T]

    

    Create a new Accumulator that keeps track of the maximum value.

    Create a new Accumulator that keeps track of the maximum value. See SCollection.withAccumulator for examples.

23. def minAccumulator[T](n: String)(implicit at: AccumulatorType[T]): Accumulator[T]

    

    Create a new Accumulator that keeps track of the minimum value.

    Create a new Accumulator that keeps track of the minimum value. See SCollection.withAccumulator for examples.

24. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

25. final def notify(): Unit

    

    Definition Classes

    AnyRef

26. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

27. def objectFile[T](path: String)(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection for an object file.

28. val options: PipelineOptions

    

29. def optionsAs[T <: PipelineOptions](implicit arg0: ClassTag[T]): T

    

    Get PipelineOptions as a more specific sub-type.

30. def parallelize[K, V](elems: Map[K, V])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): SCollection[(K, V)]

    

    Distribute a local Scala Map to form an SCollection.

31. def parallelize[T](elems: Iterable[T])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable to form an SCollection.

32. def parallelizeTimestamped[T](elems: Iterable[T], timestamps: Iterable[Instant])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable with timestamps to form an SCollection.

33. def parallelizeTimestamped[T](elems: Iterable[(T, Instant)])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable with timestamps to form an SCollection.

34. def pipeline: Pipeline

    

    Underlying pipeline.

35. def protobufFile[T](path: String)(implicit arg0: ClassTag[T], ev: <:<[T, Message]): SCollection[T]

    

    Get an SCollection for a Protobuf file.

36. def pubsubSubscription(sub: String, idLabel: String = null, timestampLabel: String = null): SCollection[String]

    

    Get an SCollection for a Pub/Sub subscription.

37. def pubsubTopic(topic: String, idLabel: String = null, timestampLabel: String = null): SCollection[String]

    

    Get an SCollection for a Pub/Sub topic.

38. def setAppName(name: String): Unit

    

    Set application name for the context.

39. def setJobName(name: String): Unit

    

    Set job name for the context (Dataflow only)

40. def sumAccumulator[T](n: String)(implicit at: AccumulatorType[T]): Accumulator[T]

    

    Create a new Accumulator that keeps track of the sum of values.

    Create a new Accumulator that keeps track of the sum of values. See SCollection.withAccumulator for examples.

41. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

42. def tableRowJsonFile(path: String): SCollection[TableRow]

    

    Get an SCollection of TableRow for a JSON file.

43. def textFile(path: String, compressionType: CompressionType = gio.TextIO.CompressionType.AUTO): SCollection[String]

    

    Get an SCollection for a text file.

44. def tfRecordFile(path: String, tfRecordOptions: TFRecordOptions = TFRecordOptions.readDefault): SCollection[Array[Byte]]

    

    Get an SCollection for a TensorFlow TFRecord file.

    Get an SCollection for a TensorFlow TFRecord file. Note that TFRecord files are not splittable.

45. def toString(): String

    

    Definition Classes

    AnyRef → Any

46. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

47. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

48. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

49. def withName(name: String): ScioContext.this.type

    

    Set a custom name for the next transform to be applied.

    Set a custom name for the next transform to be applied.

    Definition Classes

    TransformNameable

50. def wrap[T](p: PCollection[T])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Wrap a PCollection.