ScioContext

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def avroFile[T](path: String, schema: Schema = null)(implicit arg0: ClassTag[T]): SCollection[T]

   

   Get an SCollection for an Avro file.

   Get an SCollection for an Avro file.

   schema

   must be not null if T is of type GenericRecord.

6. def bigQuerySelect(sqlQuery: String, flattenResults: Boolean = false): SCollection[TableRow]

   

   Get an SCollection for a BigQuery SELECT query.

   Get an SCollection for a BigQuery SELECT query. Both Legacy SQL and Standard SQL dialects are supported. By default the query dialect will be automatically detected. To override this behavior, start the query string with #legacysql or #standardsql.

7. def bigQueryTable(tableSpec: String): SCollection[TableRow]

   

   Get an SCollection for a BigQuery table.

8. def bigQueryTable(table: TableReference): SCollection[TableRow]

   

   Get an SCollection for a BigQuery table.

9. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

10. def close(): ScioResult

    

    Close the context.

    Close the context. No operation can be performed once the context is closed.

11. def customInput[T, I >: PBegin <: PInput](name: String, transform: PTransform[I, PCollection[T]])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection with a custom input transform.

    Get an SCollection with a custom input transform. The transform should have a unique name.

12. def datastore(projectId: String, query: Query, namespace: String = null): SCollection[Entity]

    

    Get an SCollection for a Datastore query.

13. def empty[T]()(implicit arg0: ClassTag[T]): SCollection[T]

    

    Form an empty SCollection.

14. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

15. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

16. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

17. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

18. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

19. def initCounter(namespace: String, name: String): Counter

    

    Initialize a new Counter metric.

20. def initCounter[T](name: String)(implicit arg0: ClassTag[T]): Counter

    

    Initialize a new Counter metric using T as namespace.

    Initialize a new Counter metric using T as namespace. Default is "com.spotify.scio.ScioMetrics" if T is not specified.

21. def isClosed: Boolean

    

    Whether the context is closed.

22. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

23. def isTest: Boolean

    

    Whether this is a test context.

24. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

25. final def notify(): Unit

    

    Definition Classes

    AnyRef

26. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

27. def objectFile[T](path: String)(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection for an object file using default serialization.

    Get an SCollection for an object file using default serialization.

    Serialized objects are stored in Avro files to leverage Avro's block file format. Note that serialization is not guaranteed to be compatible across Scio releases.

28. val options: PipelineOptions

    

29. def optionsAs[T <: PipelineOptions](implicit arg0: ClassTag[T]): T

    

    Get PipelineOptions as a more specific sub-type.

30. def parallelize[K, V](elems: Map[K, V])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): SCollection[(K, V)]

    

    Distribute a local Scala Map to form an SCollection.

31. def parallelize[T](elems: Iterable[T])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable to form an SCollection.

32. def parallelizeTimestamped[T](elems: Iterable[T], timestamps: Iterable[Instant])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable with timestamps to form an SCollection.

33. def parallelizeTimestamped[T](elems: Iterable[(T, Instant)])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Distribute a local Scala Iterable with timestamps to form an SCollection.

34. def pipeline: Pipeline

    

    Underlying pipeline.

35. def protobufFile[T](path: String)(implicit arg0: ClassTag[T], ev: <:<[T, Message]): SCollection[T]

    

    Get an SCollection for a Protobuf file.

    Get an SCollection for a Protobuf file.

    Protobuf messages are serialized into Array[Byte] and stored in Avro files to leverage Avro's block file format.

36. def pubsubSubscription[T](sub: String, idAttribute: String = null, timestampAttribute: String = null)(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection for a Pub/Sub subscription.

37. def pubsubSubscriptionWithAttributes[T](sub: String, idAttribute: String = null, timestampAttribute: String = null)(implicit arg0: ClassTag[T]): SCollection[(T, Map[String, String])]

    

    Get an SCollection for a Pub/Sub subscription that includes message attributes.

38. def pubsubTopic[T](topic: String, idAttribute: String = null, timestampAttribute: String = null)(implicit arg0: ClassTag[T]): SCollection[T]

    

    Get an SCollection for a Pub/Sub topic.

39. def pubsubTopicWithAttributes[T](topic: String, idAttribute: String = null, timestampAttribute: String = null)(implicit arg0: ClassTag[T]): SCollection[(T, Map[String, String])]

    

    Get an SCollection for a Pub/Sub topic that includes message attributes.

40. def setAppName(name: String): Unit

    

    Set application name for the context.

41. def setJobName(name: String): Unit

    

    Set job name for the context.

42. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

43. def tableRowJsonFile(path: String): SCollection[TableRow]

    

    Get an SCollection for a BigQuery TableRow JSON file.

44. def textFile(path: String, compression: Compression = gio.Compression.AUTO): SCollection[String]

    

    Get an SCollection for a text file.

45. def toString(): String

    

    Definition Classes

    AnyRef → Any

46. def typedAvroFile[T <: HasAvroAnnotation](path: String)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T]): SCollection[T]

    

    Get a typed SCollection from an Avro schema.

    Get a typed SCollection from an Avro schema.

    Note that T must be annotated with AvroType.fromSchema, AvroType.fromPath, or AvroType.toSchema.

47. def typedBigQuery[T <: HasAnnotation](newSource: String = null)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T]): SCollection[T]

    

    Get a typed SCollection for a BigQuery SELECT query or table.

    Get a typed SCollection for a BigQuery SELECT query or table.

    Note that T must be annotated with BigQueryType.fromSchema, BigQueryType.fromTable, BigQueryType.fromQuery, or BigQueryType.toTable.

    By default the source (table or query) specified in the annotation will be used, but it can be overridden with the newSource parameter. For example:

    @BigQueryType.fromTable("publicdata:samples.gsod")
    class Row
    
    // Read from [publicdata:samples.gsod] as specified in the annotation.
    sc.typedBigQuery[Row]()
    
    // Read from [myproject:samples.gsod] instead.
    sc.typedBigQuery[Row]("myproject:samples.gsod")
    
    // Read from a query instead.
    sc.typedBigQuery[Row]("SELECT * FROM [publicdata:samples.gsod] LIMIT 1000")

    Both Legacy SQL and Standard SQL dialects are supported. By default the query dialect will be automatically detected. To override this behavior, start the query string with #legacysql or #standardsql.

48. def unionAll[T](scs: Iterable[SCollection[T]])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Create a union of multiple SCollections.

    Create a union of multiple SCollections. Supports empty lists.

49. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

50. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

51. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

52. def withName(name: String): ScioContext.this.type

    

    Set a custom name for the next transform to be applied.

    Set a custom name for the next transform to be applied.

    Definition Classes

    TransformNameable

53. def wrap[T](p: PCollection[T])(implicit arg0: ClassTag[T]): SCollection[T]

    

    Wrap a PCollection.