This transformer parses text into tabular representation. The input consists of DOCUMENT annotations and the output are TABLE annotations. The source format can be either JSON or CSV. The format of the JSON files should be:

{
  "header": [col1, col2, ..., colN],
  "rows": [
    [val11, val12, ..., val1N],
    [val22, va22, ..., val2N],
    ...
   ]
}

The CSV format support alternative delimiters (e.g. tab), as well as escaping delimiters by surrounding cell values with double quotes. For example:

column1, column2, "column with, comma"
value1, value2, value3
"escaped value", "value with, comma", "value with double ("") quote"

The transformer stores tabular data internally as JSON. The default input format is also JSON.

Example

import spark.implicits._
import com.johnsnowlabs.nlp.DocumentAssembler
import org.apache.spark.ml.Pipeline

val csvData =
  """
    |"name", "money", "age"
    |"Donald Trump", "$100,000,000", "75"
    |"Elon Musk", "$20,000,000,000,000", "55"
    |""".stripMargin.trim

val data =Seq(csvData).toDF("csv")

val documentAssembler = new DocumentAssembler()
  .setInputCol("csv")
  .setOutputCol("document")

val tableAssembler = new TableAssembler()
  .setInputCols(Array("document"))
  .setOutputCol("table")
  .setInputFormat("csv")

val pipeline = new Pipeline()
  .setStages(
    Array(documentAssembler, tableAssembler)
    ).fit(data)

val result = pipeline.transform(data)
result
  .selectExpr("explode(table) AS table")
  .select("table.result", "table.metadata.input_format")
  .show(false)

+--------------------------------------------+-------------+
|result                                      |input_format |
+--------------------------------------------+-------------+
|{                                           |csv          |
| "header": ["name","money","age"],          |             |
|  "rows":[                                  |             |
|   ["Donald Trump","$100,000,000","75"],    |             |
|   ["Elon Musk","$20,000,000,000,000","55"] |             |
|  ]                                         |             |
|}                                           |             |
+--------------------------------------------+-------------+