org.deeplearning4j.models.embeddings.loader

Class WordVectorSerializer

java.lang.Object

org.deeplearning4j.models.embeddings.loader.WordVectorSerializer

public class WordVectorSerializer
extends Object

This is utility class, providing various methods for WordVectors serialization

Author:

Adam Gibson, raver119, [email protected]

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
protected static class	WordVectorSerializer.BinaryReader
protected static class	WordVectorSerializer.CSVReader
protected static interface	WordVectorSerializer.Reader

Method Summary

Modifier and Type	Method and Description
static String	decodeB64(String word)
static String	encodeB64(String word)
static Word2Vec	fromPair(org.nd4j.linalg.primitives.Pair<InMemoryLookupTable,VocabCache> pair) Load word vectors from the given pair
static WordVectors	fromTableAndVocab(WeightLookupTable table, VocabCache vocab) Load word vectors for the given vocab and table
static float	getFloat(byte[] b) Read a string from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java
protected static TokenizerFactory	getTokenizerFactory(VectorsConfiguration configuration)
static Word2Vec	loadFullModel(String path) Deprecated. Use readWord2VecModel() or loadStaticModel() method instead
static WordVectors	loadStaticModel(File file)
static WordVectors	loadStaticModel(InputStream inputStream) This method restores previously saved w2v model.
static org.nd4j.linalg.primitives.Pair<InMemoryLookupTable,VocabCache>	loadTxt(File vectorsFile) Loads an in memory cache from the given path (sets syn0 and the vocab)
static WordVectors	loadTxtVectors(File vectorsFile) Deprecated.
static WordVectors	loadTxtVectors(InputStream stream, boolean skipFirstLine) Deprecated. Use readWord2VecModel() or loadStaticModel() method instead
static void	printOutProjectedMemoryUse(long numWords, int vectorLength, int numTables)
static Word2Vec	readAsBinary(File file)
static Word2Vec	readAsBinaryNoLineBreaks(File file)
static Word2Vec	readAsCsv(File file)
static Word2Vec	readBinaryModel(File modelFile, boolean linebreaks, boolean normalize) Read a binary word2vec file.
static float	readFloat(InputStream is) Read a float from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java
static <T extends SequenceElement> WeightLookupTable<T>	readLookupTable(File file)
static <T extends SequenceElement> WeightLookupTable<T>	readLookupTable(InputStream stream)
static ParagraphVectors	readParagraphVectors(File file) This method restores ParagraphVectors model previously saved with writeParagraphVectors()
static ParagraphVectors	readParagraphVectors(InputStream stream) This method restores ParagraphVectors model previously saved with writeParagraphVectors()
static ParagraphVectors	readParagraphVectors(String path) This method restores ParagraphVectors model previously saved with writeParagraphVectors()
static ParagraphVectors	readParagraphVectorsFromText(File file) Deprecated. Use readParagraphVectors() method instead
static ParagraphVectors	readParagraphVectorsFromText(InputStream stream) Deprecated. Use readParagraphVectors() method instead
static ParagraphVectors	readParagraphVectorsFromText(String path) Deprecated. Use readParagraphVectors() method instead
static <T extends SequenceElement> SequenceVectors<T>	readSequenceVectors(File file, boolean readExtendedTables)
static <T extends SequenceElement> SequenceVectors<T>	readSequenceVectors(InputStream stream, boolean readExtendedTables)
static <T extends SequenceElement> SequenceVectors<T>	readSequenceVectors(SequenceElementFactory<T> factory, File file) This method loads previously saved SequenceVectors model from File
static <T extends SequenceElement> SequenceVectors<T>	readSequenceVectors(SequenceElementFactory<T> factory, InputStream stream) This method loads previously saved SequenceVectors model from InputStream
static <T extends SequenceElement> SequenceVectors<T>	readSequenceVectors(String path, boolean readExtendedTables)
static String	readString(DataInputStream dis) Read a string from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java
static VocabCache<VocabWord>	readVocabCache(File file) This method reads vocab cache from provided file.
static VocabCache<VocabWord>	readVocabCache(InputStream stream) This method reads vocab cache from provided InputStream.
static Word2Vec	readWord2Vec(File file) Deprecated.
static Word2Vec	readWord2Vec(File file, boolean readExtendedTables)
static Word2Vec	readWord2Vec(InputStream stream, boolean readExtendedTable)
static Word2Vec	readWord2Vec(String path, boolean readExtendedTables)
static Word2Vec	readWord2VecFromText(File vectors, File hs, File h_codes, File h_points, VectorsConfiguration configuration) This method allows you to read ParagraphVectors from externally originated vectors and syn1.
static Word2Vec	readWord2VecModel(File file) This method 1) Binary model, either compressed or not.
static Word2Vec	readWord2VecModel(File file, boolean extendedModel) This method 1) Binary model, either compressed or not.
static Word2Vec	readWord2VecModel(String path) This method 1) Binary model, either compressed or not.
static Word2Vec	readWord2VecModel(String path, boolean extendedModel) This method 1) Binary model, either compressed or not.
static void	writeFullModel(Word2Vec vec, String path) Deprecated. Use writeWord2VecModel() method instead
static <T extends SequenceElement> void	writeLookupTable(WeightLookupTable<T> weightLookupTable, File file)
static void	writeParagraphVectors(ParagraphVectors vectors, File file) This method saves ParagraphVectors model into compressed zip file
static void	writeParagraphVectors(ParagraphVectors vectors, OutputStream stream) This method saves ParagraphVectors model into compressed zip file and sends it to output stream
static void	writeParagraphVectors(ParagraphVectors vectors, String path) This method saves ParagraphVectors model into compressed zip file located at path
static <T extends SequenceElement> void	writeSequenceVectors(SequenceVectors<T> vectors, OutputStream stream)
static <T extends SequenceElement> void	writeSequenceVectors(SequenceVectors<T> vectors, SequenceElementFactory<T> factory, File file) This method saves specified SequenceVectors model to target file
static <T extends SequenceElement> void	writeSequenceVectors(SequenceVectors<T> vectors, SequenceElementFactory<T> factory, OutputStream stream) This method saves specified SequenceVectors model to target OutputStream
static <T extends SequenceElement> void	writeSequenceVectors(SequenceVectors<T> vectors, SequenceElementFactory<T> factory, String path) This method saves specified SequenceVectors model to target file path
static void	writeTsneFormat(Glove vec, org.nd4j.linalg.api.ndarray.INDArray tsne, File csv) Write the tsne format
static void	writeTsneFormat(Word2Vec vec, org.nd4j.linalg.api.ndarray.INDArray tsne, File csv) Write the tsne format
static void	writeVocabCache(VocabCache<VocabWord> vocabCache, File file) This method saves vocab cache to provided File.
static void	writeVocabCache(VocabCache<VocabWord> vocabCache, OutputStream stream) This method saves vocab cache to provided OutputStream.
static void	writeWord2Vec(Word2Vec word2Vec, OutputStream stream)
static void	writeWord2VecModel(Word2Vec vectors, File file) This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1
static void	writeWord2VecModel(Word2Vec vectors, OutputStream stream) This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1
static void	writeWord2VecModel(Word2Vec vectors, String path) This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1
static void	writeWordVectors(Glove vectors, File file) This method saves GloVe model to the given output stream.
static void	writeWordVectors(Glove vectors, OutputStream stream) This method saves GloVe model to the given OutputStream
static void	writeWordVectors(Glove vectors, String path) This method saves GloVe model to the given output stream.
static void	writeWordVectors(InMemoryLookupTable lookupTable, InMemoryLookupCache cache, String path) Deprecated. Use writeWord2VecModel(Word2Vec, File) instead
static void	writeWordVectors(ParagraphVectors vectors, File path) Deprecated.
static void	writeWordVectors(ParagraphVectors vectors, OutputStream stream) Deprecated.
static void	writeWordVectors(ParagraphVectors vectors, String path) Deprecated.
static <T extends SequenceElement> void	writeWordVectors(WeightLookupTable<T> lookupTable, File file) This method writes word vectors to the given file.
static <T extends SequenceElement> void	writeWordVectors(WeightLookupTable<T> lookupTable, OutputStream stream) This method writes word vectors to the given OutputStream.
static <T extends SequenceElement> void	writeWordVectors(WeightLookupTable<T> lookupTable, String path) This method writes word vectors to the given path.
static void	writeWordVectors(Word2Vec vec, BufferedWriter writer) Deprecated.
static void	writeWordVectors(Word2Vec vec, File file) Deprecated.
static void	writeWordVectors(Word2Vec vec, OutputStream outputStream) Deprecated.
static void	writeWordVectors(Word2Vec vec, String path) Deprecated.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

readBinaryModel

public static Word2Vec readBinaryModel(File modelFile,
                                       boolean linebreaks,
                                       boolean normalize)
                                throws NumberFormatException,
                                       IOException

Read a binary word2vec file.

Parameters:

modelFile - the File to read

linebreaks - if true, the reader expects each word/vector to be in a separate line, terminated by a line break

Returns:

a model

Throws:

NumberFormatException

IOException

FileNotFoundException

readFloat

public static float readFloat(InputStream is)
                       throws IOException

Read a float from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java

Parameters:

is -

Returns:

Throws:

IOException

getFloat

public static float getFloat(byte[] b)

Read a string from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java

Parameters:

b -

Returns:

Throws:

IOException

readString

public static String readString(DataInputStream dis)
                         throws IOException

Read a string from a data input stream Credit to: https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java

Parameters:

dis -

Returns:

Throws:

IOException

writeWordVectors

public static <T extends SequenceElement> void writeWordVectors(WeightLookupTable<T> lookupTable,
                                                                String path)
                                                         throws IOException

This method writes word vectors to the given path. Please note: this method doesn't load whole vocab/lookupTable into memory, so it's able to process large vocabularies served over network.

Type Parameters:

T -

Parameters:

lookupTable -

path -

Throws:

IOException

writeWordVectors

public static <T extends SequenceElement> void writeWordVectors(WeightLookupTable<T> lookupTable,
                                                                File file)
                                                         throws IOException

This method writes word vectors to the given file. Please note: this method doesn't load whole vocab/lookupTable into memory, so it's able to process large vocabularies served over network.

Type Parameters:

T -

Parameters:

lookupTable -

file -

Throws:

IOException

writeWordVectors

public static <T extends SequenceElement> void writeWordVectors(WeightLookupTable<T> lookupTable,
                                                                OutputStream stream)
                                                         throws IOException

This method writes word vectors to the given OutputStream. Please note: this method doesn't load whole vocab/lookupTable into memory, so it's able to process large vocabularies served over network.

Type Parameters:

T -

Parameters:

lookupTable -

stream -

Throws:

IOException

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                ParagraphVectors vectors,
                                                @NonNull
                                                File path)

Deprecated.

This method saves paragraph vectors to the given file.

Parameters:

vectors -

path -

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                ParagraphVectors vectors,
                                                @NonNull
                                                String path)

Deprecated.

This method saves paragraph vectors to the given path.

Parameters:

vectors -

path -

writeParagraphVectors

public static void writeParagraphVectors(ParagraphVectors vectors,
                                         File file)

This method saves ParagraphVectors model into compressed zip file

Parameters:

file -

writeParagraphVectors

public static void writeParagraphVectors(ParagraphVectors vectors,
                                         String path)

This method saves ParagraphVectors model into compressed zip file located at path

Parameters:

path -

writeWord2VecModel

public static void writeWord2VecModel(Word2Vec vectors,
                                      File file)

This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1

writeWord2VecModel

public static void writeWord2VecModel(Word2Vec vectors,
                                      String path)

This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1

writeWord2VecModel

public static void writeWord2VecModel(Word2Vec vectors,
                                      OutputStream stream)
                               throws IOException

This method saves Word2Vec model into compressed zip file and sends it to output stream PLEASE NOTE: This method saves FULL model, including syn0 AND syn1

Throws:

IOException

writeParagraphVectors

public static void writeParagraphVectors(ParagraphVectors vectors,
                                         OutputStream stream)
                                  throws IOException

This method saves ParagraphVectors model into compressed zip file and sends it to output stream

Throws:

IOException

readParagraphVectors

public static ParagraphVectors readParagraphVectors(String path)
                                             throws IOException

This method restores ParagraphVectors model previously saved with writeParagraphVectors()

Returns:

Throws:

IOException

readParagraphVectors

public static ParagraphVectors readParagraphVectors(File file)
                                             throws IOException

This method restores ParagraphVectors model previously saved with writeParagraphVectors()

Returns:

Throws:

IOException

readWord2Vec

@Deprecated
public static Word2Vec readWord2Vec(File file)
                                         throws IOException

Deprecated.

This method restores Word2Vec model previously saved with writeWord2VecModel

PLEASE NOTE: This method loads FULL model, so don't use it if you're only going to use weights.

Parameters:

file -

Returns:

Throws:

IOException

readParagraphVectors

public static ParagraphVectors readParagraphVectors(InputStream stream)
                                             throws IOException

This method restores ParagraphVectors model previously saved with writeParagraphVectors()

Returns:

Throws:

IOException

readWord2VecFromText

public static Word2Vec readWord2VecFromText(@NonNull
                                            File vectors,
                                            @NonNull
                                            File hs,
                                            @NonNull
                                            File h_codes,
                                            @NonNull
                                            File h_points,
                                            @NonNull
                                            VectorsConfiguration configuration)
                                     throws IOException

This method allows you to read ParagraphVectors from externally originated vectors and syn1. So, technically this method is compatible with any other w2v implementation

Parameters:

vectors - text file with words and their weights, aka Syn0

hs - text file HS layers, aka Syn1

h_codes - text file with Huffman tree codes

h_points - text file with Huffman tree points

Returns:

Throws:

IOException

readParagraphVectorsFromText

@Deprecated
public static ParagraphVectors readParagraphVectorsFromText(@NonNull
                                                                        String path)

Deprecated. Use readParagraphVectors() method instead

Restores previously serialized ParagraphVectors model

Deprecation note: Please, consider using readParagraphVectors() method instead

Parameters:

path - Path to file that contains previously serialized model

Returns:

readParagraphVectorsFromText

@Deprecated
public static ParagraphVectors readParagraphVectorsFromText(@NonNull
                                                                        File file)

Deprecated. Use readParagraphVectors() method instead

Restores previously serialized ParagraphVectors model

Deprecation note: Please, consider using readParagraphVectors() method instead

Parameters:

file - File that contains previously serialized model

Returns:

readParagraphVectorsFromText

@Deprecated
public static ParagraphVectors readParagraphVectorsFromText(@NonNull
                                                                        InputStream stream)

Deprecated. Use readParagraphVectors() method instead

Restores previously serialized ParagraphVectors model

Deprecation note: Please, consider using readParagraphVectors() method instead

Parameters:

stream - InputStream that contains previously serialized model

Returns:

writeWordVectors

public static void writeWordVectors(@NonNull
                                    Glove vectors,
                                    @NonNull
                                    File file)

This method saves GloVe model to the given output stream.

Parameters:

vectors - GloVe model to be saved

file - path where model should be saved to

writeWordVectors

public static void writeWordVectors(@NonNull
                                    Glove vectors,
                                    @NonNull
                                    String path)

This method saves GloVe model to the given output stream.

Parameters:

vectors - GloVe model to be saved

path - path where model should be saved to

writeWordVectors

public static void writeWordVectors(@NonNull
                                    Glove vectors,
                                    @NonNull
                                    OutputStream stream)

This method saves GloVe model to the given OutputStream

Parameters:

vectors - GloVe model to be saved

stream - OutputStream where model should be saved to

writeWordVectors

@Deprecated
public static void writeWordVectors(ParagraphVectors vectors,
                                                OutputStream stream)

Deprecated.

This method saves paragraph vectors to the given output stream.

Parameters:

vectors -

stream -

writeWordVectors

@Deprecated
public static void writeWordVectors(InMemoryLookupTable lookupTable,
                                                InMemoryLookupCache cache,
                                                String path)
                                         throws IOException

Deprecated. Use writeWord2VecModel(Word2Vec, File) instead

Writes the word vectors to the given path. Note that this assumes an in memory cache

Parameters:

lookupTable -

cache -

path - the path to write

Throws:

IOException

writeFullModel

@Deprecated
public static void writeFullModel(@NonNull
                                              Word2Vec vec,
                                              @NonNull
                                              String path)

Deprecated. Use writeWord2VecModel() method instead

Saves full Word2Vec model in the way, that allows model updates without being rebuilt from scratches

Deprecation note: Please, consider using writeWord2VecModel() method instead

Parameters:

vec - - The Word2Vec instance to be saved

path - - the path for json to be saved

loadFullModel

@Deprecated
public static Word2Vec loadFullModel(@NonNull
                                                 String path)
                                          throws FileNotFoundException

Deprecated. Use readWord2VecModel() or loadStaticModel() method instead

This method loads full w2v model, previously saved with writeFullMethod call

Deprecation note: Please, consider using readWord2VecModel() or loadStaticModel() method instead

Parameters:

path - - path to previously stored w2v json model

Returns:

- Word2Vec instance

Throws:

FileNotFoundException

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                Word2Vec vec,
                                                @NonNull
                                                String path)
                                         throws IOException

Deprecated.

Writes the word vectors to the given path. Note that this assumes an in memory cache

Parameters:

vec - the word2vec to write

path - the path to write

Throws:

IOException

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                Word2Vec vec,
                                                @NonNull
                                                File file)
                                         throws IOException

Deprecated.

Writes the word vectors to the given path. Note that this assumes an in memory cache

Parameters:

vec - the word2vec to write

file - the file to write

Throws:

IOException

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                Word2Vec vec,
                                                @NonNull
                                                OutputStream outputStream)
                                         throws IOException

Deprecated.

Writes the word vectors to the given OutputStream. Note that this assumes an in memory cache.

Parameters:

vec - the word2vec to write

outputStream - - OutputStream, where all data should be sent to the path to write

Throws:

IOException

writeWordVectors

@Deprecated
public static void writeWordVectors(@NonNull
                                                Word2Vec vec,
                                                @NonNull
                                                BufferedWriter writer)
                                         throws IOException

Deprecated.

Writes the word vectors to the given BufferedWriter. Note that this assumes an in memory cache. BufferedWriter can be writer to local file, or hdfs file, or any compatible to java target.

Parameters:

vec - the word2vec to write

writer - - BufferedWriter, where all data should be written to the path to write

Throws:

IOException

fromTableAndVocab

public static WordVectors fromTableAndVocab(WeightLookupTable table,
                                            VocabCache vocab)

Load word vectors for the given vocab and table

Parameters:

table - the weights to use

vocab - the vocab to use

Returns:

wordvectors based on the given parameters

fromPair

public static Word2Vec fromPair(org.nd4j.linalg.primitives.Pair<InMemoryLookupTable,VocabCache> pair)

Load word vectors from the given pair

Parameters:

pair - the given pair

Returns:

a read only word vectors impl based on the given lookup table and vocab

loadTxtVectors

@Deprecated
public static WordVectors loadTxtVectors(File vectorsFile)
                                              throws IOException

Deprecated.

Loads an in memory cache from the given path (sets syn0 and the vocab)

Deprecation note: Please, consider using readWord2VecModel() or loadStaticModel() method instead

Parameters:

vectorsFile - the path of the file to load\

Returns:

Throws:

FileNotFoundException - if the file does not exist

IOException

loadTxt

public static org.nd4j.linalg.primitives.Pair<InMemoryLookupTable,VocabCache> loadTxt(File vectorsFile)
                                                                               throws IOException,
                                                                                      UnsupportedEncodingException

Loads an in memory cache from the given path (sets syn0 and the vocab)

Parameters:

vectorsFile - the path of the file to load

Returns:

a Pair holding the lookup table and the vocab cache.

Throws:

FileNotFoundException - if the input file does not exist

IOException

UnsupportedEncodingException

loadTxtVectors

@Deprecated
public static WordVectors loadTxtVectors(@NonNull
                                                     InputStream stream,
                                                     boolean skipFirstLine)
                                              throws IOException

Deprecated. Use readWord2VecModel() or loadStaticModel() method instead

This method can be used to load previously saved model from InputStream (like a HDFS-stream)

Deprecation note: Please, consider using readWord2VecModel() or loadStaticModel() method instead

Parameters:

stream - InputStream that contains previously serialized model

skipFirstLine - Set this TRUE if first line contains csv header, FALSE otherwise

Returns:

Throws:

IOException

writeTsneFormat

public static void writeTsneFormat(Glove vec,
                                   org.nd4j.linalg.api.ndarray.INDArray tsne,
                                   File csv)
                            throws Exception

Write the tsne format

Parameters:

vec - the word vectors to use for labeling

tsne - the tsne array to write

csv - the file to use

Throws:

Exception

writeTsneFormat

public static void writeTsneFormat(Word2Vec vec,
                                   org.nd4j.linalg.api.ndarray.INDArray tsne,
                                   File csv)
                            throws Exception

Write the tsne format

Parameters:

vec - the word vectors to use for labeling

tsne - the tsne array to write

csv - the file to use

Throws:

Exception

writeSequenceVectors

public static <T extends SequenceElement> void writeSequenceVectors(@NonNull
                                                                    SequenceVectors<T> vectors,
                                                                    @NonNull
                                                                    SequenceElementFactory<T> factory,
                                                                    @NonNull
                                                                    String path)
                                                             throws IOException

This method saves specified SequenceVectors model to target file path

Type Parameters:

T -

Parameters:

vectors - SequenceVectors model

factory - SequenceElementFactory implementation for your objects

path - Target output file path

Throws:

IOException

writeSequenceVectors

public static <T extends SequenceElement> void writeSequenceVectors(@NonNull
                                                                    SequenceVectors<T> vectors,
                                                                    @NonNull
                                                                    SequenceElementFactory<T> factory,
                                                                    @NonNull
                                                                    File file)
                                                             throws IOException

This method saves specified SequenceVectors model to target file

Type Parameters:

T -

Parameters:

vectors - SequenceVectors model

factory - SequenceElementFactory implementation for your objects

file - Target output file

Throws:

IOException

writeSequenceVectors

public static <T extends SequenceElement> void writeSequenceVectors(@NonNull
                                                                    SequenceVectors<T> vectors,
                                                                    @NonNull
                                                                    SequenceElementFactory<T> factory,
                                                                    @NonNull
                                                                    OutputStream stream)
                                                             throws IOException

This method saves specified SequenceVectors model to target OutputStream

Type Parameters:

T -

Parameters:

vectors - SequenceVectors model

factory - SequenceElementFactory implementation for your objects

stream - Target output stream

Throws:

IOException

writeSequenceVectors

public static <T extends SequenceElement> void writeSequenceVectors(@NonNull
                                                                    SequenceVectors<T> vectors,
                                                                    @NonNull
                                                                    OutputStream stream)
                                                             throws IOException

Throws:

IOException

readSequenceVectors

public static <T extends SequenceElement> SequenceVectors<T> readSequenceVectors(@NonNull
                                                                                 String path,
                                                                                 boolean readExtendedTables)
                                                                          throws IOException

Throws:

IOException

readSequenceVectors

public static <T extends SequenceElement> SequenceVectors<T> readSequenceVectors(@NonNull
                                                                                 File file,
                                                                                 boolean readExtendedTables)
                                                                          throws IOException

Throws:

IOException

readSequenceVectors

public static <T extends SequenceElement> SequenceVectors<T> readSequenceVectors(@NonNull
                                                                                 InputStream stream,
                                                                                 boolean readExtendedTables)
                                                                          throws IOException

Throws:

IOException

readSequenceVectors

public static <T extends SequenceElement> SequenceVectors<T> readSequenceVectors(@NonNull
                                                                                 SequenceElementFactory<T> factory,
                                                                                 @NonNull
                                                                                 File file)
                                                                          throws IOException

This method loads previously saved SequenceVectors model from File

Type Parameters:

T -

Parameters:

factory -

file -

Returns:

Throws:

IOException

readSequenceVectors

public static <T extends SequenceElement> SequenceVectors<T> readSequenceVectors(@NonNull
                                                                                 SequenceElementFactory<T> factory,
                                                                                 @NonNull
                                                                                 InputStream stream)
                                                                          throws IOException

This method loads previously saved SequenceVectors model from InputStream

Type Parameters:

T -

Parameters:

factory -

stream -

Returns:

Throws:

IOException

writeVocabCache

public static void writeVocabCache(@NonNull
                                   VocabCache<VocabWord> vocabCache,
                                   @NonNull
                                   File file)
                            throws IOException

This method saves vocab cache to provided File. Please note: it saves only vocab content, so it's suitable mostly for BagOfWords/TF-IDF vectorizers

Parameters:

vocabCache -

file -

Throws:

UnsupportedEncodingException

IOException

writeVocabCache

public static void writeVocabCache(@NonNull
                                   VocabCache<VocabWord> vocabCache,
                                   @NonNull
                                   OutputStream stream)
                            throws IOException

This method saves vocab cache to provided OutputStream. Please note: it saves only vocab content, so it's suitable mostly for BagOfWords/TF-IDF vectorizers

Parameters:

vocabCache -

stream -

Throws:

UnsupportedEncodingException

IOException

readVocabCache

public static VocabCache<VocabWord> readVocabCache(@NonNull
                                                   File file)
                                            throws IOException

This method reads vocab cache from provided file. Please note: it reads only vocab content, so it's suitable mostly for BagOfWords/TF-IDF vectorizers

Parameters:

file -

Returns:

Throws:

IOException

readVocabCache

public static VocabCache<VocabWord> readVocabCache(@NonNull
                                                   InputStream stream)
                                            throws IOException

This method reads vocab cache from provided InputStream. Please note: it reads only vocab content, so it's suitable mostly for BagOfWords/TF-IDF vectorizers

Parameters:

stream -

Returns:

Throws:

IOException

readWord2VecModel

public static Word2Vec readWord2VecModel(@NonNull
                                         File file)

This method 1) Binary model, either compressed or not. Like well-known Google Model 2) Popular CSV word2vec text format 3) DL4j compressed format

Please note: Only weights will be loaded by this method.

Parameters:

file -

Returns:

readWord2VecModel

public static Word2Vec readWord2VecModel(String path)

This method 1) Binary model, either compressed or not. Like well-known Google Model 2) Popular CSV word2vec text format 3) DL4j compressed format

Please note: Only weights will be loaded by this method.

Parameters:

path -

Returns:

readWord2VecModel

public static Word2Vec readWord2VecModel(String path,
                                         boolean extendedModel)

This method 1) Binary model, either compressed or not. Like well-known Google Model 2) Popular CSV word2vec text format 3) DL4j compressed format

Please note: if extended data isn't available, only weights will be loaded instead.

Parameters:

path -

extendedModel - if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded

Returns:

readAsBinaryNoLineBreaks

public static Word2Vec readAsBinaryNoLineBreaks(@NonNull
                                                File file)

readAsBinary

public static Word2Vec readAsBinary(@NonNull
                                    File file)

readAsCsv

public static Word2Vec readAsCsv(@NonNull
                                 File file)

readWord2VecModel

public static Word2Vec readWord2VecModel(@NonNull
                                         File file,
                                         boolean extendedModel)

This method 1) Binary model, either compressed or not. Like well-known Google Model 2) Popular CSV word2vec text format 3) DL4j compressed format

Please note: if extended data isn't available, only weights will be loaded instead.

Parameters:

file -

extendedModel - if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded

Returns:

getTokenizerFactory

protected static TokenizerFactory getTokenizerFactory(VectorsConfiguration configuration)

loadStaticModel

public static WordVectors loadStaticModel(InputStream inputStream)
                                   throws IOException

This method restores previously saved w2v model. File can be in one of the following formats: 1) Binary model, either compressed or not. Like well-known Google Model 2) Popular CSV word2vec text format 3) DL4j compressed format In return you get StaticWord2Vec model, which might be used as lookup table only in multi-gpu environment.

Parameters:

file - File should point to previously saved w2v model

Returns:

Throws:

IOException

loadStaticModel

public static WordVectors loadStaticModel(@NonNull
                                          File file)

encodeB64

public static String encodeB64(String word)

decodeB64

public static String decodeB64(String word)

writeWord2Vec

public static void writeWord2Vec(@NonNull
                                 Word2Vec word2Vec,
                                 @NonNull
                                 OutputStream stream)
                          throws IOException

Throws:

IOException

readWord2Vec

public static Word2Vec readWord2Vec(@NonNull
                                    String path,
                                    boolean readExtendedTables)
                             throws IOException

Throws:

IOException

writeLookupTable

public static <T extends SequenceElement> void writeLookupTable(WeightLookupTable<T> weightLookupTable,
                                                                @NonNull
                                                                File file)
                                                         throws IOException

Throws:

IOException

readLookupTable

public static <T extends SequenceElement> WeightLookupTable<T> readLookupTable(File file)
                                                                        throws IOException

Throws:

IOException

readLookupTable

public static <T extends SequenceElement> WeightLookupTable<T> readLookupTable(InputStream stream)
                                                                        throws IOException

Throws:

IOException

readWord2Vec

public static Word2Vec readWord2Vec(@NonNull
                                    File file,
                                    boolean readExtendedTables)
                             throws IOException

Throws:

IOException

readWord2Vec

public static Word2Vec readWord2Vec(@NonNull
                                    InputStream stream,
                                    boolean readExtendedTable)
                             throws IOException

Throws:

IOException

printOutProjectedMemoryUse

public static void printOutProjectedMemoryUse(long numWords,
                                              int vectorLength,
                                              int numTables)