public class ARCWriterProcessor extends WriterPoolProcessor
ANNOTATION_UNWRITTEN, compress, directory, frequentFlushes, maxFileSizeBytes, maxTotalBytesToWrite, maxWaitForIdleMs, poolMaxActive, prefix, serverCache, skipIdenticalDigests, startNewFilesOnCheckpoint, storePaths, template, writeBufferSize
Constructor and Description |
---|
ARCWriterProcessor() |
Modifier and Type | Method and Description |
---|---|
long |
getDefaultMaxFileSize() |
List<ConfigPath> |
getDefaultStorePaths() |
List<String> |
getMetadata() |
protected ProcessResult |
innerProcessResult(CrawlURI curi)
Writes a CrawlURI and its associated data to store file.
|
protected void |
setupPool(AtomicInteger serialNo)
Set up pool of files.
|
protected ProcessResult |
write(CrawlURI curi,
long recordLength,
InputStream in,
String ip) |
calcOutputDirs, checkBytesWritten, copyForwardWriteTagIfDupe, doCheckpoint, fromCheckpointJson, getCompress, getDirectory, getFrequentFlushes, getHostAddress, getMaxFileSizeBytes, getMaxTotalBytesToWrite, getMaxWaitForIdleMs, getMetadataProvider, getPool, getPoolMaxActive, getPrefix, getSerialNo, getServerCache, getSkipIdenticalDigests, getStartNewFilesOnCheckpoint, getStorePaths, getTemplate, getTotalBytesWritten, getWriteBufferSize, innerProcess, innerRejectProcess, setCompress, setDirectory, setFrequentFlushes, setMaxFileSizeBytes, setMaxTotalBytesToWrite, setMaxWaitForIdleMs, setMetadataProvider, setPool, setPoolMaxActive, setPrefix, setServerCache, setSkipIdenticalDigests, setStartNewFilesOnCheckpoint, setStorePaths, setTemplate, setTotalBytesWritten, setWriteBufferSize, shouldProcess, shouldWrite, start, stop, toCheckpointJson
finishCheckpoint, flattenVia, getBeanName, getEnabled, getKeyedProperties, getRecordedSize, getShouldProcessRule, getURICount, hasHttpAuthenticationCredential, isRunning, isSuccess, process, report, setBeanName, setEnabled, setRecoveryCheckpoint, setShouldProcessRule, startCheckpoint
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
finishCheckpoint, setRecoveryCheckpoint, startCheckpoint
public long getDefaultMaxFileSize()
getDefaultMaxFileSize
in class WriterPoolProcessor
public List<ConfigPath> getDefaultStorePaths()
getDefaultStorePaths
in class WriterPoolProcessor
protected void setupPool(AtomicInteger serialNo)
WriterPoolProcessor
setupPool
in class WriterPoolProcessor
protected ProcessResult innerProcessResult(CrawlURI curi)
innerProcessResult
in class WriterPoolProcessor
curi
- CrawlURI to process.protected ProcessResult write(CrawlURI curi, long recordLength, InputStream in, String ip) throws IOException
IOException
public List<String> getMetadata()
getMetadata
in interface WriterPoolSettings
getMetadata
in class WriterPoolProcessor
Copyright © 2003–2019 Internet Archive. All rights reserved.