public class FetchStats extends CrawledBytesHistotable implements Serializable, FetchStatusCodes, Reporter
Modifier and Type | Class and Description |
---|---|
static interface |
FetchStats.CollectsFetchStats |
static interface |
FetchStats.HasFetchStats |
static class |
FetchStats.Stage |
AbstractMap.SimpleEntry<K,V>, AbstractMap.SimpleImmutableEntry<K,V>
Modifier and Type | Field and Description |
---|---|
static String |
FETCH_DISREGARDS |
static String |
FETCH_FAILURES |
static String |
FETCH_NONRESPONSES |
static String |
FETCH_RESPONSES |
static String |
FETCH_SUCCESSES |
protected long |
lastSuccessTime |
static String |
ROBOTS_DENIALS |
static String |
SUCCESS_BYTES |
static String |
TOTAL_BYTES |
static String |
TOTAL_SCHEDULED |
DUPLICATE, DUPLICATECOUNT, NOTMODIFIED, NOTMODIFIEDCOUNT, NOVEL, NOVELCOUNT, OTHERDUPLICATE, OTHERDUPLICATECOUNT, WARC_NOVEL_CONTENT_BYTES, WARC_NOVEL_URLS
S_BLOCKED_BY_CUSTOM_PROCESSOR, S_BLOCKED_BY_QUOTA, S_BLOCKED_BY_RUNTIME_LIMIT, S_BLOCKED_BY_USER, S_CONNECT_FAILED, S_CONNECT_LOST, S_DEEMED_CHAFF, S_DEEMED_NOT_FOUND, S_DEFERRED, S_DELETED_BY_USER, S_DNS_SUCCESS, S_DOMAIN_PREREQUISITE_FAILURE, S_DOMAIN_UNRESOLVABLE, S_GETBYNAME_SUCCESS, S_NOT_FOUND, S_OTHER_PREREQUISITE_FAILURE, S_OUT_OF_SCOPE, S_PREREQUISITE_UNSCHEDULABLE_FAILURE, S_PROCESSING_THREAD_KILLED, S_ROBOTS_PRECLUDED, S_ROBOTS_PREREQUISITE_FAILURE, S_RUNTIME_EXCEPTION, S_SERIOUS_ERROR, S_TIMEOUT, S_TOO_MANY_EMBED_HOPS, S_TOO_MANY_LINK_HOPS, S_TOO_MANY_RETRIES, S_UNATTEMPTED, S_UNFETCHABLE_URI, S_UNQUEUEABLE, S_WHOIS_GENERIC_FINISHED, S_WHOIS_SUCCESS
A_ANNOTATIONS, A_CONTENT_TYPE, A_CREDENTIALS_KEY, A_DELAY_FACTOR, A_DISTANCE_FROM_SEED, A_DNS_FETCH_TIME, A_DNS_SERVER_IP_LABEL, A_FETCH_BEGAN_TIME, A_FETCH_COMPLETED_TIME, A_FORCE_RETIRE, A_FTP_CONTROL_CONVERSATION, A_FTP_FETCH_STATUS, A_HERITABLE_KEYS, A_HTML_BASE, A_HTTP_AUTH_CHALLENGES, A_HTTP_PROXY_HOST, A_HTTP_PROXY_PORT, A_HTTP_RESPONSE_HEADERS, A_MINIMUM_DELAY, A_MIRROR_PATH, A_NONFATAL_ERRORS, A_PRECALC_PRECEDENCE, A_PREREQUISITE_URI, A_RETRY_DELAY, A_RRECORD_SET_LABEL, A_RUNTIME_EXCEPTION, A_SOURCE_TAG, A_SUBMIT_DATA, A_SUBMIT_ENCTYPE, A_WARC_RESPONSE_HEADERS, A_WARC_STATS, A_WHOIS_SERVER_IP, HEADER_TRUNC, LENGTH_TRUNC, TIMER_TRUNC, TRUNC_SUFFIX
Constructor and Description |
---|
FetchStats() |
Modifier and Type | Method and Description |
---|---|
long |
getDupByHashBytes() |
long |
getDupByHashUrls() |
long |
getFetchDisregards() |
long |
getFetchNonResponses() |
long |
getFetchResponses() |
long |
getFetchSuccesses() |
long |
getLastSuccessTime() |
long |
getNotModifiedBytes() |
long |
getNotModifiedUrls() |
long |
getNovelBytes() |
long |
getNovelUrls() |
long |
getOtherDupBytes() |
long |
getOtherDupUrls() |
long |
getRecordedFinishes() |
long |
getRemaining() |
long |
getRobotsDenials() |
long |
getSuccessBytes() |
long |
getTotalBytes() |
long |
getTotalScheduled() |
void |
reportTo(PrintWriter writer) |
String |
shortReportLegend() |
String |
shortReportLine() |
void |
shortReportLineTo(PrintWriter writer) |
Map<String,Object> |
shortReportMap() |
void |
tally(CrawlURI curi,
FetchStats.Stage stage) |
accumulate, getTotalUrls, summary
add, entryString, get, getEntryByFrequencySortedSet, getLargestValue, getSortedByCounts, getSortedByKeys, getTotal, subtract, tally, tally
ceilingEntry, ceilingKey, clear, clone, comparator, containsKey, containsValue, descendingKeySet, descendingMap, entrySet, firstEntry, firstKey, floorEntry, floorKey, forEach, headMap, headMap, higherEntry, higherKey, keySet, lastEntry, lastKey, lowerEntry, lowerKey, navigableKeySet, pollFirstEntry, pollLastEntry, put, putAll, remove, replace, replace, replaceAll, size, subMap, subMap, tailMap, tailMap, values
equals, hashCode, isEmpty, toString
finalize, getClass, notify, notifyAll, wait, wait, wait
compute, computeIfAbsent, computeIfPresent, equals, getOrDefault, hashCode, isEmpty, merge, putIfAbsent, remove
public static final String TOTAL_SCHEDULED
public static final String FETCH_SUCCESSES
public static final String FETCH_FAILURES
public static final String FETCH_DISREGARDS
public static final String FETCH_RESPONSES
public static final String ROBOTS_DENIALS
public static final String SUCCESS_BYTES
public static final String TOTAL_BYTES
public static final String FETCH_NONRESPONSES
protected long lastSuccessTime
public void tally(CrawlURI curi, FetchStats.Stage stage)
public long getFetchSuccesses()
public long getFetchResponses()
public long getSuccessBytes()
public long getTotalBytes()
getTotalBytes
in class CrawledBytesHistotable
public long getFetchNonResponses()
public long getTotalScheduled()
public long getFetchDisregards()
public long getRobotsDenials()
public long getRemaining()
public long getRecordedFinishes()
public long getNovelBytes()
public long getNovelUrls()
public long getNotModifiedBytes()
public long getNotModifiedUrls()
public long getDupByHashBytes()
public long getDupByHashUrls()
public long getOtherDupBytes()
public long getOtherDupUrls()
public void reportTo(PrintWriter writer)
public String shortReportLegend()
shortReportLegend
in interface Reporter
public String shortReportLine()
public void shortReportLineTo(PrintWriter writer)
shortReportLineTo
in interface Reporter
public Map<String,Object> shortReportMap()
shortReportMap
in interface Reporter
public long getLastSuccessTime()
Copyright © 2003–2019 Internet Archive. All rights reserved.