com.yahoo.prelude.cluster

Class ClusterSearcher

java.lang.Object

com.yahoo.component.AbstractComponent

com.yahoo.component.chain.ChainedComponent

com.yahoo.processing.Processor

com.yahoo.search.Searcher

com.yahoo.prelude.cluster.ClusterSearcher

All Implemented Interfaces:

com.yahoo.component.Component, Comparable<com.yahoo.component.Component>

public class ClusterSearcher
extends Searcher

A searcher which forwards to a cluster of monitored native Vespa backends.

Author:

bratseth, Steinar Knutsen, geirst

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private class	ClusterSearcher.Pinger

Field Summary

Fields

Modifier and Type	Field and Description
private List<Backend>	backends
private com.yahoo.statistics.Value	cacheHitRatio
private String	clusterModelName
private static long	DEFAULT_MAX_QUERY_CACHE_TIMEOUT
private static long	DEFAULT_MAX_QUERY_TIMEOUT
private Set<String>	documentTypes
private boolean	failoverToRemote
private FS4ResourcePool	fs4ResourcePool
private Hasher	hasher
private static Logger	log
private long	maxQueryCacheTimeout
private long	maxQueryTimeout
private ClusterMonitor	monitor
private Map<String,Set<String>>	rankProfiles

Fields inherited from class com.yahoo.component.AbstractComponent

isDeconstructable

Constructor Summary

Constructors

Constructor and Description
ClusterSearcher(com.yahoo.component.ComponentId id, com.yahoo.container.QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, LegacyEmulationConfig emulationConfig, QrMonitorConfig monitorConfig, com.yahoo.vespa.config.search.DispatchConfig dispatchConfig, com.yahoo.cloud.config.ClusterInfoConfig clusterInfoConfig, com.yahoo.statistics.Statistics manager, FS4ResourcePool fs4ResourcePool, com.yahoo.container.handler.VipStatus vipStatus) Creates a new ClusterSearcher.
ClusterSearcher(Set<String> documentTypes) Do not use, for internal testing purposes only.

Method Summary

Modifier and Type	Method and Description
(package private) void	addBackendSearcher(VespaBackEndSearcher searcher)
(package private) void	addValidRankProfile(String profileName, String docTypeName)
private boolean	backendCanServeDocuments(Pong pong)
private void	cacheHit()
private void	cacheMiss()
private Result	checkValidRankProfiles(Query query, Set<String> docTypes) Returns an error if the document types do not have the requested rank profile.
private Backend	createBackend(com.yahoo.container.QrSearchersConfig.Searchcluster.Dispatcher disp)
private static CacheControl	createCache(ClusterConfig config, String clusterModelName)
private List<Query>	createQueries(Query query, Set<String> docTypes)
void	deconstruct()
void	doFill(Searcher searcher, Result result, String summaryClass, Execution execution)
private Result	doSearch(Searcher searcher, Query query, Execution execution)
void	dumpPackets(PacketDumper.PacketType packetType, boolean on)
(package private) void	failed(VespaBackEndSearcher node) Called from ClusterMonitor.
void	fill(Result result, String summaryClass, Execution execution) Fill hit properties with data using the given summary class.
private Set<String>	filterValidDocumentTypes(Collection<String> restrict)
Map<String,Backend.BackendStatistics>	getBackendStatistics()
String	getClusterModelName()
(package private) ExecutorService	getExecutor()
(package private) ClusterMonitor	getMonitor()
(package private) ScheduledExecutorService	getScheduledExecutor()
private static com.yahoo.container.QrSearchersConfig.Searchcluster	getSearchClusterConfigFromClusterName(com.yahoo.container.QrSearchersConfig config, String name)
(package private) boolean	isRemote(String host) Returns false if this host is local.
private static ClusterParams	makeClusterParams(int searchclusterIndex, com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig, LegacyEmulationConfig emulConfig, int dispatchIndex)
(package private) void	ping(VespaBackEndSearcher node) Pinging a node, called from ClusterMonitor.
(package private) Set<String>	resolveDocumentTypes(Query query, IndexFacts indexFacts)
Result	search(Query query, Execution execution) Override this to implement your searcher.
private static FastSearcher	searchDispatch(int searchclusterIndex, FS4ResourcePool fs4ResourcePool, com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig, CacheParams cacheParams, LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig, Backend backend, Dispatcher dispatcher, int dispatcherIndex)
private Result	searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution)
(package private) void	setValidRankProfile(String profileName, Set<String> documentTypes)
private void	updateCacheHitRatio(Result result, Query query)
private void	validateQueryCache(Query query)
private void	validateQueryTimeout(Query query)
private static VdsStreamingSearcher	vdsCluster(int searchclusterIndex, com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig, CacheParams cacheParams, LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig)
(package private) void	working(VespaBackEndSearcher node) NodeManager method, called from ClusterMonitor.

Methods inherited from class com.yahoo.search.Searcher

ensureFilled, getLogger, process, toString

Methods inherited from class com.yahoo.component.chain.ChainedComponent

getAnnotatedDependencies, getDefaultAnnotatedDependencies, getDependencies, initDependencies

Methods inherited from class com.yahoo.component.AbstractComponent

clone, compareTo, getClassName, getId, getIdString, hasInitializedId, initId, isDeconstructable, setIsDeconstructable

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

log

private static final Logger log

monitor

private final ClusterMonitor monitor

hasher

private final Hasher hasher

cacheHitRatio

private final com.yahoo.statistics.Value cacheHitRatio

clusterModelName

private final String clusterModelName

backends

private final List<Backend> backends

documentTypes

private final Set<String> documentTypes

rankProfiles

private final Map<String,Set<String>> rankProfiles

failoverToRemote

private final boolean failoverToRemote

fs4ResourcePool

private final FS4ResourcePool fs4ResourcePool

maxQueryTimeout

private final long maxQueryTimeout

DEFAULT_MAX_QUERY_TIMEOUT

private static final long DEFAULT_MAX_QUERY_TIMEOUT

See Also:

Constant Field Values

maxQueryCacheTimeout

private final long maxQueryCacheTimeout

DEFAULT_MAX_QUERY_CACHE_TIMEOUT

private static final long DEFAULT_MAX_QUERY_CACHE_TIMEOUT

See Also:

Constant Field Values

Constructor Detail

ClusterSearcher

public ClusterSearcher(com.yahoo.component.ComponentId id,
                       com.yahoo.container.QrSearchersConfig qrsConfig,
                       ClusterConfig clusterConfig,
                       DocumentdbInfoConfig documentDbConfig,
                       LegacyEmulationConfig emulationConfig,
                       QrMonitorConfig monitorConfig,
                       com.yahoo.vespa.config.search.DispatchConfig dispatchConfig,
                       com.yahoo.cloud.config.ClusterInfoConfig clusterInfoConfig,
                       com.yahoo.statistics.Statistics manager,
                       FS4ResourcePool fs4ResourcePool,
                       com.yahoo.container.handler.VipStatus vipStatus)

Creates a new ClusterSearcher.

ClusterSearcher

ClusterSearcher(Set<String> documentTypes)

Do not use, for internal testing purposes only.

Method Detail

getSearchClusterConfigFromClusterName

private static com.yahoo.container.QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(com.yahoo.container.QrSearchersConfig config,
                                                                                                         String name)

isRemote

boolean isRemote(String host)
          throws UnknownHostException

Returns false if this host is local.

Throws:

UnknownHostException

makeClusterParams

private static ClusterParams makeClusterParams(int searchclusterIndex,
                                               com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig,
                                               LegacyEmulationConfig emulConfig,
                                               int dispatchIndex)

searchDispatch

private static FastSearcher searchDispatch(int searchclusterIndex,
                                           FS4ResourcePool fs4ResourcePool,
                                           com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig,
                                           CacheParams cacheParams,
                                           LegacyEmulationConfig emulConfig,
                                           SummaryParameters docSumParams,
                                           DocumentdbInfoConfig documentdbInfoConfig,
                                           Backend backend,
                                           Dispatcher dispatcher,
                                           int dispatcherIndex)

vdsCluster

private static VdsStreamingSearcher vdsCluster(int searchclusterIndex,
                                               com.yahoo.container.QrSearchersConfig.Searchcluster searchClusterConfig,
                                               CacheParams cacheParams,
                                               LegacyEmulationConfig emulConfig,
                                               SummaryParameters docSumParams,
                                               DocumentdbInfoConfig documentdbInfoConfig)

getBackendStatistics

public Map<String,Backend.BackendStatistics> getBackendStatistics()

createBackend

private Backend createBackend(com.yahoo.container.QrSearchersConfig.Searchcluster.Dispatcher disp)

createCache

private static CacheControl createCache(ClusterConfig config,
                                        String clusterModelName)

getClusterModelName

public String getClusterModelName()

getMonitor

ClusterMonitor getMonitor()

addBackendSearcher

void addBackendSearcher(VespaBackEndSearcher searcher)

addValidRankProfile

void addValidRankProfile(String profileName,
                         String docTypeName)

setValidRankProfile

void setValidRankProfile(String profileName,
                         Set<String> documentTypes)

checkValidRankProfiles

private Result checkValidRankProfiles(Query query,
                                      Set<String> docTypes)

Returns an error if the document types do not have the requested rank profile. For the case of multiple document types, only returns an error if we have restricted the set of documents somehow. This is because when searching over all doc types, common ancestors might not have the requested rank profile and failing on that basis is probably not reasonable.

Parameters:

query - query

docTypes - set of requested doc types for this query

Returns:

null if request rank profile is ok for the requested doc types, a result with error message if not.

fill

public void fill(Result result,
                 String summaryClass,
                 Execution execution)

Description copied from class: Searcher

Fill hit properties with data using the given summary class. Calling this on already filled results has no cost.

This needs to be overridden by federating searchers to contact search sources again by propagating the fill call down through the search chain, and by source searchers which talks to fill capable backends to request the data to be filled. Other searchers do not need to override this.

Overrides:

fill in class Searcher

Parameters:

result - the result to fill

summaryClass - the name of the collection of fields to fetch the values of, or null to use the default

doFill

public void doFill(Searcher searcher,
                   Result result,
                   String summaryClass,
                   Execution execution)

updateCacheHitRatio

private void updateCacheHitRatio(Result result,
                                 Query query)

search

public Result search(Query query,
                     Execution execution)

Description copied from class: Searcher

Override this to implement your searcher.

Searcher implementation subclasses will, depending on their type of logic, do one of the following:

• Query processors: Access the query, then call execution.search and return the result
• Result processors: Call execution.search to get the result, access it and return
• Sources (which produces results): Create a result, add the desired hits and return it.
• Federators (which forwards the search to multiple subchains): Call search on the desired subchains in parallel and get the results. Combine the results to one and return it.
• Workflows: Call execution.search as many times as desired, using different queries. Eventually return a result.

Hits come in two kinds - concrete hits are actual content of the kind requested by the user, meta hits are hits which provides information about the collection of hits, on the query, the service and so on.

The query specifies a window into a larger result list that must be returned from the searcher through hits and offset; Searchers which returns list of hits in the top level in the result must return at least hits number of hits (or if impossible; all that are available), starting at the given offset. In addition, searchers are allowed to return any number of meta hits (although this number is expected to be low). For hits contained in nested hit groups, the concept of a window defined by hits and offset is not well defined and does not apply.

Error handling in searchers:

• Unexpected events: Throw any RuntimeException. This query will fail with the exception message, and the error will be logged
• Expected events: Create (new Result(Query, ErrorMessage) or add result.setErrorIfNoOtherErrors(ErrorMessage) an error message to the Result.
• Recoverable user errors: Add a FeedbackHit explaining the condition and how to correct it.

Specified by:

search in class Searcher

Parameters:

query - the query

Returns:

the result of making this query

validateQueryTimeout

private void validateQueryTimeout(Query query)

validateQueryCache

private void validateQueryCache(Query query)

doSearch

private Result doSearch(Searcher searcher,
                        Query query,
                        Execution execution)

searchMultipleDocumentTypes

private Result searchMultipleDocumentTypes(Searcher searcher,
                                           Query query,
                                           Execution execution)

resolveDocumentTypes

Set<String> resolveDocumentTypes(Query query,
                                 IndexFacts indexFacts)

filterValidDocumentTypes

private Set<String> filterValidDocumentTypes(Collection<String> restrict)

createQueries

private List<Query> createQueries(Query query,
                                  Set<String> docTypes)

cacheHit

private void cacheHit()

cacheMiss

private void cacheMiss()

working

void working(VespaBackEndSearcher node)

NodeManager method, called from ClusterMonitor.

failed

void failed(VespaBackEndSearcher node)

Called from ClusterMonitor.

ping

void ping(VespaBackEndSearcher node)
   throws InterruptedException

Pinging a node, called from ClusterMonitor.

Throws:

InterruptedException

backendCanServeDocuments

private boolean backendCanServeDocuments(Pong pong)

dumpPackets

public void dumpPackets(PacketDumper.PacketType packetType,
                        boolean on)
                 throws IOException

Throws:

IOException

deconstruct

public void deconstruct()

Overrides:

deconstruct in class com.yahoo.component.AbstractComponent

getExecutor

ExecutorService getExecutor()

getScheduledExecutor

ScheduledExecutorService getScheduledExecutor()