Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-07-18 08:36:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

counting search requests in solr interface

Browse Source
This commit is contained in:
Michael Peter Christen
2013-12-12 03:37:19 +01:00
parent 303f5694ba
commit 09412ea3a4
6 changed files with 36 additions and 25 deletions

6
htroot/solr/select.java

@ -152,6 +152,12 @@ public class select {
}
sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time
// count remote searches if this was part of a p2p search
if (post.containsKey("partitions")) {
final int partitions = post.getInt("partitions", 30);
sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter
}
// get the ranking profile id
int profileNr = post.getInt("profileNr", 0);

3
htroot/yacy/seedlist.java

@ -46,10 +46,11 @@ public final class seedlist {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
final ArrayList<Seed> v = sb.peers.getSeedlist(maxcount, includeme, nodeonly);
final ArrayList<Seed> v = sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion);
final serverObjects prop = new serverObjects();
// write simple-encoded seed lines or json

3
source/net/yacy/peers/Protocol.java

@ -1006,13 +1006,14 @@ public final class Protocol {
final int offset,
final int count,
Seed target,
final int partitions,
final Blacklist blacklist) {
if (event.query.getQueryGoal().getOriginalQueryString(false) == null || event.query.getQueryGoal().getOriginalQueryString(false).length() == 0) {
return -1; // we cannot query solr only with word hashes, there is no clear text string
}
event.addExpectedRemoteReferences(count);
if (partitions > 0) solrQuery.set("partitions", partitions);
solrQuery.setStart(offset);
solrQuery.setRows(count);

37
source/net/yacy/peers/RemoteSearch.java

@ -165,29 +165,30 @@ public class RemoteSearch extends Thread {
nodePeers.add(s);
}
}
// start solr searches
if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) {
nodePeers.clear();
nodePeers.add(event.peers.mySeed());
}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image);
for (Seed s: nodePeers) {
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, blacklist);
event.nodeSearchThreads.add(t);
}
}
if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) {
dhtPeers.clear();
dhtPeers.add(event.peers.mySeed());
}
if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) {
nodePeers.clear();
nodePeers.add(event.peers.mySeed());
}
// start solr searches
final int targets = dhtPeers.size() + nodePeers.size();
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image);
for (Seed s: nodePeers) {
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist);
event.nodeSearchThreads.add(t);
}
}
// start search to YaCy DHT peers
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) {
final int targets = dhtPeers.size();
if (targets == 0) return;
for (int i = 0; i < targets; i++) {
for (int i = 0; i < dhtPeers.size(); i++) {
if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue;
try {
RemoteSearch rs = new RemoteSearch(
@ -271,6 +272,7 @@ public class RemoteSearch extends Thread {
final int start,
final int count,
final Seed targetPeer,
final int partitions,
final Blacklist blacklist) {
assert solrQuery != null;
@ -290,6 +292,7 @@ public class RemoteSearch extends Thread {
start,
count,
targetPeer,
partitions,
blacklist);
if (urls >= 0) {
// urls is an array of url hashes. this is only used for log output

6
source/net/yacy/peers/SeedDB.java

@ -717,7 +717,7 @@ public final class SeedDB implements AlternativeDomainNames {
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false);
List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);
String line;
for (Seed seed: seedlist) {
line = seed.genSeedStr(null);
@ -731,7 +731,7 @@ public final class SeedDB implements AlternativeDomainNames {
return v;
}
public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly) {
public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly, float minversion) {
final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);
// store own peer seed
@ -739,7 +739,7 @@ public final class SeedDB implements AlternativeDomainNames {
// store active peer seeds
Seed ys;
Iterator<Seed> se = this.seedsConnected(true, false, null, (float) 0.0);
Iterator<Seed> se = this.seedsConnected(true, false, null, minversion);
while (se.hasNext() && v.size() < maxcount) {
ys = se.next();
if (ys != null && (!nodeonly || ys.getFlagRootNode())) v.add(ys);

6
source/net/yacy/search/query/SearchEvent.java

@ -287,7 +287,7 @@ public final class SearchEvent {
// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset = this.query.itemsPerPage;
@ -1375,7 +1375,7 @@ public final class SearchEvent {
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
}
@ -1396,7 +1396,7 @@ public final class SearchEvent {
if (this.localsolrsearch == null || !this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) {
// at the end of a list, trigger a next solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += this.query.itemsPerPage;
}