Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-03-11 13:21:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

Automatically adjust crawling load limit to the local machine cpu cores

Browse Source 
The settings in the default configuration file is historic. Many
machines have much more CPU cores today and now an auto-scaling to this
hardware is better.
This commit is contained in:
Michael Peter Christen 2024-11-25 00:30:36 +01:00
parent 16e031caeb
commit feca150672
3 changed files with 11 additions and 3 deletions

3
source/net/yacy/htroot/Crawler_p.java
View File

@ -803,7 +803,8 @@ public class Crawler_p {
// we must increase the load limit because a conservative load limit will prevent a high crawling speed
// however this must not cause that the load limit is reduced again because that may go against the users requirements
// in case they set the limit themself, see https://github.com/yacy/yacy_search_server/issues/363
float loadprereq = wantedPPM <= 10 ? 1.0f : wantedPPM <= 100 ? 2.0f : wantedPPM >= 1000 ? 8.0f : 3.0f;
float numberOfCores2 = 2.0f * (float) Runtime.getRuntime().availableProcessors();
float loadprereq = wantedPPM <= 10 ? 1.0f : wantedPPM <= 100 ? 2.0f : wantedPPM >= 1000 ? numberOfCores2 : 3.0f;
loadprereq = Math.max(loadprereq, sb.getConfigFloat(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_LOADPREREQ, loadprereq));
BusyThread thread;

7
source/net/yacy/search/Switchboard.java
View File

@ -546,6 +546,13 @@ public final class Switchboard extends serverSwitch {
solrWebgraphConfigurationWork.commit();
} catch (final IOException e) {ConcurrentLog.logException(e);}
// define load limitation according to current number of cpu cores
if (this.firstInit) {
float numberOfCores2 = 2.0f * (float) Runtime.getRuntime().availableProcessors();
sb.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_LOADPREREQ, numberOfCores2);
sb.setConfig(SwitchboardConstants.SURROGATES_LOADPREREQ, numberOfCores2);
}
// define boosts
Ranking.setMinTokenLen(this.getConfigInt(SwitchboardConstants.SEARCH_RANKING_SOLR_DOUBLEDETECTION_MINLENGTH, 3));
Ranking.setQuantRate(this.getConfigFloat(SwitchboardConstants.SEARCH_RANKING_SOLR_DOUBLEDETECTION_QUANTRATE, 0.5f));

4
source/net/yacy/search/SwitchboardConstants.java
View File

@ -283,13 +283,13 @@ public final class SwitchboardConstants {
public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi";
/** Default maximum system load allowing remote RWI searches */
public static final float REMOTESEARCH_MAXLOAD_RWI_DEFAULT = 8.0f;
public static final float REMOTESEARCH_MAXLOAD_RWI_DEFAULT = 2.0f * (float) Runtime.getRuntime().availableProcessors();
/** Setting key to configure the maximum system load allowing remote Solr searches */
public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr";
/** Default maximum system load allowing remote Solr searches */
public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f;
public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = (float) Runtime.getRuntime().availableProcessors();
/** Key of the setting controlling whether https should be preferred for remote searches, when available on the target peer */
public static final String REMOTESEARCH_HTTPS_PREFERRED = "remotesearch.https.preferred";