mirror of
https://github.com/yacy/yacy_search_server.git
synced 2025-07-19 08:44:42 -04:00
Optionally allow external Solr server with self-signed certificate
This is necessary when you want to attach to a dedicated external Solr server protected with basic http authentication and requested over https but having only a self-signed certificate.
This commit is contained in:
luccioman
defaults
htroot
source/net/yacy
cora
federate
peers
search
@ -1285,6 +1285,8 @@ federated.service.solr.indexing.sharding = MODULO_HOST_MD5
|
||||
federated.service.solr.indexing.lazy = true
|
||||
federated.service.solr.indexing.timeout = 60000
|
||||
federated.service.solr.indexing.writeEnabled = true
|
||||
# Set to true when TLS is enabled with a self-signed certificated on a main external Solr with authentication required ( its URL looks like https://user:password@localhost:8984/solr/)
|
||||
federated.service.solr.indexing.authenticated.allowSelfSigned = false
|
||||
|
||||
# temporary definition of backend services to use.
|
||||
# After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db.
|
||||
|
||||
@ -36,6 +36,10 @@
|
||||
<dt>Use remote Solr server(s) <input type="checkbox" name="solr.indexing.solrremote" id="solr_indexing_solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# onclick="if(!document.getElementById('config').solr_indexing_solrremote.checked) {document.getElementById('config').core_service_fulltext.checked = true;}"/></dt>
|
||||
<dd>It's easy to <a href="http://www.yacy-websearch.net/wiki/index.php/Dev:Solr" target="_blank">attach an external Solr to YaCy</a>.
|
||||
This external Solr can be used instead the internal Solr. It can also be used additionally to the internal Solr, then both Solr indexes are mirrored.
|
||||
</dd>
|
||||
|
||||
<dt>Allow self-signed certificates <input type="checkbox" name="federated.service.solr.indexing.authenticated.allowSelfSigned" #(federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#:: checked="checked"#(/federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#/></dt>
|
||||
<dd>Tick this when the remote Solr server is password protected and is requested over HTTPS but provides only a self-signed certificate (not a validated one by an official Certificate Authority). The Solr URL could be for example something like <i>https://user:password@localhost:8984/solr</i>.
|
||||
</dd>
|
||||
|
||||
#(table)#::
|
||||
|
||||
@ -104,6 +104,10 @@ public class IndexFederated_p {
|
||||
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
|
||||
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
|
||||
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
|
||||
|
||||
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post
|
||||
.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));
|
||||
|
||||
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
|
||||
final StringBuilder s = new StringBuilder();
|
||||
String s0;
|
||||
@ -142,10 +146,13 @@ public class IndexFederated_p {
|
||||
// switch on
|
||||
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
|
||||
final int solrtimeout = sb.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 10000);
|
||||
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
|
||||
try {
|
||||
if (usesolr) {
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
|
||||
sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
|
||||
} else {
|
||||
sb.index.fulltext().disconnectRemoteSolr();
|
||||
@ -191,6 +198,9 @@ public class IndexFederated_p {
|
||||
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
|
||||
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
|
||||
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
|
||||
prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",
|
||||
env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));
|
||||
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
|
||||
String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
|
||||
int mc = 0;
|
||||
|
||||
@ -96,7 +96,13 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
|
||||
msp.add(CommonParams.QT, "/"); // important to override default append of /select
|
||||
msp.add(CommonParams.ROWS, Integer.toString(query.itemsPerPage));
|
||||
try {
|
||||
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000);
|
||||
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
|
||||
if (Switchboard.getSwitchboard() != null) {
|
||||
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
}
|
||||
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000, trustSelfSignedOnAuthenticatedServer);
|
||||
try {
|
||||
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
|
||||
if (Switchboard.getSwitchboard() != null) {
|
||||
|
||||
@ -135,7 +135,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
|
||||
public static void main(final String args[]) {
|
||||
RemoteSolrConnector solr;
|
||||
try {
|
||||
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000);
|
||||
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000, false);
|
||||
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
|
||||
instances.add(instance);
|
||||
solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, true), true, "solr");
|
||||
|
||||
@ -77,18 +77,51 @@ public class RemoteInstance implements SolrInstance {
|
||||
private final Map<String, ConcurrentUpdateSolrClient> server;
|
||||
private final int timeout;
|
||||
|
||||
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
|
||||
/**
|
||||
* @param urlList
|
||||
* the list of URLs of remote Solr shard instances. Must not be null.
|
||||
* @param coreNames
|
||||
* the Solr core names for the main collection and the webgraph
|
||||
* @param defaultCoreName
|
||||
* the core name of the main collection
|
||||
* @param timeout
|
||||
* the connection timeout in milliseconds
|
||||
* @param trustSelfSignedOnAuthenticatedServer
|
||||
* when true, self-signed certificates are accepcted for an https
|
||||
* connection to a remote server with authentication credentials
|
||||
* @throws IOException
|
||||
* when a connection could not be opened to a remote Solr instance
|
||||
*/
|
||||
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames,
|
||||
String defaultCoreName, final int timeout, final boolean trustSelfSignedOnAuthenticatedServer)
|
||||
throws IOException {
|
||||
urlList.replace(' ', ',');
|
||||
String[] urls = CommonPattern.COMMA.split(urlList);
|
||||
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
|
||||
for (final String u: urls) {
|
||||
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout);
|
||||
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer);
|
||||
instances.add(instance);
|
||||
}
|
||||
return instances;
|
||||
}
|
||||
|
||||
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
|
||||
/**
|
||||
* @param url
|
||||
* the remote Solr URL. A default localhost URL is assumed when null.
|
||||
* @param coreNames
|
||||
* the Solr core names for the main collection and the webgraph
|
||||
* @param defaultCoreName
|
||||
* the core name of the main collection
|
||||
* @param timeout
|
||||
* the connection timeout in milliseconds
|
||||
* @param trustSelfSignedOnAuthenticatedServer
|
||||
* when true, self-signed certificates are accepcted for an https
|
||||
* connection to a remote server with authentication credentials
|
||||
* @throws IOException
|
||||
* when a connection could not be opened to the remote Solr instance
|
||||
*/
|
||||
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
|
||||
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer) throws IOException {
|
||||
this.timeout = timeout;
|
||||
this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
|
||||
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
|
||||
@ -140,8 +173,7 @@ public class RemoteInstance implements SolrInstance {
|
||||
}
|
||||
}
|
||||
if (solraccount.length() > 0) {
|
||||
/* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
|
||||
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
|
||||
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, trustSelfSignedOnAuthenticatedServer);
|
||||
} else if(u.isHTTPS()){
|
||||
/* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
|
||||
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);
|
||||
|
||||
@ -1102,7 +1102,14 @@ public final class Protocol {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
|
||||
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
|
||||
if (Switchboard.getSwitchboard() != null) {
|
||||
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
}
|
||||
|
||||
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
|
||||
try {
|
||||
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
|
||||
if (Switchboard.getSwitchboard() != null) {
|
||||
|
||||
@ -255,7 +255,11 @@ public class AutoSearch extends AbstractBusyThread {
|
||||
solrQuery.set(CommonParams.ROWS, sb.getConfig(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, "20"));
|
||||
this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")");
|
||||
try {
|
||||
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection
|
||||
final boolean trustSelfSignedOnAuthenticatedServer = this.sb.getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
|
||||
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
|
||||
try {
|
||||
SolrConnector solrConnector = new RemoteSolrConnector(instance,
|
||||
sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
|
||||
|
||||
@ -585,10 +585,13 @@ public final class Switchboard extends serverSwitch {
|
||||
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
|
||||
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
|
||||
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
|
||||
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
|
||||
if (usesolr && solrurls != null && solrurls.length() > 0) {
|
||||
try {
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
|
||||
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
|
||||
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
|
||||
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
|
||||
@ -1486,10 +1489,13 @@ public final class Switchboard extends serverSwitch {
|
||||
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
|
||||
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
|
||||
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
|
||||
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
|
||||
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
|
||||
|
||||
if (usesolr && solrurls != null && solrurls.length() > 0) {
|
||||
try {
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
|
||||
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
|
||||
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
|
||||
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
|
||||
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
|
||||
|
||||
@ -316,6 +316,13 @@ public final class SwitchboardConstants {
|
||||
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";
|
||||
public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT = "federated.service.solr.indexing.timeout";
|
||||
public static final String FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED = "federated.service.solr.indexing.writeEnabled";
|
||||
|
||||
/** Setting key controlling whether a self-signed certificate is acceptable from a remote Solr instance requested with authentication credentials.
|
||||
* This has no impact on connections to remote Solr instances used in p2p search for which self-signed certificates are always accepted. */
|
||||
public static final String FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED = "federated.service.solr.indexing.authenticated.allowSelfSigned";
|
||||
|
||||
/** Default value controlling whether a self-signed certificate is acceptable from a remote Solr instance with authentication credentials. */
|
||||
public static final boolean FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT = false;
|
||||
|
||||
public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext";
|
||||
public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp";
|
||||
|
||||
Reference in New Issue
Block a user