mirror of
https://github.com/yacy/yacy_search_server.git
synced 2025-07-20 08:54:37 -04:00
fixed cleaning of index
This commit is contained in:
source/net/yacy
@ -20,6 +20,7 @@
|
||||
|
||||
package net.yacy.cora.federate.solr.instance;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@ -120,6 +121,17 @@ public class InstanceMirror {
|
||||
return esc;
|
||||
}
|
||||
|
||||
public RemoteSolrConnector getDefaultRemoteConnector(boolean useBinaryResponseWriter) throws IOException {
|
||||
if (this.remoteSolrInstance == null) return null;
|
||||
String coreName = this.getDefaultCoreName();
|
||||
if (coreName == null) return null;
|
||||
RemoteSolrConnector esc = this.remoteConnectorCache.get(coreName);
|
||||
if (esc != null) return esc;
|
||||
esc = new RemoteSolrConnector(this.remoteSolrInstance, useBinaryResponseWriter);
|
||||
this.remoteConnectorCache.put(coreName, esc);
|
||||
return esc;
|
||||
}
|
||||
|
||||
public EmbeddedSolrConnector getEmbeddedConnector(String corename) {
|
||||
if (this.embeddedSolrInstance == null) return null;
|
||||
EmbeddedSolrConnector esc = this.embeddedConnectorCache.get(corename);
|
||||
|
@ -2580,8 +2580,7 @@ public final class Switchboard extends serverSwitch {
|
||||
} else {
|
||||
// we consider this as fail urls to have a tracking of the problem
|
||||
if (rejectReason != null && !rejectReason.startsWith("double in")) {
|
||||
final CrawlProfile profile = this.crawler.get(UTF8.getBytes(response.profile().handle()));
|
||||
this.crawlStacker.nextQueue.errorURL.push(response.url(), profile, FailCategory.FINAL_LOAD_CONTEXT, rejectReason, -1);
|
||||
this.crawlStacker.nextQueue.errorURL.push(response.url(), response.profile(), FailCategory.FINAL_LOAD_CONTEXT, rejectReason, -1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2600,8 +2599,10 @@ public final class Switchboard extends serverSwitch {
|
||||
) {
|
||||
// get the hyperlinks
|
||||
final Map<DigestURL, String> hl = Document.getHyperlinks(documents);
|
||||
for (Map.Entry<DigestURL, String> entry: Document.getImagelinks(documents).entrySet()) {
|
||||
if (TextParser.supportsExtension(entry.getKey()) == null) hl.put(entry.getKey(), entry.getValue());
|
||||
if (response.profile().indexMedia()) {
|
||||
for (Map.Entry<DigestURL, String> entry: Document.getImagelinks(documents).entrySet()) {
|
||||
if (TextParser.supportsExtension(entry.getKey()) == null) hl.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
// add all media links also to the crawl stack. They will be re-sorted to the NOLOAD queue and indexed afterwards as pure links
|
||||
|
@ -168,10 +168,9 @@ public final class Fulltext {
|
||||
}
|
||||
|
||||
public RemoteSolrConnector getDefaultRemoteSolrConnector() {
|
||||
if (this.solrInstances.getRemote() == null) return null;
|
||||
try {
|
||||
return new RemoteSolrConnector(this.solrInstances.getRemote(), true);
|
||||
} catch (final IOException e) {
|
||||
return this.solrInstances.getDefaultRemoteConnector(true);
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -217,7 +216,9 @@ public final class Fulltext {
|
||||
synchronized (this.solrInstances) {
|
||||
EmbeddedInstance instance = this.solrInstances.getEmbedded();
|
||||
if (instance != null) {
|
||||
for (String name: instance.getCoreNames()) new EmbeddedSolrConnector(instance, name).clear();
|
||||
for (String name: instance.getCoreNames()) {
|
||||
this.solrInstances.getEmbeddedConnector(name).clear();
|
||||
}
|
||||
}
|
||||
this.commit(false);
|
||||
this.solrInstances.clearCaches();
|
||||
@ -228,7 +229,9 @@ public final class Fulltext {
|
||||
synchronized (this.solrInstances) {
|
||||
ShardInstance instance = this.solrInstances.getRemote();
|
||||
if (instance != null) {
|
||||
for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, true, name).clear();
|
||||
for (String name: instance.getCoreNames()) {
|
||||
this.solrInstances.getRemoteConnector(name).clear();
|
||||
}
|
||||
}
|
||||
this.solrInstances.clearCaches();
|
||||
}
|
||||
|
Reference in New Issue
Block a user