Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-07-17 08:26:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Integrated HostBrowser back to administration interface; it can appear

Browse Source 
with and without navigation bar.
This commit is contained in:
Michael Peter Christen
2014-03-31 18:19:24 +02:00
parent 94366ba2e5
commit ff82a80eb3
4 changed files with 50 additions and 31 deletions

22
htroot/HostBrowser.html

@ -47,7 +47,7 @@ function updatepage(str) {
html += "<td>URL from index (total results = " + totalResults + ")<\/td>";
for (var i = 0; i < firstChannel.items.length; i++) {
item = firstChannel.items[i];
html += "<tr class=\"TableCellLight\"><td align=\"left\"><a href=\"HostBrowser.html?pathsearch=&amp;path=" + item.link + "\">" + item.link + "<\/a><\/td>";
html += "<tr class=\"TableCellLight\"><td align=\"left\"><a href=\"HostBrowser.html?admin=#[admin]#&pathsearch=&amp;path=" + item.link + "\">" + item.link + "<\/a><\/td>";
}
html += "<\/table>";
}
@ -70,11 +70,15 @@ function updatepage(str) {
<script type="text/javascript">
document.getElementById("header_hostbrowser").className += " active";
</script>
::
#%env/templates/header.template%#
#%env/templates/submenuWebStructure.template%#
#(/topmenu)#
<h2>Host Browser</h2>
<p>Browse the index of #[ucount]# documents. Enter a host or an URL for a file list or view a list of <a href="HostBrowser.html?hosts=">all hosts</a>, <a href="HostBrowser.html?hosts=crawling">only hosts with urls pending in the crawler</a> or <a href="HostBrowser.html?hosts=error">only with load errors</a>.</p>
<p>Browse the index of #[ucount]# documents. Enter a host or an URL for a file list or view a list of <a href="HostBrowser.html?admin=#[admin]#&hosts=">all hosts</a>, <a href="HostBrowser.html?admin=#[admin]#&hosts=crawling">only hosts with urls pending in the crawler</a> or <a href="HostBrowser.html?admin=#[admin]#&hosts=error">only with load errors</a>.</p>
<form action="HostBrowser.html" id="searchform" method="get" onkeyup="xmlhttpPost(); return false;">
<fieldset class="yacys">
<input type="hidden" name="admin" id="admin" value="#[admin]#" />
Host/URL:
<input id="search" type="text" name="path" value="#[path]#" size="80" maxlength="250" />
<input type="submit" name="list" value="Browse Host" class="btn btn-primary" style="width:160px;"/>
@ -92,7 +96,7 @@ function updatepage(str) {
<fieldset><legend>Host List</legend>
#{list}#
<div style="float:left; padding:1px 5px 1px 5px;">
<div style="width:180px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><img src="env/grafics/#(type)#invisible.png::burn-e.gif::construction.gif#(/type)#" align="left" width="12" height="8">&nbsp;<a href="HostBrowser.html?path=#[host]#&facetcount=#[count]#">#[host]#</a></div></div>
<div style="width:180px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><img src="env/grafics/#(type)#invisible.png::burn-e.gif::construction.gif#(/type)#" align="left" width="12" height="8">&nbsp;<a href="HostBrowser.html?admin=#[admin]#&path=#[host]#&facetcount=#[count]#">#[host]#</a></div></div>
<div style="width:120px; text-align:right; float: left; white-space:nowrap; overflow:hidden;"><span class="commit">#[count]#</span>#(crawler)#::/<span class="pending">#[pending]#</span>#(/crawler)##(errors)#::/<span class="info">#[exclcount]#</span>/<span class="error">#[failcount]#</span>#(/errors)# URLs</div>
</div>
#{/list}#
@ -108,7 +112,7 @@ function updatepage(str) {
#(files)#::
<fieldset><legend>Browser for <a href="#[path]#" target="_blank">#[path]#</a></legend>
<p>documents stored for host: #[hostsize]#; documents stored for subpath: #[subpathloadsize]#; unloaded documents detected in subpath: #[subpathdetectedsize]# <!-- #(complete)#;<a href="HostBrowser.html?complete=true&path=#[path]#">get complete list</a>::<a href="HostBrowser.html?path=#[path]#">directory view</a>#(/complete)#-->
<p>documents stored for host: #[hostsize]#; documents stored for subpath: #[subpathloadsize]#; unloaded documents detected in subpath: #[subpathdetectedsize]# <!-- #(complete)#;<a href="HostBrowser.html?admin=#[admin]#&complete=true&path=#[path]#">get complete list</a>::<a href="HostBrowser.html?admin=#[admin]#&path=#[path]#">directory view</a>#(/complete)#-->
</p>
<table class="sortable" border="0" style="float:left">
<tr>
@ -123,7 +127,7 @@ function updatepage(str) {
#(root)#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"></td>
<td align="left" nowrap ><a href="HostBrowser.html?path=#[path]#" class="listing">..</a></td>
<td align="left" nowrap ><a href="HostBrowser.html?admin=#[admin]#&path=#[path]#" class="listing">..</a></td>
<td align="right" colspan="5" nowrap></td>
</tr>::
#(/root)#
@ -133,7 +137,7 @@ function updatepage(str) {
<td align="center"><div id="info"><a href="ViewFile.html?url=#[url]#"><img src="env/grafics/doc.gif"/></a><span>Show Metadata</span></div></td>
<td align="left" nowrap class=#(stored)#"listingem"::"listing"#(/stored)#><a href="#[url]#" target="_blank">#[url]#</a></td>
#(stored)#
#(load)#<td align="left" colspan="5" nowrap class="listingem">link, detected from context</td>::<td align="left" colspan="5" nowrap class="listingnok"><a href="HostBrowser.html?load=#[url]#&path=#[path]#">load &amp; index</a>#(/load)#</td>::
#(load)#<td align="left" colspan="5" nowrap class="listingem">link, detected from context</td>::<td align="left" colspan="5" nowrap class="listingnok"><a href="HostBrowser.html?admin=#[admin]#&load=#[url]#&path=#[path]#">load &amp; index</a>#(/load)#</td>::
<td align="left" colspan="1" nowrap class="listingok">indexed</td><td align="left" colspan="4" nowrap class="listingem">#[comment]#</td>::
<td align="left" colspan="5" nowrap class="pending">loading</td>::
<td align="left" colspan="5" nowrap class="listingnok">#[error]#</td>
@ -141,7 +145,7 @@ function updatepage(str) {
</tr>::<!--folder-->
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"><img src="env/grafics/dir.gif"/></td>
<td align="left" nowrap class="listing"><a href="HostBrowser.html?path=#[url]#" class="listing">#[url]#</a></td>
<td align="left" nowrap class="listing"><a href="HostBrowser.html?admin=#[admin]#&path=#[url]#" class="listing">#[url]#</a></td>
<td align="right" class="commit">#[stored]#</td>
<td align="right" class="listing">#[linked]#</td>
<td align="right" #(pendingVisible)#class="listingem"::class="pending"#(/pendingVisible)#>#[pending]#</td>
@ -158,7 +162,7 @@ function updatepage(str) {
<fieldset><legend>Outbound Links, outgoing from #[host]# - Host List</legend>
#{list}#
<div style="float:left; padding:1px 5px 1px 5px;">
<div style="width:160px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><a href="HostBrowser.html?path=#[link]#">#[host]#</a></div></div>
<div style="width:160px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><a href="HostBrowser.html?admin=#[admin]#&path=#[link]#">#[host]#</a></div></div>
<div style="width:80px; text-align:right; float: left; white-space:nowrap; overflow:hidden;">#[count]# URLs</div>
</div>
#{/list}#
@ -174,7 +178,7 @@ function updatepage(str) {
<fieldset><legend>Inbound Links, incoming to #[host]# - Host List</legend>
#{list}#
<div style="float:left; padding:1px 5px 1px 5px;">
<div style="width:160px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><a href="HostBrowser.html?path=#[host]#">#[host]#</a></div></div>
<div style="width:160px; text-align:left; float: left; white-space:nowrap; overflow:hidden;"><div id="info"><a href="HostBrowser.html?admin=#[admin]#&path=#[host]#">#[host]#</a></div></div>
<div style="width:80px; text-align:right; float: left; white-space:nowrap; overflow:hidden;">#[count]# URLs</div>
</div>
#{/list}#

54
htroot/HostBrowser.java

@ -77,11 +77,11 @@ public class HostBrowser {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
Fulltext fulltext = sb.index.fulltext();
final boolean admin = sb.verifyAuthentication(header);
final boolean autoload = admin && sb.getConfigBool("browser.autoload", true);
final boolean authorized = sb.verifyAuthentication(header);
final boolean autoload = authorized && sb.getConfigBool("browser.autoload", true);
final boolean load4everyone = sb.getConfigBool("browser.load4everyone", false);
final boolean loadRight = autoload || load4everyone; // add config later
final boolean searchAllowed = sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true) || admin;
final boolean searchAllowed = sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true) || authorized;
final serverObjects prop = new serverObjects();
@ -90,10 +90,17 @@ public class HostBrowser {
prop.put("result", "");
prop.put("hosts", 0);
prop.put("files", 0);
prop.put("admin", admin ? 1 : 0);
prop.put("admin", "false");
boolean admin = false;
if (admin) { // show top nav to admins
prop.put("topmenu",1);
String referer = header.get("Referer", "");
if ((post != null && post.getBoolean("admin")) || referer.contains("HostBrowser.html?admin=true")) {
prop.put("topmenu", 2);
prop.put("admin", "true");
admin = true;
} else if (authorized) { // show top nav to admins
prop.put("topmenu", 1);
} else { // for other respect setting in Search Design Configuration
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
}
@ -110,7 +117,7 @@ public class HostBrowser {
}
String path = post == null ? "" : post.get("path", "").trim();
if (admin) sb.index.fulltext().commit(true);
if (authorized) sb.index.fulltext().commit(true);
if (post == null || env == null) {
prop.putNum("ucount", fulltext.collectionSize());
return prop;
@ -125,7 +132,7 @@ public class HostBrowser {
!path.startsWith("smb://") &&
!path.startsWith("file://"))) { path = "http://" + path; }
prop.putHTML("path", path);
prop.put("delete", admin && path.length() > 0 ? 1 : 0);
prop.put("delete", authorized && path.length() > 0 ? 1 : 0);
DigestURL pathURI = null;
try {pathURI = new DigestURL(path);} catch (final MalformedURLException e) {}
@ -159,7 +166,7 @@ public class HostBrowser {
}
}
if (admin && post.containsKey("deleteLoadErrors")) {
if (authorized && post.containsKey("deleteLoadErrors")) {
try {
fulltext.getDefaultConnector().deleteByQuery("-" + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200 AND "
+ CollectionSchema.httpstatus_i.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM); // make sure field exists
@ -181,19 +188,19 @@ public class HostBrowser {
boolean onlyCrawling = "crawling".equals(post.get("hosts", ""));
boolean onlyErrors = "error".equals(post.get("hosts", ""));
int maxcount = admin ? 2 * 3 * 2 * 5 * 7 * 2 * 3 : 360; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums
int maxcount = authorized ? 2 * 3 * 2 * 5 * 7 * 2 * 3 : 360; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums
// collect hosts from index
ReversibleScoreMap<String> hostscore = fulltext.getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_QUERY, maxcount, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
if (hostscore == null) hostscore = new ClusteredScoreMap<String>();
// collect hosts from crawler
final Map<String, Integer[]> crawler = (admin) ? sb.crawlQueues.noticeURL.getDomainStackHosts(StackType.LOCAL, sb.robots) : new HashMap<String, Integer[]>();
final Map<String, Integer[]> crawler = (authorized) ? sb.crawlQueues.noticeURL.getDomainStackHosts(StackType.LOCAL, sb.robots) : new HashMap<String, Integer[]>();
// collect the errorurls
Map<String, ReversibleScoreMap<String>> exclfacets = admin ? fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName()) : null;
Map<String, ReversibleScoreMap<String>> exclfacets = authorized ? fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName()) : null;
ReversibleScoreMap<String> exclscore = exclfacets == null ? new ClusteredScoreMap<String>() : exclfacets.get(CollectionSchema.host_s.getSolrFieldName());
Map<String, ReversibleScoreMap<String>> failfacets = admin ? fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName()) : null;
Map<String, ReversibleScoreMap<String>> failfacets = authorized ? fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName()) : null;
ReversibleScoreMap<String> failscore = failfacets == null ? new ClusteredScoreMap<String>() : failfacets.get(CollectionSchema.host_s.getSolrFieldName());
int c = 0;
@ -201,6 +208,7 @@ public class HostBrowser {
String host;
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.put("hosts_list_" + c + "_admin", admin ? "true" : "false");
prop.putHTML("hosts_list_" + c + "_host", host);
boolean inCrawler = crawler.containsKey(host);
int exclcount = exclscore.get(host);
@ -233,11 +241,11 @@ public class HostBrowser {
if (path.length() > 0) {
boolean delete = false;
boolean reload404 = false;
if (admin && post.containsKey("delete")) {
if (authorized && post.containsKey("delete")) {
// delete the complete path!! That includes everything that matches with this prefix.
delete = true;
}
if (admin && post.containsKey("reload404")) {
if (authorized && post.containsKey("reload404")) {
// try to re-load all urls that have load errors and matches with this prefix.
reload404 = true;
}
@ -248,6 +256,7 @@ public class HostBrowser {
if (p > 0) path = path.substring(0, p + 1);
}
prop.put("files_complete", complete ? 1 : 0);
prop.put("files_complete_admin", admin ? "true" : "false");
prop.putHTML("files_complete_path", path);
p = path.substring(0, path.length() - 1).lastIndexOf('/');
if (p < 8) {
@ -255,13 +264,14 @@ public class HostBrowser {
} else {
prop.put("files_root", 0);
prop.putHTML("files_root_path", path.substring(0, p + 1));
prop.put("files_root_admin", admin ? "true" : "false");
}
try {
// generate file list from path
DigestURL uri = new DigestURL(path);
String host = uri.getHost();
prop.putHTML("outbound_host", host);
if (admin) prop.putHTML("outbound_admin_host", host); //used for WebStructurePicture_p link
if (authorized) prop.putHTML("outbound_admin_host", host); //used for WebStructurePicture_p link
prop.putHTML("inbound_host", host);
String hosthash = ASCII.String(uri.hash(), 6, 6);
String[] pathparts = uri.getPaths();
@ -324,12 +334,12 @@ public class HostBrowser {
if (collections != null) reloadURLCollection.addAll(collections);
reloadURLs.add(u);
}
if (admin) errorDocs.put(u, error);
if (authorized) errorDocs.put(u, error);
}
}
} else if (complete) {
if (error == null) storedDocs.add(u); else {
if (admin) errorDocs.put(u, error);
if (authorized) errorDocs.put(u, error);
}
}
if ((complete || u.startsWith(path)) && !storedDocs.contains(u)) inboundLinks.add(u); // add the current link
@ -369,7 +379,7 @@ public class HostBrowser {
}
// collect from crawler
List<Request> domainStackReferences = (admin) ? sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000) : new ArrayList<Request>(0);
List<Request> domainStackReferences = (authorized) ? sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000) : new ArrayList<Request>(0);
Set<String> loadingLinks = new HashSet<String>();
for (Request crawlEntry: domainStackReferences) loadingLinks.add(crawlEntry.url().toNormalform(true));
@ -430,6 +440,7 @@ public class HostBrowser {
// this is a folder
prop.put("files_list_" + c + "_type", 1);
prop.putHTML("files_list_" + c + "_type_url", entry.getKey());
prop.putHTML("files_list_" + c + "_type_admin", admin ? "true" : "false");
int linked = ((int[]) entry.getValue())[0];
int stored = ((int[]) entry.getValue())[1];
int crawler = ((int[]) entry.getValue())[2];
@ -452,6 +463,7 @@ public class HostBrowser {
// this is a file
prop.put("files_list_" + c + "_type", 0);
prop.putHTML("files_list_" + c + "_type_url", entry.getKey());
prop.putHTML("files_list_" + c + "_type_admin", admin ? "true" : "false");
StoreType type = (StoreType) entry.getValue();
try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;}
HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash());
@ -463,7 +475,7 @@ public class HostBrowser {
if (type == StoreType.INDEX) {
String ids = ASCII.String(uri.hash());
InfoCacheEntry ice = infoCache.get(ids);
prop.put("files_list_" + c + "_type_stored_comment", ice.toString()); // ice.toString() contains html, therefore do not use putHTML here
prop.put("files_list_" + c + "_type_stored_comment", ice == null ? "" : ice.toString()); // ice.toString() contains html, therefore do not use putHTML here
}
prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0);
if (error) {
@ -503,6 +515,7 @@ public class HostBrowser {
Iterator<String> i = score.keys(false);
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.put("inbound_list_" + c + "_admin", admin ? "true" : "false");
prop.putHTML("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host));
prop.put("inbound_list_" + c + "_count", score.get(host));
c++;
@ -525,6 +538,7 @@ public class HostBrowser {
prop.putHTML("outbound_list_" + c + "_host", host);
prop.put("outbound_list_" + c + "_count", score.get(host));
prop.put("outbound_list_" + c + "_link", outboundHosts.get(host).getMinKey());
prop.put("outbound_list_" + c + "_admin", admin ? "true" : "false");
c++;
}
prop.put("outbound_list", c);

2
htroot/env/templates/header.template vendored

@ -76,8 +76,8 @@
<li><h3>Monitoring</h3></li>
<li><a href="Status.html" class="MenuItemLink">System Status</a></li>
<li><a href="Network.html" accesskey="w" class="MenuItemLink">YaCy Peer-to-Peer Network</a></li>
<li><a href="HostBrowser.html?admin=true" class="MenuItemLink">Web Visualization</a></li>
<li><a href="Crawler_p.html" class="MenuItemLink lock">Crawler Monitor</a></li>
<li><a href="WatchWebStructure_p.html?host=auto&amp;depth=3&amp;time=1000" class="MenuItemLink lock">Web Visualization</a></li>
<li><a href="AccessGrid_p.html" class="MenuItemLink lock">Network Access</a></li>
<!-- <li><a href="terminal_p.html" accesskey="t" class="MenuItemLink lock">Terminal</a></li> -->
</ul>

3
htroot/env/templates/submenuWebStructure.template vendored

@ -1,7 +1,8 @@
<div class="SubMenu">
<h3>Web Visualization</h3>
<ul class="SubMenu">
<li><a href="WatchWebStructure_p.html?host=auto&amp;depth=2&amp;time=1000" class="MenuItemLink lock">Web Structure</a></li>
<li><a href="HostBrowser.html?admin=true" class="MenuItemLink">Host Browser</a></li>
<li><a href="WatchWebStructure_p.html?host=auto&amp;depth=3&amp;time=1000" class="MenuItemLink lock">Web Structure</a></li>
<li><a href="Collage.html" class="MenuItemLink">Image Collage</a></li>
</ul>
</div>