Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-03-11 13:21:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

Improved URLLicence reliability for use by conccurrent non authaurized

Browse Source 
users.
Removed URLLicence generation when unnecessary (authorized users)
This commit is contained in:
luc 2016-01-08 20:42:57 +01:00
parent e3d53f0248
commit cfdbc2b487
5 changed files with 188 additions and 34 deletions
htroot
yacysearchitem.htmlyacysearchitem.java
source/net/yacy
cora/federate/solr/responsewriter
YJsonResponseWriter.java
data
URLLicense.java
test/net/yacy/data
URLLicenseConcurrentTest.java

2
htroot/yacysearchitem.html
View File

@ -1,7 +1,7 @@
#(content)#::
<div class="searchresults">
<h4 class="linktitle">
<img width="16" height="16" src="ViewImage.png?width=16&amp;height=16&amp;code=#[faviconCode]#&amp;isStatic=true" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" />
<img width="16" height="16" src="#[faviconUrl]#" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" />
<a href="#[link]#" target="#[target]#">#[title]#</a></h4>
<div class="urlactions">
#(heuristic)#::

49
htroot/yacysearchitem.java
View File

@ -194,8 +194,7 @@ public class yacysearchitem {
boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");
String resultFileName = resultURL.getFileName();
prop.putHTML("content_target", target);
//if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10, null, ClientIdentification.yacyIntranetCrawlerAgent);
prop.putHTML("content_faviconCode", URLLicense.aquireLicense(faviconURL)); // acquire license for favicon url loading
prop.putHTML("content_faviconUrl", processFaviconURL(authenticated, faviconURL));
prop.put("content_urlhash", urlhash);
prop.put("content_ranking", Float.toString(result.score()));
Date[] events = result.events();
@ -310,7 +309,7 @@ public class yacysearchitem {
if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
// image search; shows thumbnails
processImage(sb, prop, item, theSearch, target_special_pattern, timeout);
processImage(sb, prop, item, theSearch, target_special_pattern, timeout, authenticated);
theSearch.query.transmitcount = item + 1;
return prop;
}
@ -342,6 +341,28 @@ public class yacysearchitem {
return prop;
}
/**
* @param authenticated
* true when current user is authenticated
* @param faviconURL
* url icon of web site
* @return url to propose in search result or empty string when faviconURL
* is null
*/
private static String processFaviconURL(final boolean authenticated, DigestURL faviconURL) {
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
StringBuilder contentFaviconURL = new StringBuilder();
if (faviconURL != null) {
contentFaviconURL.append("ViewImage.png?width=16&height=16&isStatic=true");
if (authenticated) {
contentFaviconURL.append("&url=").append(faviconURL.toNormalform(true));
} else {
contentFaviconURL.append("&code=").append(URLLicense.aquireLicense(faviconURL));
}
}
return contentFaviconURL.toString();
}
/**
@ -352,9 +373,10 @@ public class yacysearchitem {
* @param theSearch search event
* @param target_special_pattern
* @param timeout result getting timeOut
* @param authenticated set to true when user authentication is ok
*/
private static void processImage(final Switchboard sb, final serverObjects prop, final int item,
final SearchEvent theSearch, final String target_special_pattern, long timeout) {
final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean authenticated) {
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
try {
SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout);
@ -366,9 +388,24 @@ public class yacysearchitem {
/* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */
final String viewImageExt = !imageUrlExt.isEmpty() && ViewImage.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png";
/* Thumb URL */
prop.putHTML("content_item_hrefCache", "ViewImage." + viewImageExt + "?maxwidth=" + DEFAULT_IMG_WIDTH + "&maxheight=" + DEFAULT_IMG_HEIGHT + "&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring);
StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")
.append(DEFAULT_IMG_WIDTH).append("&maxheight=").append(DEFAULT_IMG_HEIGHT)
.append("&isStatic=true&quadratic");
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
if(authenticated) {
thumbURLBuilder.append("&url=").append(imageUrlstring);
} else {
thumbURLBuilder.append("&code=").append(URLLicense.aquireLicense(image.imageUrl));
}
String thumbURL = thumbURLBuilder.toString();
prop.putHTML("content_item_hrefCache", thumbURL);
/* Full size preview URL */
prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?code="+license+"&isStatic=true&url=" + imageUrlstring);
if(authenticated) {
prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?isStatic=true&url=" + imageUrlstring);
} else {
/* Not authenticated : full preview URL must be the same as thumb URL */
prop.putHTML("content_item_hrefFullPreview", thumbURL);
}
prop.putHTML("content_item_href", imageUrlstring);
prop.putHTML("content_item_target", target);
prop.put("content_item_code", license);

21
source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
View File

@ -30,14 +30,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.data.URLLicense;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.util.NamedList;
@ -50,6 +42,13 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
/**
* write the opensearch result in YaCys special way to include as much as in opensearch is included.
@ -159,8 +158,6 @@ public class YJsonResponseWriter implements QueryResponseWriter {
String filename = url.getFileName();
solitaireTag(writer, "link", u);
solitaireTag(writer, "file", filename);
// get image license
if (Classification.isImageExtension(MultiProtocolURL.getFileExtension(filename))) URLLicense.aquireLicense(urlhash, url.toNormalform(true));
} catch (final MalformedURLException e) {}
continue;
}
@ -221,7 +218,9 @@ public class YJsonResponseWriter implements QueryResponseWriter {
if (i < responseCount - 1) {
writer.write(",\n".toCharArray());
}
} catch (final Throwable ee) {}
} catch (final Throwable ee) {
ConcurrentLog.fine("YJsonResponseWriter", "Document writing error : " + ee.getMessage());
}
}
writer.write("],\n".toCharArray());

49
source/net/yacy/data/URLLicense.java
View File

@ -28,36 +28,53 @@ package net.yacy.data;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.storage.SizeLimitedMap;
/**
* This class defines a license-generation for URLs.
* It is used in case of preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet,
* but to prevent them to use this servlet as a proxy.
*/
public class URLLicense {
// this class defines a license-generation for URLs
// it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
private static final int maxQueue = 10000;
/** Map URLs by licence keys */
private static final Map<String, String> permissions = Collections.synchronizedMap(new SizeLimitedMap<String, String>(maxQueue));
/**
* Generates and stores a unique licence key for delayed url data fetching.
* @param url URL for whose data should be fectched later
* @return licence key generated or null when url is null
*/
public static String aquireLicense(final DigestURL url) {
if (url == null) return "";
// generate license key
String license = ASCII.String(url.hash());
// store reference to url with license key
if (url == null) return null;
/* Generate license key : it must absolutely be a unique key, not related to url parameter (thus url.hash can not be used).
* If the same key is generated for each call of this method with the same url parameter,
* problem may occur concurrent non authorized users try to fetch same url content.
* Example scenario (emulated in URLLicenseConcurrentTest) :
* 1 - userA aquireLicence for url
* 2 - userB aquireLicence for same url as A
* 3 - userA releaseLicense : he can now fetch url content
* 4 - userB releaseLicense : if the same license was generated, it has been already released and url content can not be fetched! */
String license = UUID.randomUUID().toString();
permissions.put(license, url.toNormalform(true));
// return the license key
return license;
}
public static String aquireLicense(final String license, final String url) {
// store reference to url with license key
permissions.put(license, url);
// return the license key
return license;
}
/**
* Use it to retrieve source url and to ensures YaCy url containing this licence code can not be reused by non-authorized users.
* @param license unique code associated to source url
* @return source url or null licence is no more valid
* @throws NullPointerException when license is null
*/
public static String releaseLicense(final String license) {
return permissions.remove(license);
}

101
test/net/yacy/data/URLLicenseConcurrentTest.java Normal file
View File

@ -0,0 +1,101 @@
// URLLicense.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 03.07.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.data;
import java.net.MalformedURLException;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.ConcurrentLog;
/**
* Test URLLicence reliability when used by concurrent threads
*
* @author luc
*
*/
public class URLLicenseConcurrentTest {
/**
* Thread emulating a client who tries to fetch some url content.
* @author luc
*
*/
private static class ClientThread extends Thread {
private String testURL = "http://yacy.net";
private int steps = 100000;
@Override
public void run() {
System.out.println(this.getName() + " started...");
DigestURL url = null;
try {
url = new DigestURL(this.testURL);
} catch (MalformedURLException e1) {
e1.printStackTrace();
}
String normalizedURL = url.toNormalform(true);
for (int step = 0; step < this.steps; step++) {
String license = URLLicense.aquireLicense(url);
// You can eventually call here Thread.sleep()
String retrievedURL = URLLicense.releaseLicense(license);
if (!normalizedURL.equals(retrievedURL)) {
System.err.println("Licence lost! license : " + license + ", step : " + step + ", Thread : " + this.getName());
}
}
System.out.println(this.getName() + " finished!");
}
}
/**
* Runs clients concurrently : until the end, no error message should be displayed in console.
* @param args
*/
public static void main(String args[]) {
long beginTime = System.nanoTime();
try {
ClientThread[] threads = new ClientThread[10];
for (int i = 0; i < threads.length; i++) {
threads[i] = new URLLicenseConcurrentTest.ClientThread();
threads[i].setName("ClientThread" + i);
threads[i].start();
}
for (int i = 0; i < threads.length; i++) {
try {
threads[i].join();
} catch (InterruptedException e) {
}
}
} finally {
long time = System.nanoTime() - beginTime;
System.out.println("Test run in " + time / 1000000 + "ms");
ConcurrentLog.shutdown();
}
}
}