Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-07-19 08:44:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

re-design ob kelondroMap

Browse Source 
- this class is replaced by an object that can hold any type of object
- this object must be defined as a class that implements kelondroObjectsEntry
- the kelodroMap is now implemented as kelondroMapObjects

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3297 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
2007-01-29 23:51:10 +00:00
parent 214a18dea7
commit 9c05e2a820
17 changed files with 739 additions and 570 deletions

22
source/de/anomic/data/blogBoard.java

@ -67,7 +67,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
public class blogBoard {
@ -78,12 +78,12 @@ public class blogBoard {
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
private kelondroMap datbase = null;
private kelondroMapObjects datbase = null;
public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true, false));
datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true, false), 500);
}
}
@ -238,16 +238,12 @@ public class blogBoard {
return read(key, datbase);
}
private entry read(String key, kelondroMap base) {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
Map record = base.get(key);
if (record == null) return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new GregorianCalendar(GMTTimeZone).getTime(), "".getBytes());
return new entry(key, record);
} catch (IOException e) {
return null;
}
private entry read(String key, kelondroMapObjects base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
Map record = base.getMap(key);
if (record == null) return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new GregorianCalendar(GMTTimeZone).getTime(), "".getBytes());
return new entry(key, record);
}
public boolean importXML(String input) {

35
source/de/anomic/data/bookmarksDB.java

@ -78,16 +78,16 @@ import de.anomic.kelondro.kelondroCachedObject;
import de.anomic.kelondro.kelondroCachedObjectMap;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.net.URL;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
public class bookmarksDB {
kelondroMap tagsTable;
kelondroMapObjects tagsTable;
//kelondroMap bookmarksTable;
kelondroCachedObjectMap bookmarksTable;
kelondroMap datesTable;
kelondroMapObjects datesTable;
HashMap tagCache;
HashMap bookmarkCache;
@ -135,17 +135,17 @@ public class bookmarksDB {
bookmarkCache=new HashMap();
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.bookmarksTable = new kelondroCachedObjectMap(new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)));
this.bookmarksTable = new kelondroCachedObjectMap(new kelondroMapObjects(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 500));
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.tagsTable = new kelondroMapObjects(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 500);
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true, false));
this.datesTable = new kelondroMapObjects(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true, false), 500);
if (!datesExisted) rebuildDates();
}
@ -195,13 +195,11 @@ public class bookmarksDB {
public Tag loadTag(String hash){
Map map;
Tag ret=null;
try {
map = tagsTable.get(hash);
if(map!=null){
ret=new Tag(hash, map);
tagCache.put(hash, ret);
}
} catch (IOException e) {}
map = tagsTable.getMap(hash);
if(map!=null){
ret=new Tag(hash, map);
tagCache.put(hash, ret);
}
return ret;
}
@ -284,14 +282,9 @@ public class bookmarksDB {
}
public bookmarksDate getDate(String date){
Map map;
try {
map=datesTable.get(date);
if(map==null) return new bookmarksDate(date);
return new bookmarksDate(date, map);
} catch (IOException e) {
return null;
}
map=datesTable.getMap(date);
if(map==null) return new bookmarksDate(date);
return new bookmarksDate(date, map);
}
public boolean renameTag(String oldName, String newName){
String tagHash=tagHash(oldName);

44
source/de/anomic/data/messageBoard.java

@ -53,7 +53,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
public class messageBoard {
@ -64,13 +64,13 @@ public class messageBoard {
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
private kelondroMap database = null;
private kelondroMapObjects database = null;
private int sn = 0;
public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false));
database = new kelondroMapObjects(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false), 500);
}
sn = 0;
}
@ -216,39 +216,25 @@ public class messageBoard {
}
public String write(entry message) {
// writes a message and returns key
try {
database.set(message.key, message.record);
return message.key;
} catch (IOException e) {
return null;
}
// writes a message and returns key
try {
database.set(message.key, message.record);
return message.key;
} catch (IOException e) {
return null;
}
}
public entry read(String key) {
try {
Map record = database.get(key);
Map record = database.getMap(key);
return new entry(key, record);
} catch (IOException e) {
return null;
}
}
/*
public boolean has(String key) {
try {
return database.has(key);
} catch (IOException e) {
return false;
}
}
*/
public void remove(String key) {
try {
database.remove(key);
} catch (IOException e) {
}
try {
database.remove(key);
} catch (IOException e) {
}
}
public Iterator keys(String category, boolean up) throws IOException {

18
source/de/anomic/data/userDB.java

@ -58,7 +58,7 @@ import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.server.serverCodings;
public final class userDB {
@ -66,7 +66,7 @@ public final class userDB {
public static final int USERNAME_MAX_LENGTH = 128;
public static final int USERNAME_MIN_LENGTH = 4;
kelondroMap userTable;
kelondroMapObjects userTable;
private final File userTableFile;
private final int bufferkb;
private long preloadTime;
@ -78,7 +78,7 @@ public final class userDB {
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true, false));
this.userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true, false), 10);
}
public int dbCacheNodeChunkSize() {
@ -96,7 +96,7 @@ public final class userDB {
} catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false));
userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 10);
}
public void close() {
@ -119,13 +119,9 @@ public final class userDB {
if(userName.length()>128){
userName=userName.substring(0, 127);
}
try {
Map record = userTable.get(userName);
if (record == null) return null;
return new Entry(userName, record);
} catch (IOException e) {
return null;
}
Map record = userTable.getMap(userName);
if (record == null) return null;
return new Entry(userName, record);
}
public Entry createEntry(String userName, HashMap userProps) throws IllegalArgumentException{

18
source/de/anomic/data/wikiBoard.java

@ -53,7 +53,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroRecords;
public class wikiBoard {
@ -65,18 +65,18 @@ public class wikiBoard {
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
private kelondroMap datbase = null;
private kelondroMap bkpbase = null;
private kelondroMapObjects datbase = null;
private kelondroMapObjects bkpbase = null;
private static HashMap authors = new HashMap();
public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true, false));
datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false));
bkpbase = new kelondroMapObjects(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false), 500);
}
}
@ -305,11 +305,11 @@ public class wikiBoard {
return read(key, datbase);
}
private entry read(String key, kelondroMap base) {
private entry read(String key, kelondroMapObjects base) {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
Map record = base.get(key);
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
Map record = base.getMap(key);
if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes());
return new entry(key, record);
} catch (IOException e) {

6
source/de/anomic/kelondro/kelondroCachedObjectMap.java

@ -26,13 +26,11 @@ package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
public class kelondroCachedObjectMap {
private kelondroMap db;
private kelondroMapObjects db;
private HashMap cache;
public kelondroCachedObjectMap(kelondroMap db){
public kelondroCachedObjectMap(kelondroMapObjects db){
this.db=db;
cache=new HashMap();
}

689
source/de/anomic/kelondro/kelondroMap.java → source/de/anomic/kelondro/kelondroMapObjects.java

@ -1,398 +1,291 @@
// kelondroMap.java
// -----------------------
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroMap {
private static final int cachesize = 500;
private kelondroDyn dyn;
private kelondroMScoreCluster cacheScore;
private HashMap cache;
private long startup;
private String[] sortfields, accfields;
private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation
private HashMap accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMap(kelondroDyn dyn) {
this(dyn, null, null);
}
public kelondroMap(kelondroDyn dyn, String[] sortfields, String[] accfields) {
this.dyn = dyn;
this.cache = new HashMap();
this.cacheScore = new kelondroMScoreCluster();
this.startup = System.currentTimeMillis();
this.elementCount = 0;
// create fast ordering clusters and acc fields
this.sortfields = sortfields;
this.accfields = accfields;
kelondroMScoreCluster[] cluster = null;
if (sortfields == null) sortClusterMap = null; else {
sortClusterMap = new HashMap();
cluster = new kelondroMScoreCluster[sortfields.length];
for (int i = 0; i < sortfields.length; i++) {
cluster[i] = new kelondroMScoreCluster();
}
}
Long[] accumulator = null;
if (accfields == null) accMap = null; else {
accMap = new HashMap();
accumulator = new Long[accfields.length];
for (int i = 0; i < accfields.length; i++) {
accumulator[i] = new Long(0);
}
}
// fill cluster and accumulator with values
if ((sortfields != null) || (accfields != null)) try {
kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false);
String key, value;
long valuel;
Map map;
while (it.hasNext()) {
key = (String) it.next();
map = get(key);
if (map == null) break;
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value));
}
if (accfields != null) for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (value != null) try {
valuel = Long.parseLong(value);
accumulator[i] = new Long(accumulator[i].longValue() + valuel);
} catch (NumberFormatException e) {}
}
elementCount++;
}
} catch (IOException e) {}
// fill cluster
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
// fill acc map
if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]);
}
public int keySize() {
return dyn.row().width(0);
}
public int cacheNodeChunkSize() {
return dyn.cacheNodeChunkSize();
}
public int cacheObjectChunkSize() {
return dyn.cacheObjectChunkSize();
}
public int[] cacheNodeStatus() {
return dyn.cacheNodeStatus();
}
public long[] cacheObjectStatus() {
return dyn.cacheObjectStatus();
}
public synchronized void set(String key, Map newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
// update elementCount
if ((sortfields != null) || (accfields != null)) {
final Map oldMap = get(key, false);
if (oldMap == null) {
// new element
elementCount++;
} else {
// element exists, update acc
if (accfields != null) updateAcc(oldMap, false);
}
}
// write entry
writeKra(key, newMap, "");
// check for space in cache
checkCacheSpace();
// write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, newMap);
// update sortCluster
if (sortClusterMap != null) updateSortCluster(key, newMap);
// update accumulators with new values (add)
if (accfields != null) updateAcc(newMap, true);
}
private synchronized void writeKra(final String key, final Map newMap, String comment) throws IOException {
// write map to kra
final kelondroRA kra = dyn.getRA(key);
kra.writeMap(newMap, comment);
kra.close();
}
private void updateAcc(Map map, boolean add) {
String value;
long valuel;
Long accumulator;
for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (value != null) {
try {
valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]);
if (add) {
accMap.put(accfields[i], new Long(accumulator.longValue() + valuel));
} else {
accMap.put(accfields[i], new Long(accumulator.longValue() - valuel));
}
} catch (NumberFormatException e) {}
}
}
}
private void updateSortCluster(final String key, final Map map) {
String value;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) {
cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
cluster.setScore(key, kelondroMScoreCluster.string2score(value));
sortClusterMap.put(sortfields[i], cluster);
}
}
}
public synchronized void remove(String key) throws IOException {
// update elementCount
if (key == null) return;
if ((sortfields != null) || (accfields != null)) {
final Map map = get(key);
if (map != null) {
// update count
elementCount--;
// update accumulators (subtract)
if (accfields != null) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(key);
}
}
// remove from cache
cacheScore.deleteScore(key);
cache.remove(key);
// remove from file
dyn.remove(key);
}
private void deleteSortCluster(final String key) {
if (key == null) return;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
cluster.deleteScore(key);
sortClusterMap.put(sortfields[i], cluster);
}
}
public synchronized Map get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
}
private synchronized Map get(final String key, final boolean storeCache) throws IOException {
// load map from cache
Map map = (Map) cache.get(key);
if (map != null) return map;
// load map from kra
if (!(dyn.existsDyn(key))) return null;
//final kelondroRA kra = new kelondroBufferedRA(dyn.getRA(key), dyn.cacheObjectChunkSize(), 0);
final kelondroRA kra = dyn.getRA(key);
map = kra.readMap();
kra.close();
if (storeCache) {
// cache it also
checkCacheSpace();
// write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, map);
}
// return value
return map;
}
private synchronized void checkCacheSpace() {
// check for space in cache
if (cache.size() >= cachesize) {
// delete one entry
final String delkey = (String) cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
}
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating);
}
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating, firstKey);
}
public synchronized Iterator keys(final boolean up, /* sorted by */ String field) {
// sorted iteration using the sortClusters
if (sortClusterMap == null) return null;
final kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field);
if (cluster == null) return null; // sort field does not exist
//System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString());
return cluster.scores(up);
}
public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException {
return new mapIterator(keys(up, rotating));
}
public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
return new mapIterator(keys(up, rotating, firstKey));
}
public synchronized mapIterator maps(final boolean up, final String field) {
return new mapIterator(keys(up, field));
}
public synchronized long getAcc(final String field) {
final Long accumulator = (Long) accMap.get(field);
if (accumulator == null) return -1;
return accumulator.longValue();
}
public synchronized int size() {
if ((sortfields != null) || (accfields != null)) return elementCount;
try {
return dyn.sizeDyn();
} catch (IOException e) {
return 0;
}
}
public void close() throws IOException {
// finish queue
//writeWorker.terminate(true);
// close cluster
if (sortClusterMap != null) {
for (int i = 0; i < sortfields.length; i++) sortClusterMap.remove(sortfields[i]);
sortClusterMap = null;
}
cache = null;
cacheScore = null;
// close file
dyn.close();
}
public class mapIterator implements Iterator {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
Iterator keyIterator;
boolean finish;
public mapIterator(Iterator keyIterator) {
this.keyIterator = keyIterator;
this.finish = false;
}
public boolean hasNext() {
return (!(finish)) && (keyIterator.hasNext());
}
public Object next() {
final String nextKey = (String) keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
try {
final Map map = get(nextKey);
//assert (map != null) : "nextKey = " + nextKey;
if (map == null) throw new kelondroException("no more elements available");
map.put("key", nextKey);
return map;
} catch (IOException e) {
finish = true;
return null;
}
}
public void remove() {
throw new UnsupportedOperationException();
}
} // class mapIterator
} // class kelondroMap
// kelondroMapObjects.java
// -----------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as part of kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroMapObjects extends kelondroObjects {
private String[] sortfields, accfields;
private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation
private HashMap accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMapObjects(kelondroDyn dyn, int cachesize) {
this(dyn, cachesize, null, null);
}
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] accfields) {
super(dyn, cachesize);
// create fast ordering clusters and acc fields
this.sortfields = sortfields;
this.accfields = accfields;
kelondroMScoreCluster[] cluster = null;
if (sortfields == null) sortClusterMap = null; else {
sortClusterMap = new HashMap();
cluster = new kelondroMScoreCluster[sortfields.length];
for (int i = 0; i < sortfields.length; i++) {
cluster[i] = new kelondroMScoreCluster();
}
}
Long[] accumulator = null;
if (accfields == null) accMap = null; else {
accMap = new HashMap();
accumulator = new Long[accfields.length];
for (int i = 0; i < accfields.length; i++) {
accumulator[i] = new Long(0);
}
}
// fill cluster and accumulator with values
if ((sortfields != null) || (accfields != null)) try {
kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false);
String key, value;
long valuel;
Map map;
while (it.hasNext()) {
key = (String) it.next();
map = getMap(key);
if (map == null) break;
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value));
}
if (accfields != null) for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (value != null) try {
valuel = Long.parseLong(value);
accumulator[i] = new Long(accumulator[i].longValue() + valuel);
} catch (NumberFormatException e) {}
}
elementCount++;
}
} catch (IOException e) {}
// fill cluster
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
// fill acc map
if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]);
}
public synchronized void set(String key, Map newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
// update elementCount
if ((sortfields != null) || (accfields != null)) {
final Map oldMap = getMap(key, false);
if (oldMap == null) {
// new element
elementCount++;
} else {
// element exists, update acc
if (accfields != null) updateAcc(oldMap, false);
}
}
super.set(key, new kelondroObjectsMapEntry(newMap));
// update sortCluster
if (sortClusterMap != null) updateSortCluster(key, newMap);
// update accumulators with new values (add)
if (accfields != null) updateAcc(newMap, true);
}
private void updateAcc(Map map, boolean add) {
String value;
long valuel;
Long accumulator;
for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (value != null) {
try {
valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]);
if (add) {
accMap.put(accfields[i], new Long(accumulator.longValue() + valuel));
} else {
accMap.put(accfields[i], new Long(accumulator.longValue() - valuel));
}
} catch (NumberFormatException e) {}
}
}
}
private void updateSortCluster(final String key, final Map map) {
String value;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) {
cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
cluster.setScore(key, kelondroMScoreCluster.string2score(value));
sortClusterMap.put(sortfields[i], cluster);
}
}
}
public synchronized void remove(String key) throws IOException {
if (key == null) return;
// update elementCount
if ((sortfields != null) || (accfields != null)) {
final Map map = getMap(key);
if (map != null) {
// update count
elementCount--;
// update accumulators (subtract)
if (accfields != null) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(key);
}
}
super.remove(key);
}
public Map getMap(String key) {
try {
kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key);
if (mapEntry == null) return null;
return mapEntry.map();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
protected Map getMap(String key, boolean cache) {
try {
kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key, cache);
if (mapEntry == null) return null;
return mapEntry.map();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
private void deleteSortCluster(final String key) {
if (key == null) return;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
cluster.deleteScore(key);
sortClusterMap.put(sortfields[i], cluster);
}
}
public synchronized Iterator keys(final boolean up, /* sorted by */ String field) {
// sorted iteration using the sortClusters
if (sortClusterMap == null) return null;
final kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field);
if (cluster == null) return null; // sort field does not exist
//System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString());
return cluster.scores(up);
}
public synchronized mapIterator maps(final boolean up, final String field) {
return new mapIterator(keys(up, field));
}
public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException {
return new mapIterator(keys(up, rotating));
}
public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
return new mapIterator(keys(up, rotating, firstKey));
}
public synchronized long getAcc(final String field) {
final Long accumulator = (Long) accMap.get(field);
if (accumulator == null) return -1;
return accumulator.longValue();
}
public synchronized int size() {
if ((sortfields != null) || (accfields != null)) return elementCount;
return super.size();
}
public void close() throws IOException {
// close cluster
if (sortClusterMap != null) {
for (int i = 0; i < sortfields.length; i++) sortClusterMap.remove(sortfields[i]);
sortClusterMap = null;
}
super.close();
}
public class mapIterator implements Iterator {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
Iterator keyIterator;
boolean finish;
public mapIterator(Iterator keyIterator) {
this.keyIterator = keyIterator;
this.finish = false;
}
public boolean hasNext() {
return (!(finish)) && (keyIterator != null) && (keyIterator.hasNext());
}
public Object next() {
final String nextKey = (String) keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
final Map map = getMap(nextKey);
//assert (map != null) : "nextKey = " + nextKey;
if (map == null) throw new kelondroException("no more elements available");
map.put("key", nextKey);
return map;
}
public void remove() {
throw new UnsupportedOperationException();
}
} // class mapIterator
}

38
source/de/anomic/kelondro/kelondroMapTable.java

@ -62,19 +62,19 @@ public class kelondroMapTable {
}
public void declareMaps(
String tablename, int keysize, int nodesize,
String tablename, int keysize, int nodesize, int cacheslots,
char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, null, null, fillChar);
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, fillChar);
}
public void declareMaps(
String tablename, int keysize, int nodesize,
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0);
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, accfields, fillChar, 0x800, 0);
}
public void declareMaps(
String tablename, int keysize, int nodesize,
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar,
long buffersize /*bytes*/, long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
@ -83,7 +83,7 @@ public class kelondroMapTable {
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false);
kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, accfields);
mTables.put(tablename, map);
}
@ -96,7 +96,7 @@ public class kelondroMapTable {
}
public synchronized void update(String tablename, String key, Map map) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename);
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.update: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
table.set(key, map);
@ -111,10 +111,10 @@ public class kelondroMapTable {
}
public synchronized Map selectMap(String tablename, String key) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename);
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
return table.get(key);
return table.getMap(key);
}
public synchronized kelondroRow.Entry selectByte(String tablename, String key) throws IOException {
@ -123,20 +123,20 @@ public class kelondroMapTable {
return tree.get(key.getBytes());
}
public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename);
public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating) throws IOException {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist.");
return table.maps(up, rotating);
}
public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating, byte[] firstKey) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename);
public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating, byte[] firstKey) throws IOException {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist.");
return table.maps(up, rotating, firstKey);
}
public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, String field) {
kelondroMap table = (kelondroMap) mTables.get(tablename);
public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, String field) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist.");
return table.maps(up, field);
}
@ -150,7 +150,7 @@ public class kelondroMapTable {
// if you need the long-values from a row-iteration, please use kelondroRecords.bytes2long to convert from byte[] to long
public synchronized void delete(String tablename, String key) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename);
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
if (table != null) {table.remove(key); mTables.put(tablename, table); return;}
@ -161,13 +161,13 @@ public class kelondroMapTable {
}
public synchronized long accumulator(String tablename, String field) {
kelondroMap table = (kelondroMap) mTables.get(tablename);
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist.");
return table.getAcc(field);
}
public synchronized int size(String tablename) {
kelondroMap table = (kelondroMap) mTables.get(tablename);
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table != null) return table.size();
kelondroIndex Tree = (kelondroIndex) tTables.get(tablename);
@ -178,7 +178,7 @@ public class kelondroMapTable {
public void close() throws IOException {
Iterator tablesIt = mTables.values().iterator();
while (tablesIt.hasNext()) ((kelondroMap) tablesIt.next()).close();
while (tablesIt.hasNext()) ((kelondroMapObjects) tablesIt.next()).close();
mTables = null;
Iterator TreeIt = tTables.values().iterator();

215
source/de/anomic/kelondro/kelondroObjects.java Normal file

@ -0,0 +1,215 @@
// kelondroObjects.java
// -----------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
public class kelondroObjects {
private kelondroDyn dyn;
private kelondroMScoreCluster cacheScore;
private HashMap cache;
private long startup;
private int cachesize;
public kelondroObjects(kelondroDyn dyn, int cachesize) {
this.dyn = dyn;
this.cache = new HashMap();
this.cacheScore = new kelondroMScoreCluster();
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
}
public int keySize() {
return dyn.row().width(0);
}
public int cacheNodeChunkSize() {
return dyn.cacheNodeChunkSize();
}
public int cacheObjectChunkSize() {
return dyn.cacheObjectChunkSize();
}
public int[] cacheNodeStatus() {
return dyn.cacheNodeStatus();
}
public long[] cacheObjectStatus() {
return dyn.cacheObjectStatus();
}
public synchronized void set(String key, kelondroObjectsEntry newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
// write entry
kelondroRA kra = dyn.getRA(key);
newMap.write(kra);
kra.close();
// check for space in cache
checkCacheSpace();
// write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, newMap);
}
public synchronized void remove(String key) throws IOException {
// update elementCount
if (key == null) return;
// remove from cache
cacheScore.deleteScore(key);
cache.remove(key);
// remove from file
dyn.remove(key);
}
public synchronized kelondroObjectsEntry get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
}
protected synchronized kelondroObjectsEntry get(final String key, final boolean storeCache) throws IOException {
// load map from cache
kelondroObjectsEntry map = (kelondroObjectsEntry) cache.get(key);
if (map != null) return map;
// load map from kra
if (!(dyn.existsDyn(key))) return null;
// read object
kelondroRA kra = dyn.getRA(key);
map = new kelondroObjectsMapEntry(kra);
kra.close();
if (storeCache) {
// cache it also
checkCacheSpace();
// write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, map);
}
// return value
return map;
}
private synchronized void checkCacheSpace() {
// check for space in cache
if (cache.size() >= cachesize) {
// delete one entry
final String delkey = (String) cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
}
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating);
}
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating, firstKey);
}
public synchronized objectIterator entries(final boolean up, final boolean rotating) throws IOException {
return new objectIterator(keys(up, rotating));
}
public synchronized objectIterator entries(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
return new objectIterator(keys(up, rotating, firstKey));
}
public synchronized int size() {
try {
return dyn.sizeDyn();
} catch (IOException e) {
return 0;
}
}
public void close() throws IOException {
// finish queue
//writeWorker.terminate(true);
cache = null;
cacheScore = null;
// close file
dyn.close();
}
public class objectIterator implements Iterator {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
Iterator keyIterator;
boolean finish;
public objectIterator(Iterator keyIterator) {
this.keyIterator = keyIterator;
this.finish = false;
}
public boolean hasNext() {
return (!(finish)) && (keyIterator.hasNext());
}
public Object next() {
final String nextKey = (String) keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
try {
final kelondroObjectsEntry obj = get(nextKey);
if (obj == null) throw new kelondroException("no more elements available");
return obj;
} catch (IOException e) {
finish = true;
return null;
}
}
public void remove() {
throw new UnsupportedOperationException();
}
} // class mapIterator
}

35
source/de/anomic/kelondro/kelondroObjectsEntry.java Normal file

@ -0,0 +1,35 @@
// kelondroObjectsEntry.java
// -------------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as part of kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
public interface kelondroObjectsEntry {
public void write(kelondroRA ra);
public void read(kelondroRA ra);
}

70
source/de/anomic/kelondro/kelondroObjectsMapEntry.java Normal file

@ -0,0 +1,70 @@
// kelondroObjectsMapEntry.java
// ----------------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as part of kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class kelondroObjectsMapEntry implements kelondroObjectsEntry {
private Map entry;
public kelondroObjectsMapEntry() {
this.entry = new HashMap();
}
public kelondroObjectsMapEntry(Map map) {
this.entry = map;
}
public kelondroObjectsMapEntry(kelondroRA ra) {
this.read(ra);
}
public void read(kelondroRA ra) {
try {
this.entry = ra.readMap();
} catch (IOException e) {
e.printStackTrace();
}
}
public void write(kelondroRA ra) {
try {
ra.writeMap(this.entry, "");
} catch (IOException e) {
e.printStackTrace();
}
}
public Map map() {
return this.entry;
}
}

18
source/de/anomic/plasma/plasmaCrawlProfile.java

@ -51,12 +51,12 @@ import java.util.Map;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.server.serverCodings;
public class plasmaCrawlProfile {
private kelondroMap profileTable;
private kelondroMapObjects profileTable;
private HashMap domsCache;
private File profileTableFile;
private int bufferkb;
@ -70,7 +70,7 @@ public class plasmaCrawlProfile {
this.preloadTime = preloadTime;
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
profileTable = new kelondroMap(dyn);
profileTable = new kelondroMapObjects(dyn, 500);
domsCache = new HashMap();
}
@ -96,7 +96,7 @@ public class plasmaCrawlProfile {
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
profileTable = new kelondroMap(dyn);
profileTable = new kelondroMapObjects(dyn, 500);
}
public void close() {
@ -222,13 +222,9 @@ public class plasmaCrawlProfile {
}
public entry getEntry(String handle) {
try {
Map m = profileTable.get(handle);
if (m == null) return null;
return new entry(m);
} catch (IOException e) {
return null;
}
Map m = profileTable.getMap(handle);
if (m == null) return null;
return new entry(m);
}
public class DomProfile {

16
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java

@ -57,13 +57,13 @@ import java.util.Map;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
public class plasmaCrawlRobotsTxt {
public static final String ROBOTS_DB_PATH_SEPARATOR = ";";
kelondroMap robotsTable;
kelondroMapObjects robotsTable;
private final File robotsTableFile;
private int bufferkb;
private long preloadTime;
@ -73,7 +73,7 @@ public class plasmaCrawlRobotsTxt {
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true, false));
robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true, false), 100);
}
public int cacheNodeChunkSize() {
@ -99,7 +99,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false));
robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 100);
}
public void close() {
@ -124,14 +124,12 @@ public class plasmaCrawlRobotsTxt {
public Entry getEntry(String hostName) {
try {
Map record = robotsTable.get(hostName);
Map record = robotsTable.getMap(hostName);
if (record == null) return null;
return new Entry(hostName, record);
} catch (IOException e) {
return null;
} catch (kelondroException e) {
resetDatabase();
return null;
resetDatabase();
return null;
}
}

14
source/de/anomic/plasma/plasmaHTCache.java

@ -78,7 +78,7 @@ import de.anomic.plasma.plasmaURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.net.URL;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.ResourceInfoFactory;
@ -97,7 +97,7 @@ public final class plasmaHTCache {
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day
kelondroMap responseHeaderDB = null;
kelondroMapObjects responseHeaderDB = null;
private final LinkedList cacheStack;
private final Map cacheAge; // a <date+hash, cache-path> - relation
public long curCacheSize;
@ -174,7 +174,7 @@ public final class plasmaHTCache {
// open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db");
try {
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false));
this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false), 500);
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);
@ -512,7 +512,7 @@ public final class plasmaHTCache {
String urlHash = plasmaURL.urlHash(url.toNormalform());
// loading data from database
Map hdb = this.responseHeaderDB.get(urlHash);
Map hdb = this.responseHeaderDB.getMap(urlHash);
if (hdb == null) return null;
// generate the cached object
@ -752,11 +752,7 @@ public final class plasmaHTCache {
if (url != null) return url;
// try responseHeaderDB
Map hdb;
try {
hdb = this.responseHeaderDB.get(urlHash);
} catch (IOException e) {
hdb = null;
}
hdb = this.responseHeaderDB.getMap(urlHash);
if (hdb != null) {
Object origRequestLine = hdb.get(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE);
if ((origRequestLine != null)&&(origRequestLine instanceof String)) {

1
source/de/anomic/yacy/yacyDHTAction.java

@ -145,6 +145,7 @@ public class yacyDHTAction implements yacyPeerAction {
if (s.getFlagAcceptRemoteIndex()) return s;
}
} catch (kelondroException e) {
System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage());
yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
seedDB.resetActiveTable();
return null;

64
source/de/anomic/yacy/yacySeedDB.java

@ -64,7 +64,7 @@ import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroRecords;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaSwitchboard;
@ -93,7 +93,7 @@ public final class yacySeedDB {
// class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
protected kelondroMap seedActiveDB, seedPassiveDB, seedPotentialDB;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
private int seedDBBufferKB;
private long preloadTime;
@ -203,18 +203,18 @@ public final class yacySeedDB {
seedPotentialDB.cacheObjectStatus() }, 3);
}
private synchronized kelondroMap openSeedTable(File seedDBFile) {
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs();
try {
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), sortFields, accFields);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields);
} catch (Exception e) {
seedDBFile.delete();
// try again
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), sortFields, accFields);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields);
}
}
protected synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {
protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) {
// this is an emergency function that should only be used if any problem with the
// seed.db is detected
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
@ -357,14 +357,14 @@ public final class yacySeedDB {
seedPassiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
} catch (IOException e){
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
} catch (kelondroException e){
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
}
}
@ -382,14 +382,14 @@ public final class yacySeedDB {
seedPassiveDB.set(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
} catch (kelondroException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
}
}
@ -408,14 +408,14 @@ public final class yacySeedDB {
seedPotentialDB.set(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
} catch (kelondroException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
}
}
@ -443,16 +443,12 @@ public final class yacySeedDB {
}
}
private yacySeed get(String hash, kelondroMap database) {
private yacySeed get(String hash, kelondroMapObjects database) {
if (hash == null) return null;
if ((mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
try {
Map entry = database.get(hash);
Map entry = database.getMap(hash);
if (entry == null) return null;
return new yacySeed(hash, entry);
} catch (IOException e) {
return null;
}
}
public yacySeed getConnected(String hash) {
@ -844,12 +840,12 @@ public final class yacySeedDB {
class seedEnum implements Enumeration {
kelondroMap.mapIterator it;
kelondroMapObjects.mapIterator it;
yacySeed nextSeed;
kelondroMap database;
kelondroMapObjects database;
float minVersion;
public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMap database, float minVersion) {
public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMapObjects database, float minVersion) {
this.database = database;
this.minVersion = minVersion;
try {
@ -874,7 +870,7 @@ public final class yacySeedDB {
}
}
public seedEnum(boolean up, String field, kelondroMap database) {
public seedEnum(boolean up, String field, kelondroMapObjects database) {
this.database = database;
try {
it = database.maps(up, field);

6
source/yacy.java

@ -79,7 +79,7 @@ import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroTree;
import de.anomic.net.URL;
@ -1324,9 +1324,9 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it;
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);
while (it.hasNext()) {
Map dna = (Map) it.next();