mirror of
https://github.com/yacy/yacy_search_server.git
synced 2026-10-01 22:11:59 -04:00
Michael Peter Christen
d05db9e6bb
dying-peers-when-growing problem: HeapReader can now open immutable indexes as SSTableHandleMap instances. Keys and offsets remain file-backed instead of occupying the Java heap. Lazy working copies avoid additional files until a mutation occurs. Negative offsets record deletions without tombstone sidecars or RAM maps. Mutable heaps retain RowHandleMap, while sealed segments save heap memory. With immutable indexes the memory consumption is not stopped, it is only moved from heap memory to system memory. The result should be that there is no need to assign more memory to YaCy when it gets bigger, it simply takes it itself from the system memory. This should also fix https://github.com/yacy/yacy_search_server/issues/731
434 lines
16 KiB
Java
434 lines
16 KiB
Java
// ReferenceContainerArray.java
|
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
// first published 04.01.2009 on http://yacy.net
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// LICENSE
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package net.yacy.kelondro.rwi;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.util.Date;
|
|
import java.util.Iterator;
|
|
|
|
import net.yacy.cora.order.ByteOrder;
|
|
import net.yacy.cora.order.CloneableIterator;
|
|
import net.yacy.cora.sorting.Rating;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.cora.util.SpaceExceededException;
|
|
import net.yacy.kelondro.blob.ArrayStack;
|
|
import net.yacy.kelondro.blob.ImmutableBLOB;
|
|
import net.yacy.kelondro.data.word.Word;
|
|
import net.yacy.kelondro.index.Row;
|
|
import net.yacy.kelondro.index.RowSet;
|
|
|
|
|
|
public final class ReferenceContainerArray<ReferenceType extends Reference> {
|
|
|
|
private final static long METHOD_MAXRUNTIME = 5000L;
|
|
|
|
private final ReferenceFactory<ReferenceType> factory;
|
|
private final ArrayStack array;
|
|
|
|
/**
|
|
* open a index container array based on BLOB dumps. The content of the BLOBs will not be read
|
|
* unless a .idx file exists. Only the .idx file is opened to get a fast read access to
|
|
* the BLOB. This class provides no write methods, because BLOB files should not be
|
|
* written in random access. To support deletion, a write access to the BLOB for deletion
|
|
* is still possible
|
|
* @param payloadrow the row definition for the BLOB data structure
|
|
* @param log
|
|
* @throws IOException
|
|
*/
|
|
public ReferenceContainerArray(
|
|
final File heapLocation,
|
|
final String prefix,
|
|
final ReferenceFactory<ReferenceType> factory,
|
|
final ByteOrder termOrder,
|
|
final int termSize) throws IOException {
|
|
this.factory = factory;
|
|
this.array = new ArrayStack(
|
|
heapLocation,
|
|
prefix,
|
|
termOrder,
|
|
termSize,
|
|
0,
|
|
true,
|
|
true);
|
|
}
|
|
|
|
public synchronized void close() {
|
|
this.array.close(true);
|
|
}
|
|
|
|
public void clear() throws IOException {
|
|
this.array.clear();
|
|
}
|
|
|
|
public int[] sizes() {
|
|
return (this.array == null) ? new int[0] : this.array.sizes();
|
|
}
|
|
|
|
public ByteOrder ordering() {
|
|
return this.array.ordering();
|
|
}
|
|
|
|
public File newContainerBLOBFile() {
|
|
return this.array.newBLOB(new Date());
|
|
}
|
|
|
|
public void mountBLOBFile(final File location) throws IOException {
|
|
this.array.mountBLOB(location, false);
|
|
}
|
|
|
|
public Row rowdef() {
|
|
return this.factory.getRow();
|
|
}
|
|
|
|
/**
|
|
* return an iterator object that creates top-level-clones of the indexContainers
|
|
* in the cache, so that manipulations of the iterated objects do not change
|
|
* objects in the cache.
|
|
* @throws IOException
|
|
*/
|
|
public CloneableIterator<ReferenceContainer<ReferenceType>> referenceContainerIterator(final byte[] startWordHash, final boolean rot, final boolean excludePrivate) {
|
|
try {
|
|
return new ReferenceContainerIterator(startWordHash, rot, excludePrivate);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public class ReferenceContainerIterator implements CloneableIterator<ReferenceContainer<ReferenceType>>, Iterable<ReferenceContainer<ReferenceType>> {
|
|
|
|
// this class exists, because the wCache cannot be iterated with rotation
|
|
// and because every indexContainer Object that is iterated must be returned as top-level-clone
|
|
// so this class simulates wCache.tailMap(startWordHash).values().iterator()
|
|
// plus the mentioned features
|
|
|
|
private final boolean rot, excludePrivate;
|
|
protected CloneableIterator<byte[]> iterator;
|
|
|
|
public ReferenceContainerIterator(final byte[] startWordHash, final boolean rot, final boolean excludePrivate) throws IOException {
|
|
this.rot = rot;
|
|
this.excludePrivate = excludePrivate;
|
|
this.iterator = ReferenceContainerArray.this.array.keys(true, startWordHash);
|
|
// The collection's iterator will return the values in the order that their corresponding keys appear in the tree.
|
|
}
|
|
|
|
@Override
|
|
public ReferenceContainerIterator clone(final Object secondWordHash) {
|
|
try {
|
|
return new ReferenceContainerIterator((byte[]) secondWordHash, this.rot, this.excludePrivate);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public boolean hasNext() {
|
|
if (this.iterator == null) return false;
|
|
if (this.rot) return true;
|
|
return this.iterator.hasNext();
|
|
}
|
|
|
|
@Override
|
|
public ReferenceContainer<ReferenceType> next() {
|
|
while (this.iterator.hasNext()) try {
|
|
byte[] b = this.iterator.next();
|
|
if (this.excludePrivate && Word.isPrivate(b)) continue;
|
|
return get(b);
|
|
} catch (final Throwable e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
// rotation iteration
|
|
if (!this.rot) {
|
|
return null;
|
|
}
|
|
try {
|
|
this.iterator = ReferenceContainerArray.this.array.keys(true, null);
|
|
while (this.iterator.hasNext()) {
|
|
byte[] b = this.iterator.next();
|
|
if (this.excludePrivate && Word.isPrivate(b)) continue;
|
|
return get(b);
|
|
}
|
|
return null;
|
|
} catch (final Throwable e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void remove() {
|
|
this.iterator.remove();
|
|
}
|
|
|
|
@Override
|
|
public Iterator<ReferenceContainer<ReferenceType>> iterator() {
|
|
return this;
|
|
}
|
|
|
|
@Override
|
|
public void close() {
|
|
this.iterator.close();
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
* return an iterator object that counts the number of references in indexContainers
|
|
* the startWordHash may be null to iterate all from the beginning
|
|
* @throws IOException
|
|
*/
|
|
public CloneableIterator<Rating<byte[]>> referenceCountIterator(final byte[] startWordHash, final boolean rot, final boolean excludePrivate) {
|
|
try {
|
|
return new ReferenceCountIterator(startWordHash, rot, excludePrivate);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public class ReferenceCountIterator implements CloneableIterator<Rating<byte[]>>, Iterable<Rating<byte[]>> {
|
|
|
|
private final boolean rot, excludePrivate;
|
|
private CloneableIterator<byte[]> iterator;
|
|
|
|
public ReferenceCountIterator(final byte[] startWordHash, final boolean rot, final boolean excludePrivate) throws IOException {
|
|
this.rot = rot;
|
|
this.excludePrivate = excludePrivate;
|
|
this.iterator = ReferenceContainerArray.this.array.keys(true, startWordHash);
|
|
// The collection's iterator will return the values in the order that their corresponding keys appear in the tree.
|
|
}
|
|
|
|
@Override
|
|
public ReferenceCountIterator clone(final Object secondWordHash) {
|
|
try {
|
|
return new ReferenceCountIterator((byte[]) secondWordHash, this.rot, this.excludePrivate);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public boolean hasNext() {
|
|
if (this.iterator == null) return false;
|
|
if (this.rot) return true;
|
|
return this.iterator.hasNext();
|
|
}
|
|
|
|
@Override
|
|
public Rating<byte[]> next() {
|
|
byte[] reference;
|
|
while (this.iterator.hasNext()) try {
|
|
reference = this.iterator.next();
|
|
if (this.excludePrivate && Word.isPrivate(reference)) continue;
|
|
return new Rating<byte[]>(reference, count(reference));
|
|
} catch (final Throwable e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
// rotation iteration
|
|
if (!this.rot) {
|
|
return null;
|
|
}
|
|
while (this.iterator.hasNext()) try {
|
|
this.iterator = ReferenceContainerArray.this.array.keys(true, null);
|
|
reference = this.iterator.next();
|
|
if (this.excludePrivate && Word.isPrivate(reference)) continue;
|
|
return new Rating<byte[]>(reference, count(reference));
|
|
} catch (final Throwable e) {
|
|
ConcurrentLog.logException(e);
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
public void remove() {
|
|
this.iterator.remove();
|
|
}
|
|
|
|
@Override
|
|
public Iterator<Rating<byte[]>> iterator() {
|
|
return this;
|
|
}
|
|
|
|
@Override
|
|
public void close() {
|
|
this.iterator.close();
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
* test if a given key is in the heap
|
|
* this works with heaps in write- and read-mode
|
|
* @param key
|
|
* @return true, if the key is used in the heap; false otherwise
|
|
* @throws IOException
|
|
*/
|
|
public boolean has(final byte[] termHash) {
|
|
return this.array.containsKey(termHash);
|
|
}
|
|
|
|
/**
|
|
* get a indexContainer from a heap
|
|
* @param key
|
|
* @return the indexContainer if one exist, null otherwise
|
|
* @throws IOException
|
|
* @throws SpaceExceededException
|
|
*/
|
|
public ReferenceContainer<ReferenceType> get(final byte[] termHash) throws IOException, SpaceExceededException {
|
|
final long timeout = System.currentTimeMillis() + METHOD_MAXRUNTIME;
|
|
final Iterator<byte[]> entries = this.array.getAll(termHash).iterator();
|
|
if (entries == null || !entries.hasNext()) return null;
|
|
final byte[] a = entries.next();
|
|
int k = 1;
|
|
ReferenceContainer<ReferenceType> c = new ReferenceContainer<ReferenceType>(this.factory, termHash, RowSet.importRowSet(a, this.factory.getRow()));
|
|
if (System.currentTimeMillis() > timeout) {
|
|
ConcurrentLog.warn("KELONDRO", "ReferenceContainerArray: timout in get() (1): " + k + " tables searched. timeout = " + METHOD_MAXRUNTIME);
|
|
return c;
|
|
}
|
|
while (entries.hasNext()) {
|
|
c = c.merge(new ReferenceContainer<ReferenceType>(this.factory, termHash, RowSet.importRowSet(entries.next(), this.factory.getRow())));
|
|
k++;
|
|
if (System.currentTimeMillis() > timeout) {
|
|
ConcurrentLog.warn("KELONDRO", "ReferenceContainerArray: timout in get() (2): " + k + " tables searched. timeout = " + METHOD_MAXRUNTIME);
|
|
return c;
|
|
}
|
|
}
|
|
return c;
|
|
}
|
|
|
|
public int count(final byte[] termHash) throws IOException {
|
|
final long timeout = System.currentTimeMillis() + METHOD_MAXRUNTIME;
|
|
final Iterator<Long> entries = this.array.lengthAll(termHash).iterator();
|
|
if (entries == null || !entries.hasNext()) return 0;
|
|
final Long a = entries.next();
|
|
int k = 1;
|
|
int c = RowSet.importRowCount(a, this.factory.getRow());
|
|
assert c >= 0;
|
|
if (System.currentTimeMillis() > timeout) {
|
|
ConcurrentLog.warn("KELONDRO", "ReferenceContainerArray: timeout in count() (1): " + k + " tables searched. timeout = " + METHOD_MAXRUNTIME);
|
|
return c;
|
|
}
|
|
while (entries.hasNext()) {
|
|
c += RowSet.importRowCount(entries.next(), this.factory.getRow());
|
|
assert c >= 0;
|
|
k++;
|
|
if (System.currentTimeMillis() > timeout) {
|
|
ConcurrentLog.warn("KELONDRO", "ReferenceContainerArray: timeout in count() (2): " + k + " tables searched. timeout = " + METHOD_MAXRUNTIME);
|
|
return c;
|
|
}
|
|
}
|
|
assert c >= 0;
|
|
return c;
|
|
}
|
|
|
|
/**
|
|
* delete a indexContainer from the heap cache. This can only be used for write-enabled heaps
|
|
* @param wordHash
|
|
* @return the indexContainer if the cache contained the container, null otherwise
|
|
* @throws IOException
|
|
*/
|
|
public void delete(final byte[] termHash) throws IOException {
|
|
// returns the index that had been deleted
|
|
this.array.delete(termHash);
|
|
}
|
|
|
|
public int reduce(final byte[] termHash, final ContainerReducer<ReferenceType> reducer) throws IOException, SpaceExceededException {
|
|
return this.array.reduce(termHash, new BLOBReducer(termHash, reducer));
|
|
}
|
|
|
|
public class BLOBReducer implements ImmutableBLOB.Reducer {
|
|
|
|
ContainerReducer<ReferenceType> rewriter;
|
|
byte[] wordHash;
|
|
|
|
public BLOBReducer(final byte[] wordHash, final ContainerReducer<ReferenceType> rewriter) {
|
|
this.rewriter = rewriter;
|
|
this.wordHash = wordHash;
|
|
}
|
|
|
|
@Override
|
|
public byte[] rewrite(final byte[] b) throws SpaceExceededException {
|
|
if (b == null) return null;
|
|
final ReferenceContainer<ReferenceType> c = this.rewriter.reduce(new ReferenceContainer<ReferenceType>(ReferenceContainerArray.this.factory, this.wordHash, RowSet.importRowSet(b, ReferenceContainerArray.this.factory.getRow())));
|
|
if (c == null) return null;
|
|
final byte bb[] = c.exportCollection();
|
|
assert bb.length <= b.length;
|
|
return bb;
|
|
}
|
|
}
|
|
|
|
public interface ContainerReducer<ReferenceType extends Reference> {
|
|
|
|
public ReferenceContainer<ReferenceType> reduce(ReferenceContainer<ReferenceType> container);
|
|
|
|
}
|
|
|
|
public CloneableIterator<byte[]> keys(boolean up, boolean rotating) throws IOException {
|
|
return this.array.keys(up, rotating);
|
|
}
|
|
|
|
public int entries() {
|
|
return this.array.entries();
|
|
}
|
|
|
|
public boolean shrinkBestSmallFiles(final IODispatcher merger, final long targetFileSize) {
|
|
final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize);
|
|
if (ff == null) return false;
|
|
ConcurrentLog.info("KELONDRO", "RICELL-shrink1: unmountBestMatch(2.0, " + targetFileSize + ")");
|
|
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
|
|
return true;
|
|
}
|
|
|
|
public boolean shrinkAnySmallFiles(final IODispatcher merger, final long targetFileSize) {
|
|
final File[] ff = this.array.unmountSmallest(targetFileSize);
|
|
if (ff == null) return false;
|
|
ConcurrentLog.info("KELONDRO", "RICELL-shrink2: unmountSmallest(" + targetFileSize + ")");
|
|
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
|
|
return true;
|
|
}
|
|
|
|
public boolean shrinkUpToMaxSizeFiles(final IODispatcher merger, final long maxFileSize) {
|
|
final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize);
|
|
if (ff == null) return false;
|
|
ConcurrentLog.info("KELONDRO", "RICELL-shrink3: unmountBestMatch(2.0, " + maxFileSize + ")");
|
|
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
|
|
return true;
|
|
}
|
|
|
|
public boolean shrinkOldFiles(final IODispatcher merger) {
|
|
final File ff = this.array.unmountOldest();
|
|
if (ff == null) return false;
|
|
ConcurrentLog.info("KELONDRO", "RICELL-shrink4/rewrite: unmountOldest()");
|
|
merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile());
|
|
return true;
|
|
}
|
|
}
|