Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-03-11 13:21:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

Ensure file input streams proper closing in both success and failures

Browse Source 
Also add when possible a warning level log message on input stream
closing error instead of failing silently. This could help understanding
some IO exceptions such as "too many files open".
This commit is contained in:
luccioman 2017-06-03 04:00:46 +02:00
parent d98c04853d
commit a04feac064
23 changed files with 335 additions and 125 deletions

8
source/net/yacy/document/parser/images/bmpParser.java
View File

@ -268,6 +268,14 @@ public class bmpParser {
ConcurrentLog.logException(e);
} catch (final IOException e) {
ConcurrentLog.logException(e);
} finally {
if(fis != null) {
try {
fis.close();
} catch(IOException ioe) {
ConcurrentLog.logException(ioe);
}
}
}
try {

13
source/net/yacy/document/parser/images/genericImageParser.java
View File

@ -301,9 +301,11 @@ public class genericImageParser extends AbstractParser implements Parser {
final File image = new File(args[0]);
final genericImageParser parser = new genericImageParser();
AnchorURL uri;
FileInputStream inStream = null;
try {
uri = new AnchorURL("http://localhost/" + image.getName());
final Document[] document = parser.parse(uri, "image/" + MultiProtocolURL.getFileExtension(uri.getFileName()), StandardCharsets.UTF_8.name(), new VocabularyScraper(), 0, new FileInputStream(image));
inStream = new FileInputStream(image);
final Document[] document = parser.parse(uri, "image/" + MultiProtocolURL.getFileExtension(uri.getFileName()), StandardCharsets.UTF_8.name(), new VocabularyScraper(), 0, inStream);
System.out.println(document[0].toString());
} catch (final MalformedURLException e) {
e.printStackTrace();
@ -313,6 +315,15 @@ public class genericImageParser extends AbstractParser implements Parser {
e.printStackTrace();
} catch (final InterruptedException e) {
e.printStackTrace();
} finally {
if(inStream != null) {
try {
inStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
ConcurrentLog.shutdown();
}
}

52
source/net/yacy/document/parser/images/icoParser.java
View File

@ -104,29 +104,41 @@ public class icoParser {
}
public static void main(final String[] args) {
// read a ICO and write it as png
System.setProperty("java.awt.headless", "true");
final File in = new File(args[0]);
final File out = new File(args[1]);
try {
// read a ICO and write it as png
System.setProperty("java.awt.headless", "true");
final File in = new File(args[0]);
final File out = new File(args[1]);
final byte[] file = new byte[(int) in.length()];
FileInputStream fis = null;
try {
fis = new FileInputStream(in);
fis.read(file);
} catch (final FileNotFoundException e) {
ConcurrentLog.logException(e);
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
final byte[] file = new byte[(int) in.length()];
FileInputStream fis = null;
try {
fis = new FileInputStream(in);
fis.read(file);
} catch (final FileNotFoundException e) {
ConcurrentLog.logException(e);
} catch (final IOException e) {
ConcurrentLog.logException(e);
} finally {
if(fis != null) {
try {
fis.close();
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
}
final icoParser parser = new icoParser(file);
final icoParser parser = new icoParser(file);
try {
ImageIO.write(parser.getImage(0), "PNG", out);
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
try {
ImageIO.write(parser.getImage(0), "PNG", out);
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
} finally {
ConcurrentLog.shutdown();
}
}
}

6
source/net/yacy/gui/framework/Switchboard.java
View File

@ -174,7 +174,11 @@ public class Switchboard {
log.info("error: file dispatcher.properties cannot be readed. Exit");
System.exit(-1);
} finally {
if (fis != null) try { fis.close(); } catch (IOException ex) { }
if (fis != null) try {
fis.close();
} catch (IOException ex) {
log.warn("Could not close input stream on file " + propFile);
}
}
}

39
source/net/yacy/kelondro/blob/Gap.java
View File

@ -62,23 +62,36 @@ public class Gap extends TreeMap<Long, Integer> {
super();
// read the index dump and fill the index
DataInputStream is;
FileInputStream fis = null;
try {
is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), (Integer.SIZE + Long.SIZE) * 1024)); // equals 16*1024*recordsize
fis = new FileInputStream(file);
is = new DataInputStream(new BufferedInputStream(fis, (Integer.SIZE + Long.SIZE) * 1024)); // equals 16*1024*recordsize
} catch (final OutOfMemoryError e) {
is = new DataInputStream(new FileInputStream(file));
if(fis != null) {
/* Reuse if possible the already created FileInputStream */
is = new DataInputStream(fis);
} else {
is = new DataInputStream(new FileInputStream(file));
}
}
long p;
int l;
while (true) {
try {
p = is.readLong();
l = is.readInt();
this.put(Long.valueOf(p), Integer.valueOf(l));
} catch (final IOException e) {
break;
}
try {
long p;
int l;
while (true) {
try {
p = is.readLong();
l = is.readInt();
this.put(Long.valueOf(p), Integer.valueOf(l));
} catch (final IOException e) {
break;
}
}
} finally {
if(is != null) {
is.close();
}
}
is.close();
is = null;
}

11
source/net/yacy/kelondro/blob/HeapReader.java
View File

@ -738,10 +738,17 @@ public class HeapReader {
public entries(final File blobFile, final int keylen) throws IOException {
if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist");
FileInputStream fis = null;
try {
this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 256 * 1024));
fis = new FileInputStream(blobFile);
this.is = new DataInputStream(new BufferedInputStream(fis, 256 * 1024));
} catch (final OutOfMemoryError e) {
this.is = new DataInputStream(new FileInputStream(blobFile));
if(fis != null) {
/* Reuse if possible the already created FileInputStream */
this.is = new DataInputStream(fis);
} else {
this.is = new DataInputStream(new FileInputStream(blobFile));
}
}
this.keylen = keylen;
this.blobFile = blobFile;

34
source/net/yacy/kelondro/index/RowHandleMap.java
View File

@ -88,22 +88,32 @@ public final class RowHandleMap implements HandleMap, Iterable<Map.Entry<byte[],
this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath());
// read the index dump and fill the index
InputStream is;
FileInputStream fis = null;
try {
is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
fis = new FileInputStream(file);
is = new BufferedInputStream(fis, 1024 * 1024);
} catch (final OutOfMemoryError e) {
is = new FileInputStream(file);
if(fis != null) {
/* Reuse if possible the already created FileInputStream */
is = fis;
} else {
is = new FileInputStream(file);
}
}
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
final byte[] a = new byte[keylength + idxbytes];
int c;
Row.Entry entry;
while (true) {
c = is.read(a);
if (c <= 0) break;
entry = this.rowdef.newEntry(a); // may be null if a is not well-formed
if (entry != null) this.index.addUnique(entry);
try {
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
final byte[] a = new byte[keylength + idxbytes];
int c;
Row.Entry entry;
while (true) {
c = is.read(a);
if (c <= 0) break;
entry = this.rowdef.newEntry(a); // may be null if a is not well-formed
if (entry != null) this.index.addUnique(entry);
}
} finally {
is.close();
}
is.close();
is = null;
assert this.index.size() == file.length() / (keylength + idxbytes);
optimize();

39
source/net/yacy/kelondro/index/RowHandleSet.java
View File

@ -107,15 +107,24 @@ public final class RowHandleSet implements HandleSet, Iterable<byte[]>, Cloneabl
public RowHandleSet(final int keylength, final ByteOrder objectOrder, final File file) throws IOException, SpaceExceededException {
this(keylength, objectOrder, (int) (file.length() / (keylength + 8)));
// read the index dump and fill the index
final InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
final byte[] a = new byte[keylength];
int c;
while (true) {
c = is.read(a);
if (c <= 0) break;
this.index.addUnique(this.rowdef.newEntry(a));
FileInputStream fis = new FileInputStream(file);
InputStream is = null;
try {
is = new BufferedInputStream(fis, 1024 * 1024);
final byte[] a = new byte[keylength];
int c;
while (true) {
c = is.read(a);
if (c <= 0) break;
this.index.addUnique(this.rowdef.newEntry(a));
}
} finally {
if(is != null) {
is.close();
} else if(fis != null) {
fis.close();
}
}
is.close();
assert this.index.size() == file.length() / keylength;
}
@ -388,13 +397,15 @@ public final class RowHandleSet implements HandleSet, Iterable<byte[]>, Cloneabl
// read from file
ObjectInputStream in = new ObjectInputStream(new FileInputStream(f));
RowHandleSet s1 = (RowHandleSet) in.readObject();
in.close();
for (byte[] b: s1) {
System.out.println(UTF8.String(b));
try {
RowHandleSet s1 = (RowHandleSet) in.readObject();
for (byte[] b: s1) {
System.out.println(UTF8.String(b));
}
s1.close();
} finally {
in.close();
}
s1.close();
} catch(IOException e) {
e.printStackTrace();
} catch (final ClassNotFoundException e) {

9
source/net/yacy/kelondro/util/FileUtils.java
View File

@ -292,6 +292,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@ -319,6 +320,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@ -411,6 +413,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
fis = null;
@ -478,13 +481,13 @@ public final class FileUtils {
set.add(line.trim().toLowerCase());
}
}
br.close();
} catch (final IOException e ) {
} finally {
if ( br != null ) {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
}
}
}
@ -622,7 +625,6 @@ public final class FileUtils {
while ( (line = br.readLine()) != null ) {
if (!line.isEmpty()) list.add(line);
}
br.close();
} catch (final IOException e ) {
// list is empty
} finally {
@ -630,6 +632,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}
@ -692,13 +695,13 @@ public final class FileUtils {
temp.append(line).append(CR).append(LF);
}
}
br.close();
} catch (final IOException e ) {
} finally {
if ( br != null ) {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}

4
source/net/yacy/kelondro/util/SetTools.java
View File

@ -529,7 +529,9 @@ public final class SetTools {
}
} catch (final IOException e) {
} finally {
if (br != null) try{br.close();}catch(final Exception e){}
if (br != null) try{br.close();}catch(final Exception e){
ConcurrentLog.warn("SetTools", "Could not close input stream on file " + file);
}
}
return list;
}

7
source/net/yacy/kelondro/util/XMLTables.java
View File

@ -57,8 +57,11 @@ public class XMLTables {
this.timestamp = System.currentTimeMillis();
if (propFile.exists()) {
final XMLDecoder xmldec = new XMLDecoder(new FileInputStream(propFile));
tables = (HashMap<String, Map<String, String>>) xmldec.readObject();
xmldec.close();
try {
tables = (HashMap<String, Map<String, String>>) xmldec.readObject();
} finally {
xmldec.close();
}
} else {
tables = new HashMap<String, Map<String, String>>();
}

26
source/net/yacy/repository/Blacklist.java
View File

@ -795,14 +795,32 @@ public class Blacklist {
private final void loadDHTCache(final BlacklistType type) {
File cachefile = DHTCacheFile(type);
if (cachefile.exists()) {
FileInputStream fileInStream = null;
ObjectInputStream in = null;
try {
ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile));
RowHandleSet rhs = (RowHandleSet) in.readObject();
this.cachedUrlHashs.put(type, rhs == null ? new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0) : rhs);
in.close();
fileInStream = new FileInputStream(cachefile);
in = new ObjectInputStream(fileInStream);
RowHandleSet rhs = (RowHandleSet) in.readObject();
this.cachedUrlHashs.put(type, rhs == null ? new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0) : rhs);
return;
} catch (final Throwable e) {
ConcurrentLog.logException(e);
} finally {
if(in != null) {
try {
in.close();
} catch(IOException ioe) {
log.warn("Could not close object input stream on file " + cachefile);
}
} else if(fileInStream != null){
/* An error may have been thrown while constructing the ObjectInputStream :
* by the way the file input stream still has to be closed properly */
try {
fileInStream.close();
} catch(IOException ioe) {
log.warn("Could not close input stream on file " + cachefile);
}
}
}
}
this.cachedUrlHashs.put(type, new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0));

11
source/net/yacy/search/AutoSearch.java
View File

@ -88,7 +88,15 @@ public class AutoSearch extends AbstractBusyThread {
ConcurrentLog.info(AutoSearch.class.getName(), "read queries from file " + pfile.getAbsolutePath());
Properties prop = new Properties();
FileInputStream fileIn = new FileInputStream(pfile);
prop.load(fileIn);
try {
prop.load(fileIn);
} finally {
try {
fileIn.close();
} catch(IOException ioe) {
ConcurrentLog.warn(AutoSearch.class.getName(), "Could not close input stream on file " + pfile);
}
}
if (prop.size() > 0) {
Set<Object> all = prop.keySet();
for (Object s : all) {
@ -98,7 +106,6 @@ public class AutoSearch extends AbstractBusyThread {
}
}
}
fileIn.close();
}
} catch (final IOException e) {
ConcurrentLog.warn(AutoSearch.class.getName(), "Error reading config file");

74
source/net/yacy/search/Switchboard.java
View File

@ -87,11 +87,15 @@ import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.servlet.http.HttpServletRequest;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.google.common.io.Files;
import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
import net.yacy.contentcontrol.SMWListSyncThread;
@ -136,9 +140,9 @@ import net.yacy.crawler.data.Cache;
import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.data.CrawlQueues;
import net.yacy.crawler.data.NoticedURL;
import net.yacy.crawler.data.NoticedURL.StackType;
import net.yacy.crawler.data.ResultImages;
import net.yacy.crawler.data.ResultURLs;
import net.yacy.crawler.data.NoticedURL.StackType;
import net.yacy.crawler.data.ResultURLs.EventOrigin;
import net.yacy.crawler.data.Transactions;
import net.yacy.crawler.retrieval.Request;
@ -161,11 +165,11 @@ import net.yacy.document.Condenser;
import net.yacy.document.Document;
import net.yacy.document.LibraryProvider;
import net.yacy.document.Parser;
import net.yacy.document.Parser.Failure;
import net.yacy.document.ProbabilisticClassifier;
import net.yacy.document.TextParser;
import net.yacy.document.VocabularyScraper;
import net.yacy.document.Parser.Failure;
import net.yacy.document.Tokenizer;
import net.yacy.document.VocabularyScraper;
import net.yacy.document.content.DCEntry;
import net.yacy.document.content.SurrogateReader;
import net.yacy.document.importer.OAIListFriendsLoader;
@ -190,11 +194,11 @@ import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Dispatcher;
import net.yacy.peers.EventChannel;
import net.yacy.peers.Network;
import net.yacy.peers.NewsPool;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB;
@ -226,10 +230,6 @@ import net.yacy.utils.crypt;
import net.yacy.utils.upnp.UPnP;
import net.yacy.visualization.CircleTool;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.google.common.io.Files;
public final class Switchboard extends serverSwitch {
@ -2005,7 +2005,9 @@ public final class Switchboard extends serverSwitch {
ConcurrentLog.logException(e);
} finally {
moved = infile.renameTo(outfile);
if (zis != null) try {zis.close();} catch (final IOException e) {}
if (zis != null) try {zis.close();} catch (final IOException e) {
log.warn("Could not close zip input stream on file " + infile);
}
}
return moved;
} else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) {
@ -2025,9 +2027,12 @@ public final class Switchboard extends serverSwitch {
} else if (s.endsWith(".jsonlist") || s.endsWith(".flatjson")) {
// parse a file that can be generated with yacy_grid_parser
// see https://github.com/yacy/yacy_grid_parser/blob/master/README.md
FileInputStream fis = null;
BufferedReader br = null;
try {
InputStream is = new BufferedInputStream(new FileInputStream(infile));
BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
fis = new FileInputStream(infile);
InputStream is = new BufferedInputStream(fis);
br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
String line;
while ((line = br.readLine()) != null) {
JSONTokener jt = new JSONTokener(line);
@ -2098,10 +2103,29 @@ public final class Switchboard extends serverSwitch {
}
Switchboard.this.index.putDocument(surrogate);
}
is.close();
br.close();
br = null;
fis = null;
moved = infile.renameTo(outfile);
} catch (IOException ex) {
log.warn("IO Error processing flatjson file " + infile);
} finally {
/* Properly release file system resources even in failure cases */
if(br != null) {
/* buffered reader was successfully created : close it and its underlying streams */
try {
br.close();
} catch (IOException e) {
log.warn("Could not close reader on file " + infile);
}
} else if(fis != null) {
/* no buffered reader : maybe a case of exhausted memory. Anyway file input stream has to be closed. */
try {
fis.close();
} catch (IOException e) {
log.warn("Could not close input stream on file " + infile);
}
}
}
return moved;
}
@ -2123,9 +2147,19 @@ public final class Switchboard extends serverSwitch {
try {
final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}});
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile));
FileUtils.copy(bis, os);
os.close();
bis.close();
try {
FileUtils.copy(bis, os);
} finally {
try {
os.close();
} finally {
try {
bis.close();
} catch(IOException ignored) {
log.warn("Could not close input stream on file " + outfile);
}
}
}
if ( gzfile.exists() ) {
FileUtils.deletedelete(outfile);
}
@ -2138,7 +2172,9 @@ public final class Switchboard extends serverSwitch {
log.info("processed surrogate " + infile);
}
}
if (is != null) try {is.close();} catch (IOException e) {}
if (is != null) try {is.close();} catch (IOException e) {
log.warn("Could not close input stream on file " + infile);
}
}
return moved;
}
@ -2610,9 +2646,10 @@ public final class Switchboard extends serverSwitch {
if ( !isRobinsonMode() && this.peers.newsPool.size(NewsPool.OUTGOING_DB) == 0 ) {
// read profile
final Properties profile = new Properties();
final File profileFile = new File(this.dataPath, "DATA/SETTINGS/profile.txt");
FileInputStream fileIn = null;
try {
fileIn = new FileInputStream(new File(this.dataPath, "DATA/SETTINGS/profile.txt"));
fileIn = new FileInputStream(profileFile);
profile.load(fileIn);
} catch (final IOException e ) {
} finally {
@ -2620,6 +2657,7 @@ public final class Switchboard extends serverSwitch {
try {
fileIn.close();
} catch (final Exception e ) {
log.warn("Could not close input stream on file " + profileFile);
}
}
}

4
source/net/yacy/server/http/TemplateEngine.java
View File

@ -427,7 +427,9 @@ public final class TemplateEngine {
//file not found?
ConcurrentLog.severe("FILEHANDLER","Include Error with file " + UTF8.String(filename) + ": " + e.getMessage());
} finally {
if (br != null) try { br.close(); br=null; } catch (final Exception e) {}
if (br != null) try { br.close(); br=null; } catch (final Exception e) {
ConcurrentLog.warn("FILEHANDLER","Could not close buffered reader on file " + UTF8.String(filename));
}
}
final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
structure.append(ASCII.getBytes("<fileinclude file=\"")).append(filename).append(close_tagn);

13
source/net/yacy/utils/PKCS12Tool.java
View File

@ -95,8 +95,17 @@ public class PKCS12Tool {
} else{
System.err.println("Creating new java keystore '" + jksFile + "'");
}
jks.load(jksFileIn,(jksPassword!=null)?jksPassword.toCharArray():null);
if (jksFileIn != null) jksFileIn.close();
try {
jks.load(jksFileIn,(jksPassword!=null)?jksPassword.toCharArray():null);
} finally {
if (jksFileIn != null) {
try {
jksFileIn.close();
} catch(IOException ioe) {
System.err.println("Error while closing input stream on file " + jksFile);
}
}
}
final Enumeration<String> pkcs12Aliases = aliases();
while (pkcs12Aliases.hasMoreElements()) {

22
source/net/yacy/utils/cryptbig.java
View File

@ -231,6 +231,8 @@ public class cryptbig {
<compressed-after-encryption-flag>
<binary>
*/
InputStream fin = null;
OutputStream fout = null;
try {
final File inFile = new File(inFileName);
final String inFileDate = dateFormatter.format(new Date(inFile.lastModified())); // 17 byte
@ -248,8 +250,8 @@ public class cryptbig {
System.out.println("TEST: preserving X-String : " + X);
// start encryption
final InputStream fin = new CipherInputStream(new FileInputStream(inFile), this.ecipher);
final OutputStream fout = new FileOutputStream(outFileName);
fin = new CipherInputStream(new FileInputStream(inFile), this.ecipher);
fout = new FileOutputStream(outFileName);
// write magic and properties of original file
// - we encrypt the original date, the encryption date, the file size, the flag
@ -272,13 +274,23 @@ public class cryptbig {
}
catch (final javax.crypto.IllegalBlockSizeException e) {System.err.println("ERROR:" + e.getMessage());}
catch (final javax.crypto.BadPaddingException e) {System.err.println("ERROR:" + e.getMessage());}
// finished files
fin.close();
fout.close();
} catch (final FileNotFoundException e) {
System.err.println("ERROR: file '" + inFileName + "' not found");
} catch (final IOException e) {
System.err.println("ERROR: IO trouble");
} finally {
try {
if(fin != null) {
fin.close();
}
} catch (IOException ignored) {
}
try {
if(fout != null) {
fout.close();
}
} catch(IOException ignored) {
}
}
}

13
source/net/yacy/utils/tarTools.java
View File

@ -59,8 +59,10 @@ public class tarTools {
*/
public static InputStream getInputStream(final String tarPath) throws FileNotFoundException {
if (tarPath.endsWith(".gz")) {
FileInputStream fileInStream = null;
try {
return new GZIPInputStream(new FileInputStream(new File(tarPath)));
fileInStream = new FileInputStream(new File(tarPath));
return new GZIPInputStream(fileInStream);
} catch (FileNotFoundException e) {
/*
* FileNotFoundException is is a subClass of IOException but the
@ -68,6 +70,15 @@ public class tarTools {
*/
throw e;
} catch (final IOException e) {
if(fileInStream != null) {
try {
/* release the now useless firstly opened file input stream
* (we can not reuse it as the header has been read by the GZIPInputStream) */
fileInStream.close();
} catch (IOException e1) {
ConcurrentLog.warn("UNTAR", "Could not close input stream on file " + tarPath);
}
}
// this might happen if the stream is not in gzip format.
// there may be a 'gz' extension, but it may still be a raw tar file
// this can be caused by 'one too much gzip-content header' that was attached

2
source/net/yacy/utils/translation/TranslationManager.java
View File

@ -192,13 +192,13 @@ public class TranslationManager extends TranslatorXliff {
while ((line = br.readLine()) != null) {
content.append(line).append(net.yacy.server.serverCore.CRLF_STRING);
}
br.close();
} catch (final IOException e) {
} finally {
if (br != null) {
try {
br.close();
} catch (final Exception e) {
ConcurrentLog.fine("TRANSLATOR", "Could not close buffered reader on file " + checkfile);
}
}
}

20
test/java/net/yacy/document/parser/htmlParserTest.java
View File

@ -77,12 +77,22 @@ public class htmlParserTest extends TestCase {
System.out.println("parse file: " + filename);
htmlParser p = new htmlParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
Document doc = docs[0];
String txt = doc.getCharset();
assertTrue("get Charset", txt != null);
System.out.println("detected charset = " + txt);
} finally {
if(inStream != null) {
System.out.println("Could not close input stream on file " + file);
}
}
Document doc = docs[0];
String txt = doc.getCharset();
assertTrue("get Charset", txt != null);
System.out.println("detected charset = " + txt);
}
}

14
test/java/net/yacy/document/parser/images/genericImageParserTest.java
View File

@ -29,11 +29,15 @@ public class genericImageParserTest {
System.out.println("parse file: " + filename);
genericImageParser p = new genericImageParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file));
Document doc = docs[0];
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
FileInputStream inStream = new FileInputStream(file);
try {
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
Document doc = docs[0];
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
} finally {
inStream.close();
}
}
}

13
test/java/net/yacy/document/parser/images/metadataImageParserTest.java
View File

@ -30,11 +30,16 @@ public class metadataImageParserTest {
System.out.println("parse file: " + filename);
metadataImageParser p = new metadataImageParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file));
FileInputStream inStream = new FileInputStream(file);
try {
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
Document doc = docs[0];
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
Document doc = docs[0];
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
} finally {
inStream.close();
}
}
}

26
test/java/net/yacy/document/parser/pdfParserTest.java
View File

@ -2,6 +2,7 @@ package net.yacy.document.parser;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection;
import static junit.framework.TestCase.assertEquals;
import net.yacy.cora.document.id.AnchorURL;
@ -30,16 +31,25 @@ public class pdfParserTest {
System.out.println("parse file: " + filename);
pdfParser p = new pdfParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file));
FileInputStream inStream = new FileInputStream(file);
try {
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
Document doc = docs[0];
int ilinks = doc.getAnchors().size();
assertEquals("number of links in pdf", 1, ilinks);
Document doc = docs[0];
int ilinks = doc.getAnchors().size();
assertEquals("number of links in pdf", 1, ilinks);
Collection<AnchorURL> links = doc.getAnchors();
System.out.println("number of links detected = " + ilinks);
for (AnchorURL aurl : links) {
System.out.println(" found: " + aurl.toString());
Collection<AnchorURL> links = doc.getAnchors();
System.out.println("number of links detected = " + ilinks);
for (AnchorURL aurl : links) {
System.out.println(" found: " + aurl.toString());
}
} finally {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + file);
}
}
}