mirror of
https://github.com/yacy/yacy_search_server.git
synced 2025-07-19 08:44:42 -04:00
enhanced network scanner
- textarea input field can now be used to paste in a large list of hosts - /31er subnet is possible (only one host) - auto-detect subdomains for ftp and www subdomains
This commit is contained in:
@ -41,20 +41,19 @@
|
||||
</legend>
|
||||
<dl>
|
||||
<dt>Scan Range</dt>
|
||||
<dd>
|
||||
<dd align="top">
|
||||
<input type="radio" name="source" id="sourcehost" value="hosts"#(intranet.checked)# checked="checked"::#(/intranet.checked)# />Scan sub-range with given host
|
||||
<input type="text" name="scanhosts" value="#[scanhosts]#" size="60" maxlength="400" /><br/><br/>
|
||||
|
||||
<textarea name="scanhosts" id="scanhosts" value="#[scanhosts]#" cols="64" rows="3" size="41"></textarea><br/><br/>
|
||||
<input type="radio" name="source" id="sourcenet" value="intranet"#(intranet.checked)#:: checked="checked"#(/intranet.checked)# />Full Intranet Scan: #[intranethosts]#<br/>
|
||||
#(intranetHint)#::<div class="info"> Do not use intranet scan results, you are not in an intranet environment!</div>#(/intranetHint)#
|
||||
</dd>
|
||||
<dt>Time-Out</dt>
|
||||
<dd>
|
||||
<input type="text" name="timeout" value ="100" size="4"/> ms
|
||||
</dd>
|
||||
<dt>Subnet</dt>
|
||||
<dd>
|
||||
<input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
|
||||
<input type="radio" name="subnet" value="31"/>/31 (only the given host(s)) <input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
|
||||
</dd>
|
||||
<dt>Time-Out</dt>
|
||||
<dd>
|
||||
<input type="text" name="timeout" value ="500" size="4"/> ms
|
||||
</dd>
|
||||
<dt>Scan Cache</dt>
|
||||
<dd>
|
||||
|
@ -18,27 +18,33 @@
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ConcurrentModificationException;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
|
||||
import net.yacy.cora.order.Base64Order;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.Scanner;
|
||||
import net.yacy.cora.protocol.Scanner.Access;
|
||||
import net.yacy.cora.sorting.ReversibleScoreMap;
|
||||
import net.yacy.data.WorkTables;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.SwitchboardConstants;
|
||||
import net.yacy.search.query.SearchEventCache;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
@ -66,11 +72,20 @@ public class CrawlStartScanner_p
|
||||
timeout = post == null ? timeout : post.getInt("timeout", timeout);
|
||||
|
||||
// make a scanhosts entry
|
||||
String hosts = post == null ? "" : post.get("scanhosts", "");
|
||||
String hostt = post == null ? "" : post.get("scanhosts", "").trim();
|
||||
boolean listall = false;
|
||||
if (hostt.equals("*")) {
|
||||
hostt = "";
|
||||
listall = true;
|
||||
}
|
||||
String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
|
||||
Set<String> hostSet = new LinkedHashSet<String>();
|
||||
for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);
|
||||
|
||||
final Set<InetAddress> ips = Domains.myIntranetIPs();
|
||||
prop.put("intranethosts", ips.toString());
|
||||
prop.put("intranetHint", sb.isIntranetMode() ? 0 : 1);
|
||||
if ( hosts.isEmpty() ) {
|
||||
if ( hostSet.isEmpty() ) {
|
||||
InetAddress ip;
|
||||
if ( sb.isIntranetMode() ) {
|
||||
if ( !ips.isEmpty() ) {
|
||||
@ -85,11 +100,23 @@ public class CrawlStartScanner_p
|
||||
}
|
||||
}
|
||||
if ( ip != null ) {
|
||||
hosts = ip.getHostAddress();
|
||||
hostSet.add(ip.getHostAddress());
|
||||
}
|
||||
}
|
||||
prop.put("scanhosts", hosts);
|
||||
String hos = ""; for (String s: hostSet) hos += s + "\n";
|
||||
prop.put("scanhosts", hos.trim());
|
||||
|
||||
if (listall) {
|
||||
// get a list of all hosts in the index
|
||||
ReversibleScoreMap<String> hostscore = null;
|
||||
try {
|
||||
hostscore = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_TERM, 1000, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
|
||||
} catch (IOException e) {}
|
||||
if (hostscore != null) {
|
||||
for (String s: hostscore) hostSet.add(s);
|
||||
}
|
||||
}
|
||||
|
||||
// parse post requests
|
||||
if ( post != null ) {
|
||||
int repeat_time = 0;
|
||||
@ -105,18 +132,35 @@ public class CrawlStartScanner_p
|
||||
|
||||
// scan a range of ips
|
||||
if (post.containsKey("scan")) {
|
||||
|
||||
boolean scanftp = "on".equals(post.get("scanftp", ""));
|
||||
boolean scanhttp = "on".equals(post.get("scanhttp", ""));
|
||||
boolean scanhttps = "on".equals(post.get("scanhttps", ""));
|
||||
boolean scansmb = "on".equals(post.get("scansmb", ""));
|
||||
|
||||
final Set<InetAddress> scanbase = new HashSet<InetAddress>();
|
||||
|
||||
// select host base to scan
|
||||
if ("hosts".equals(post.get("source", ""))) {
|
||||
for (String host: hosts.split(",")) {
|
||||
for (String host: hostSet) {
|
||||
if (host.startsWith("http://")) host = host.substring(7);
|
||||
if (host.startsWith("https://")) host = host.substring(8);
|
||||
if (host.startsWith("ftp://")) host = host.substring(6);
|
||||
if (host.startsWith("smb://")) host = host.substring(6);
|
||||
final int p = host.indexOf('/', 0);
|
||||
if (p >= 0) host = host.substring(0, p);
|
||||
if (host.length() > 0) scanbase.add(Domains.dnsResolve(host));
|
||||
InetAddress ip;
|
||||
if (host.length() > 0) {
|
||||
ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);
|
||||
if (scanftp && !hostSet.contains("ftp." + host)) {
|
||||
ip = Domains.dnsResolve("ftp." + host);
|
||||
if (ip != null) scanbase.add(ip);
|
||||
}
|
||||
if ((scanhttp || scanhttps) && !hostSet.contains("www." + host)) {
|
||||
ip = Domains.dnsResolve("www." + host);
|
||||
if (ip != null) scanbase.add(ip);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ("intranet".equals(post.get("source", ""))) {
|
||||
@ -124,12 +168,12 @@ public class CrawlStartScanner_p
|
||||
}
|
||||
|
||||
// start a scanner
|
||||
final Scanner scanner = new Scanner(scanbase, CONCURRENT_RUNNER, timeout);
|
||||
List<InetAddress> addresses = scanner.genlist(subnet);
|
||||
if ("on".equals(post.get("scanftp", ""))) scanner.addFTP(addresses);
|
||||
if ("on".equals(post.get("scanhttp", ""))) scanner.addHTTP(addresses);
|
||||
if ("on".equals(post.get("scanhttps", ""))) scanner.addHTTPS(addresses);
|
||||
if ("on".equals(post.get("scansmb", ""))) scanner.addSMB(addresses);
|
||||
final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);
|
||||
List<InetAddress> addresses = Scanner.genlist(scanbase, subnet);
|
||||
if (scanftp) scanner.addFTP(addresses);
|
||||
if (scanhttp) scanner.addHTTP(addresses);
|
||||
if (scanhttps) scanner.addHTTPS(addresses);
|
||||
if (scansmb) scanner.addSMB(addresses);
|
||||
scanner.start();
|
||||
scanner.terminate();
|
||||
if ("on".equals(post.get("accumulatescancache", "")) && !"scheduler".equals(post.get("rescan", ""))) {
|
||||
@ -183,7 +227,7 @@ public class CrawlStartScanner_p
|
||||
post,
|
||||
"CrawlStartScanner_p.html",
|
||||
WorkTables.TABLE_API_TYPE_CRAWLER,
|
||||
"network scanner for hosts: " + hosts,
|
||||
"network scanner for hosts: " + hostSet.toString(),
|
||||
repeat_time,
|
||||
repeat_unit.substring(3));
|
||||
}
|
||||
|
@ -29,13 +29,12 @@ import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
@ -121,9 +120,6 @@ public class Scanner extends Thread {
|
||||
}
|
||||
|
||||
private final static Map<Service, Access> scancache = new ConcurrentHashMap<Service, Access>();
|
||||
//private static long scancacheUpdateTime = 0;
|
||||
//private static long scancacheValidUntilTime = Long.MAX_VALUE;
|
||||
private static Set<InetAddress> scancacheScanrange = new HashSet<InetAddress>();
|
||||
|
||||
public static int scancacheSize() {
|
||||
return scancache.size();
|
||||
@ -132,9 +128,6 @@ public class Scanner extends Thread {
|
||||
public static void scancacheReplace(final Scanner newScanner) {
|
||||
scancache.clear();
|
||||
scancache.putAll(newScanner.services());
|
||||
//scancacheUpdateTime = System.currentTimeMillis();
|
||||
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
|
||||
scancacheScanrange = newScanner.scanrange;
|
||||
}
|
||||
|
||||
public static void scancacheExtend(final Scanner newScanner) {
|
||||
@ -145,9 +138,6 @@ public class Scanner extends Thread {
|
||||
if (entry.getValue() != Access.granted) i.remove();
|
||||
}
|
||||
scancache.putAll(newScanner.services());
|
||||
//scancacheUpdateTime = System.currentTimeMillis();
|
||||
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
|
||||
scancacheScanrange = newScanner.scanrange;
|
||||
}
|
||||
|
||||
public static Iterator<Map.Entry<Service, Scanner.Access>> scancacheEntries() {
|
||||
@ -163,18 +153,17 @@ public class Scanner extends Thread {
|
||||
*/
|
||||
public static boolean acceptURL(final MultiProtocolURI url) {
|
||||
// if the scan range is empty, then all urls are accepted
|
||||
if (scancacheScanrange == null || scancacheScanrange.isEmpty()) return true;
|
||||
if (scancache == null || scancache.isEmpty()) return true;
|
||||
|
||||
//if (System.currentTimeMillis() > scancacheValidUntilTime) return true;
|
||||
final InetAddress a = url.getInetAddress(); // try to avoid that!
|
||||
if (a == null) return true;
|
||||
final InetAddress n = normalize(a);
|
||||
if (!scancacheScanrange.contains(n)) return true;
|
||||
final Access access = scancache.get(new Service(url.getProtocol(), a));
|
||||
if (access == null) return false;
|
||||
return access == Access.granted;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
private static InetAddress normalize(final InetAddress a) {
|
||||
if (a == null) return null;
|
||||
final byte[] b = a.getAddress();
|
||||
@ -186,28 +175,22 @@ public class Scanner extends Thread {
|
||||
return a;
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
private final int runnerCount;
|
||||
private final Set<InetAddress> scanrange;
|
||||
private final BlockingQueue<Service> scanqueue;
|
||||
private final Map<Service, Access> services;
|
||||
private final Map<Runner, Object> runner;
|
||||
private final int timeout;
|
||||
|
||||
public Scanner(final Set<InetAddress> scanrange, final int concurrentRunner, final int timeout) {
|
||||
public Scanner(final int concurrentRunner, final int timeout) {
|
||||
this.runnerCount = concurrentRunner;
|
||||
this.scanrange = new HashSet<InetAddress>();
|
||||
for (final InetAddress a: scanrange) this.scanrange.add(normalize(a));
|
||||
this.scanqueue = new LinkedBlockingQueue<Service>();
|
||||
this.services = Collections.synchronizedMap(new HashMap<Service, Access>());
|
||||
this.runner = new ConcurrentHashMap<Runner, Object>();
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public Scanner(final int concurrentRunner, final int timeout) {
|
||||
this(Domains.myIntranetIPs(), concurrentRunner, timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Service uri;
|
||||
@ -332,23 +315,37 @@ public class Scanner extends Thread {
|
||||
* @param subnet the subnet: 24 will generate 254 addresses, 16 will generate 256 * 254; must be >= 16 and <= 24
|
||||
* @return
|
||||
*/
|
||||
public final List<InetAddress> genlist(final int subnet) {
|
||||
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(10);
|
||||
for (final InetAddress i: this.scanrange) {
|
||||
int ul = subnet >= 24 ? i.getAddress()[2] : (1 << (24 - subnet)) - 1;
|
||||
for (int br = subnet >= 24 ? i.getAddress()[2] : 0; br <= ul; br++) {
|
||||
for (int j = 1; j < 255; j++) {
|
||||
final byte[] address = i.getAddress();
|
||||
address[2] = (byte) br;
|
||||
address[3] = (byte) j;
|
||||
try {
|
||||
c.add(InetAddress.getByAddress(address));
|
||||
} catch (final UnknownHostException e) {
|
||||
public static final List<InetAddress> genlist(Collection<InetAddress> base, final int subnet) {
|
||||
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
|
||||
for (final InetAddress i: base) {
|
||||
genlist(c, i, subnet);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
public static final List<InetAddress> genlist(InetAddress base, final int subnet) {
|
||||
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
|
||||
genlist(c, base, subnet);
|
||||
return c;
|
||||
}
|
||||
private static final void genlist(ArrayList<InetAddress> c, InetAddress base, final int subnet) {
|
||||
if (subnet == 31) {
|
||||
try {
|
||||
c.add(InetAddress.getByAddress(base.getAddress()));
|
||||
} catch (UnknownHostException e) {}
|
||||
} else {
|
||||
int ul = subnet >= 24 ? base.getAddress()[2] : (1 << (24 - subnet)) - 1;
|
||||
for (int br = subnet >= 24 ? base.getAddress()[2] : 0; br <= ul; br++) {
|
||||
for (int j = 1; j < 255; j++) {
|
||||
final byte[] address = base.getAddress();
|
||||
address[2] = (byte) br;
|
||||
address[3] = (byte) j;
|
||||
try {
|
||||
c.add(InetAddress.getByAddress(address));
|
||||
} catch (final UnknownHostException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
public Map<Service, Access> services() {
|
||||
@ -365,7 +362,7 @@ public class Scanner extends Thread {
|
||||
public static void main(final String[] args) {
|
||||
//try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {}
|
||||
final Scanner scanner = new Scanner(100, 10);
|
||||
List<InetAddress> addresses = scanner.genlist(20);
|
||||
List<InetAddress> addresses = genlist(Domains.myIntranetIPs(), 20);
|
||||
scanner.addFTP(addresses);
|
||||
scanner.addHTTP(addresses);
|
||||
scanner.addHTTPS(addresses);
|
||||
|
Reference in New Issue
Block a user