Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

new graphic icons. minor clean ups.

Browse Source
This commit is contained in:
Matt Wells 2013-11-15 14:47:05 -07:00
parent b1e98aa4b8
commit 5e30728a3a
12 changed files with 173 additions and 43 deletions

6
Conf.cpp
View File

@ -182,6 +182,7 @@ bool Conf::init ( char *dir ) { // , long hostId ) {
//g_conf.m_testSearchEnabled = false;
/*
//
// are we running in Matt Wells's data center?
// if so, we want to be able to use the seo tools that are not part
@ -204,7 +205,12 @@ bool Conf::init ( char *dir ) { // , long hostId ) {
//if(hh[0]=='s' && hh[1]=='p' && is_digit(hh[2])) ) priv = true;
if ( priv ) g_conf.m_isMattWells = true;
else g_conf.m_isMattWells = false;
*/
g_conf.m_isMattWells = false;
#ifdef IS_MATTWELLS
g_conf.m_isMattWells = true;
#endif
// this is not possible
/*

2
Facebook.h
View File

@ -238,7 +238,7 @@ class FBRec {
#endif
// facebook id for matt wells
#define MATTWELLS 100003532411011LL
#define FB_MATTWELLS 100003532411011LL
//#define APPNAME "Event Widget"

2
Makefile
View File

@ -77,7 +77,7 @@ ifeq ("titan","$(HOST)")
# in 2013. So it just uses clone() and does its own "threading". Unfortunately,
# the way it works is not even possible on newer kernels because they no longer
# allow you to override the _errno_location() function. -- matt
CPPFLAGS = -m32 -g -Wall -pipe -Wno-write-strings -Wstrict-aliasing=0 -Wno-uninitialized -static
CPPFLAGS = -m32 -g -Wall -pipe -Wno-write-strings -Wstrict-aliasing=0 -Wno-uninitialized -static -DIS_MATTWELLS
LIBS = ./libz.a ./libssl.a ./libcrypto.a ./libiconv.a ./libm.a
else
# use -m32 to force 32-bit mode compilation.

84
PageRoot.cpp
View File

@ -43,15 +43,37 @@ bool sendPageRoot ( TcpSocket *s, HttpRequest *r ){
}
bool printNav ( SafeBuf &sb , HttpRequest *r ) {
char *root = "";
char *rootSecure = "";
if ( g_conf.m_isMattWells ) {
root = "http://www.gigablast.com";
rootSecure = "https://www.gigablast.com";
}
sb.safePrintf("<center><b><p class=nav>"
"<a href=\"/about.html\">About</a>"
" &nbsp; &nbsp; <a href=\"/contact.html\">Contact</a>"
" &nbsp; &nbsp;<a href=\"/help.html\">Help</a>"
" &nbsp; &nbsp; <a href=/privacy.html>Privacy Policy</a>"
" &nbsp; &nbsp;<a href=\"/searchfeed.html\">"
"Search API</a>"
" &nbsp; &nbsp; <a href=/seoapi.html>SEO API</a>"
" &nbsp; &nbsp; <a href=/account>My Account</a> "
"<a href=%s/about.html>About</a>"
" &nbsp; &nbsp; "
"<a href=%s/contact.html>Contact</a>"
" &nbsp; &nbsp; "
"<a href=%s/help.html>Help</a>"
" &nbsp; &nbsp; "
"<a href=%s/privacy.html>Privacy Policy</a>"
" &nbsp; &nbsp; "
"<a href=%s/searchfeed.html>Search API</a>"
" &nbsp; &nbsp; "
"<a href=%s/seoapi.html>SEO API</a>"
" &nbsp; &nbsp; "
"<a href=%s/account>My Account</a> "
, root
, root
, root
, root
, root
, root
, root
, rootSecure
//" &nbsp; &nbsp; <a href=/logout>Logout</a>"
);
if ( r->isLocal() )
@ -135,7 +157,7 @@ bool printWebHomePage ( SafeBuf &sb , HttpRequest *r ) {
sb.safePrintf("<tr valign=top>\n");
sb.safePrintf("<td><div style=width:50px;height:50px;display:inline-block;background-color:red;></td>\n");
sb.safePrintf("<td align=center><div style=width:50px;height:50px;display:inline-block;background-color:red;></div></td>\n");
sb.safePrintf("<td><font size=+1><b>Open Source!</b>"
"</font><br>\n");
sb.brify2("Gigablast is now available as an <a href=https://github.com/gigablast/open-source-search-engine>open source search engine</a> on github.com. Download it today. Finally a robust, scalable search solution in C/C++ that has been in development and used commercially since 2000. <a href=/admin.html#features>Features.</a> Limited support available for free."
@ -146,17 +168,28 @@ bool printWebHomePage ( SafeBuf &sb , HttpRequest *r ) {
sb.safePrintf("<tr valign=top>\n");
sb.safePrintf("<td><div style=width:50px;height:50px;display:inline-block;background-color:green;></td>\n");
sb.safePrintf("<td><font size=+1><b>The Green Search Engine</b></font><br>\n");
sb.brify2("Gigablast is the only clean-powered web search engine. 90% of its power usage comes from wind energy. Astoundingly, Gigablast is one of ONLY four search engines in the United States indexing over a billion pages.",80);
// 204x143
sb.safePrintf("<td><img height=52px width=75px src=http://www.eventguru.com/eventguru.png></td>\n");
sb.safePrintf("<td><font size=+1><b>Event Guru Returns</b></font><br>\n");
sb.brify2("<a href=http://www.eventguru.com/>Event Guru</a> datamines events from the web. It identifies events on a web page, or even plain text, using the same rules of deduction used by the human mind. It also has Facebook integration and lots of other cool things.",80);
sb.safePrintf("<br><br></td></tr>\n");
sb.safePrintf("\n");
sb.safePrintf("\n");
/*
sb.safePrintf("<tr valign=top>\n");
sb.safePrintf("<td align=center><div style=width:50px;height:50px;display:inline-block;background-color:green;></div></td>\n");
sb.safePrintf("<td><font size=+1><b>The Green Search Engine</b></font><br>\n");
sb.brify2("Gigablast is the only clean-powered web search engine. 90% of its power usage comes from wind energy. Astoundingly, Gigablast is one of ONLY four search engines in the United States indexing over a billion pages.",80);
sb.safePrintf("<br><br></td></tr>\n");
sb.safePrintf("\n");
sb.safePrintf("\n");
*/
sb.safePrintf("<tr valign=top>\n");
sb.safePrintf("<td><div style=width:50px;height:50px;display:inline-block;background-color:0040fe;></td>\n");
sb.safePrintf("<td align=center><img src=http://www.gigablast.com/gears.png height=50 width=50></div></td>\n");
sb.safePrintf("<td><font size=+1><b>The Transparent Search Engine</b></font><br>\n");
sb.brify2("Gigablast is the first truly transparent search engine. It tells you exactly why the search results are ranked the way they are. There is nothing left to the imagination.",85);
sb.safePrintf("<br><br>");
@ -165,9 +198,9 @@ bool printWebHomePage ( SafeBuf &sb , HttpRequest *r ) {
sb.safePrintf("\n");
sb.safePrintf("<tr valign=top>\n");
sb.safePrintf("<td><div style=width:50px;height:50px;display:inline-block;background-color:f2b629;></td>\n");
sb.safePrintf("<td align=center><center><img src=http://www.gigablast.com/dollargear.png height=50 width=50></center></div></center></td>\n");
sb.safePrintf("<td><font size=+1><b>The SEO Search Engine</b></font><br>\n");
sb.brify2("When it comes to search-engine based SEO, Gigablast is the place to be. With a frothy set of unique and effective <a href=/seo>SEO tools</a>, you will find all you need to execute a simple yet effective SEO strategy. Stop the guesswork, and let a search engine tell you how to SEO it.",85);
sb.brify2("When it comes to search-engine based SEO, Gigablast is the place to be. With a frothy set of unique and effective <a href=http://www.gigablast.com/seo>SEO tools</a>, you will find all you need to execute a simple yet effective SEO strategy. Stop the guesswork, and let a search engine tell you how to SEO it.",85);
sb.safePrintf("</td></tr>\n");
@ -368,19 +401,22 @@ bool printAddUrlHomePage ( SafeBuf &sb , char *url , HttpRequest *r ) {
// . when loaded with the main page for the first time it will
// immediately replace its content...
if ( url ) {
char *root = "";
if ( g_conf.m_isMattWells )
root = "http://www.gigablast.com";
sb.safePrintf("<br>"
"<br>"
"<div id=msgbox>"
//"<b>Injecting your url. Please wait...</b>"
"<center>"
"<img src=/gears.gif width=50 height=50>"
"<img src=%s/gears.gif width=50 height=50>"
"</center>"
"<script type=text/javascript>"
//"alert('shit');"
"var client = new XMLHttpRequest();\n"
"client.onreadystatechange = handler;\n"
"var url='/addurl?u="
);
, root );
sb.urlEncode ( url );
// propagate "admin" if set
//long admin = hr->getLong("admin",-1);
@ -463,11 +499,17 @@ bool printDirHomePage ( SafeBuf &sb , HttpRequest *r ) {
sb.safePrintf("<br><br>\n");
sb.safePrintf("<br><br><br>\n");
sb.safePrintf("<a href=/>web</a> &nbsp;&nbsp;&nbsp;&nbsp; <a href=/seo>seo</a> &nbsp;&nbsp;&nbsp;&nbsp; <b>directory</b> &nbsp;&nbsp;&nbsp;&nbsp; \n");
sb.safePrintf("<a href=/>web</a> &nbsp;&nbsp;&nbsp;&nbsp; <a href=http://www.gigablast.com/seo>seo</a> &nbsp;&nbsp;&nbsp;&nbsp; <b>directory</b> &nbsp;&nbsp;&nbsp;&nbsp; \n");
sb.safePrintf("<a href=http://www.gigablast.com/events>events</a>"
" &nbsp;&nbsp;&nbsp;&nbsp; \n");
sb.safePrintf("<a href=/adv.html>advanced search</a>");
sb.safePrintf(" &nbsp;&nbsp;&nbsp;&nbsp; ");
sb.safePrintf("<a href=/addurl title=\"Instantly add your url to "
"Gigablast's index\">add url</a>");
char *root = "";
if ( g_conf.m_isMattWells )
root = "http://www.gigablast.com";
sb.safePrintf("<a href=%s/addurl title=\"Instantly add your url to "
"Gigablast's index\">add url</a>"
, root );
sb.safePrintf("\n");
sb.safePrintf("<br><br>\n");
// submit to HTTPS now
@ -1589,7 +1631,7 @@ void doneInjectingWrapper3 ( void *st ) {
rand32);
sb.urlEncode(url);
sb.safePrintf(">Check it</a> or "
"<a href=/seo?u=");
"<a href=http://www.gigablast.com/seo?u=");
sb.urlEncode(url);
sb.safePrintf(">SEO it</a>"
".</b>");

64
Proxy.cpp
View File

@ -58,6 +58,7 @@ struct StateControl{
HttpRequest m_hr;
Host *m_forwardHost;
float m_pending;
bool m_isEventGuru;
};
#define UIF_ADMIN 0x01
@ -368,6 +369,8 @@ bool Proxy::handleRequest (TcpSocket *s){
char *host = hr.getHost();
char *hdom = host;
if ( strncasecmp(hdom,"www.",4) == 0 ) hdom += 4;
if ( strncasecmp(hdom,"www2.",5) == 0 ) hdom += 5;
if ( strncasecmp(hdom,"www1.",5) == 0 ) hdom += 5;
// auto redirect eventguru.com to www.eventguru.com so cookies
// are consistent
if ( ! redir &&
@ -385,9 +388,19 @@ bool Proxy::handleRequest (TcpSocket *s){
redirLen = gbstrlen(redir);
}
bool isEventGuru = false;
if ( strcasecmp(hdom,"eventguru.com") == 0 )
isEventGuru = true;
#ifdef MATTWELLS
#define HTTPS_REDIR 1
#endif
if ( redirLen > 0 && redir ) {
//redirect:
#ifdef HTTPS_REDIR
redirect:
#endif
HttpMime m;
m.makeRedirMime (redir,redirLen);
// . move the reply to a send buffer
@ -429,6 +442,10 @@ bool Proxy::handleRequest (TcpSocket *s){
char *path = hr.getPath();
//long pathLen = hr.getPathLen();
// serve events on the gigablast.com domain:
if ( path && strncmp(path,"/events",7) == 0 )
isEventGuru = true;
/*
bool badPage = false;
if ( n < 0 ) badPage = true;
@ -500,6 +517,32 @@ bool Proxy::handleRequest (TcpSocket *s){
if ( ! strncmp(path,"/?id=" ,5 ) ) handleIt = false;
// log the request iff filename does not end in .gif .jpg .
char *f = NULL;
long flen = 0;
if ( isEventGuru ) {
f = hr.getFilename();
flen = hr.getFilenameLen();
}
// proxy will handle eventguru images i guess
bool isGif = ( f && flen >= 4 && strncmp(&f[flen-4],".gif",4) == 0 );
bool isJpg = ( f && flen >= 4 && strncmp(&f[flen-4],".jpg",4) == 0 );
bool isBmp = ( f && flen >= 4 && strncmp(&f[flen-4],".bmp",4) == 0 );
bool isPng = ( f && flen >= 4 && strncmp(&f[flen-4],".png",4) == 0 );
bool isIco = ( f && flen >= 4 && strncmp(&f[flen-4],".ico",4) == 0 );
bool isPic = (isGif | isJpg | isBmp | isPng || isIco);
// use event guru favicon?
//if ( isEventGuru && isIco && strcmp(f,"favicon.ico") == 0 ) {
// f = "eventguru_favicon.ico";
// flen = gbstrlen(f);
//}
// eventguru.com host: in mime?
if ( isEventGuru && ! isPic )
handleIt = false;
// only proxy holds the accounting info
if ( ! strncmp ( path ,"/account", 8 ) ) {
printRequest(s, &hr);
@ -513,12 +556,14 @@ bool Proxy::handleRequest (TcpSocket *s){
if ( tcp == &g_httpServer.m_ssltcp ) max = g_conf.m_httpsMaxSockets;
else max = g_conf.m_httpMaxSockets;
#ifdef _HTTPS_REDIR_
#ifdef HTTPS_REDIR
// if hitting root page then tell them to go to https
// if not autobanned... but if it is an autobanned request on root
// page it should have go the turing test above!
if ( n == PAGE_ROOT &&
! g_isYippy &&
// not event guru homepage
! isEventGuru &&
// if not already on https
tcp != &g_httpServer.m_ssltcp &&
// do not redirect http://www.gigablast.com/?c=dmoz3 (directory)!
@ -1263,6 +1308,8 @@ bool Proxy::forwardRequest ( StateControl *stC ) {
p[5] = '9';
break;
}
// code is invalid if is not for an old client
//if ( userId32b == 0 ) code = NULL;
}
@ -1663,7 +1710,7 @@ void Proxy::gotReplyPage ( void *state, UdpSlot *slot ) {
// do not print login bars in the xml!! do not print for ixquick
// which gets results in html...
if ( ! stC->m_raw && ! stC->m_ch )
if ( ! stC->m_raw && ! stC->m_ch && ! stC->m_isEventGuru )
newReply = storeLoginBar ( reply ,
size , // transmit size
size , // allocsize
@ -5151,11 +5198,16 @@ void Proxy::printUsers ( SafeBuf *sb ) {
// but if admin we should still have set our cookie
// adminsessid to our current session id so we know we are
// also the admin!
sb->safePrintf("<td><a href=/account?login=%s&password=%s>"
"%s</td>"
sb->safePrintf("<td><nobr>%li. "
"<a href=/account?login=%s&password=%s>"
"%s</a></nobr></td>"
,i
,ui->m_login
,ui->m_password
,ui->m_login);
,ui->m_login
//,ui->m_userId32
);
}
sb->safePrintf("</tr>\n");
sb->safePrintf("</table>\n");
}

41
TcpServer.cpp
View File

@ -1092,7 +1092,7 @@ bool TcpServer::closeLeastUsed ( long maxIdleTime ) {
// . g_errno will be set by Loop if there was a kinda socket reset error
void readSocketWrapper ( int sd , void *state ) {
// debug msg
// log("........... TcpServer::readSocketWrapper\n");
//log("........... TcpServer::readSocketWrapper\n");
// extract our this ptr
TcpServer *THIS = (TcpServer *)state;
// get a TcpSocket from sd
@ -1239,8 +1239,13 @@ long TcpServer::readSocket ( TcpSocket *s ) {
// do the read
int n;
if (m_useSSL)
n = SSL_read ( s->m_ssl, s->m_readBuf + s->m_readOffset, avail );
if (m_useSSL) {
//long long now1 = gettimeofdayInMilliseconds();
n = SSL_read(s->m_ssl, s->m_readBuf + s->m_readOffset, avail );
//long long now2 = gettimeofdayInMilliseconds();
//long long took = now2 - now1 ;
//if ( took >= 2 ) log("tcp: ssl_read took %llims", took);
}
else
n = ::read ( s->m_sd, s->m_readBuf + s->m_readOffset, avail );
@ -1469,8 +1474,13 @@ long TcpServer::writeSocket ( TcpSocket *s ) {
// send this piece
int n;
retry10:
if (m_useSSL)
if (m_useSSL) {
//long long now1 = gettimeofdayInMilliseconds();
n = SSL_write ( s->m_ssl, msg + s->m_sendOffset, toSend );
//long long now2 = gettimeofdayInMilliseconds();
//long long took = now2 - now1 ;
//if ( took >= 2 ) log("tcp: ssl_write took %llims", took);
}
else
n = ::send ( s->m_sd , msg + s->m_sendOffset , toSend , 0 );
// cancel harmless errors, return -1 on severe ones
@ -1612,8 +1622,12 @@ connected:
int r;
s->m_ssl = SSL_new(m_ctx);
SSL_set_fd(s->m_ssl, s->m_sd);
//long long now1 = gettimeofdayInMilliseconds();
SSL_set_connect_state(s->m_ssl);
r = SSL_connect(s->m_ssl);
//long long now2 = gettimeofdayInMilliseconds();
//long long took = now2 - now1 ;
//if ( took >= 2 ) log("tcp: ssl_connect took %llims", took);
if (!s->m_ssl) {
log("ssl: SSL is NULL after connect.");
char *xx = NULL; *xx = 0;
@ -2073,9 +2087,19 @@ bool TcpServer::sslAccept ( TcpSocket *s ) {
}
//log("ssl: SSL_accept %li",newsd);
long long now1 = gettimeofdayInMilliseconds();
retry19:
// javier put this in here, but it was not non-blocking!!!
// . javier put this in here, but it was not non-blocking!!!
// . it is non-blocking now, however, when it does block and
// complete the accept it takes 10ms on sp1, a server from ~2009
// using a custom build of the lastest libssl.a from about 2013.
// . this accept needs to be put in a thread then, maybe multiple
// threads
int r = SSL_accept(s->m_ssl);
long long now2 = gettimeofdayInMilliseconds();
long long took = now2 - now1 ;
if ( took >= 2 )
log("tcp: ssl_accept %li took %llims", (long)newsd, took);
// did it block?
if ( r < 0 && errno == EINTR ) goto retry19;
// copy errno to g_errno
@ -2084,7 +2108,7 @@ bool TcpServer::sslAccept ( TcpSocket *s ) {
if ( g_errno == SSL_ERROR_WANT_READ ||
g_errno == SSL_ERROR_WANT_WRITE ||
g_errno == EAGAIN ) {
//log("ssl: SSL_accept blocked %li",newsd);
//log("ssl: SSL_accept would block %li",newsd);
return true;
}
// any other?
@ -2098,8 +2122,9 @@ bool TcpServer::sslAccept ( TcpSocket *s ) {
}
// log this so we can monitor if we get too many of these per second
// because they take like 10ms each on sp1!!! mdw
log("ssl: SSL_accept (~10ms) completed %li",newsd);
// because they take like 10ms each on sp1!!! (even with non-blocking
// sockets, they'll block for 10ms) - mdw 2013
//log("ssl: SSL_accept (~10ms) completed %li",newsd);
// ok, we got it
s->m_sockState = ST_READING;
return true;

4
Threads.cpp
View File

@ -301,6 +301,10 @@ bool Threads::init ( ) {
// generic multipurpose
if ( ! g_threads.registerType (GENERIC_THREAD,100/*maxThreads*/,100) )
return log("thread: Failed to register thread type." );
// for call SSL_accept() which blocks for 10ms even when socket
// is non-blocking...
//if (!g_threads.registerType (SSLACCEPT_THREAD,20/*maxThreads*/,100))
// return log("thread: Failed to register thread type." );
#ifndef _PTHREADS_

1
Threads.h
View File

@ -21,6 +21,7 @@ pid_t getpidtid();
#define SAVETREE_THREAD 4
#define UNLINK_THREAD 5
#define GENERIC_THREAD 6
//#define SSLACCEPT_THREAD 7
#define GB_SIGRTMIN (SIGRTMIN+4)
#define MAX_NICENESS 2
// . a ThreadQueue has a list of thread entries

BIN
html/dollargear.png Normal file
View File

Binary file not shown.
Side by Side

After

(image error) Size: 9.9 KiB

BIN
html/eventguru.png Normal file
View File

Binary file not shown.
Side by Side

After

(image error) Size: 39 KiB

BIN
html/gears.png Normal file
View File

Binary file not shown.
Side by Side

After

(image error) Size: 9.6 KiB

12
html/robots.txt
View File

@ -1,18 +1,18 @@
User-Agent: googlebot
Disallow: /search
Disallow: /search?
User-Agent: bingbot
Disallow: /search
Disallow: /search?
User-Agent: msnbot
Disallow: /search
Disallow: /search?
User-Agent: slurp
Disallow: /search
Disallow: /search?
User-Agent: gigabot
Disallow: /search
Disallow: /search?
User-Agent: *
Disallow: /search
Disallow: /search?