add sendEmailThroughMandrill() to send

through mail chimp http api.

HttpServer.cpp

@@ -129,7 +129,8 @@ bool HttpServer::getDoc ( char   *url      ,
 			  char    *proto ,
 			  bool     doPost ,
 			  char    *cookie ,
-			  char    *additionalHeader ) { 
+			  char    *additionalHeader ,
+			  char    *fullRequest ) { 
 	// sanity
 	if ( ip == -1 ) 
 		log("http: you probably didn't mean to set ip=-1 did you? "
@@ -152,24 +153,37 @@ bool HttpServer::getDoc ( char   *url      ,
 		tcp = &m_ssltcp;
 		defPort = 443;
 	}
-	// this returns false and sets g_errno on error
-	if ( ! r.set ( url , offset , size , ifModifiedSince ,
-		       userAgent , proto , doPost , cookie ,
-		       additionalHeader ) ) return true;
 
-	if ( g_conf.m_logDebugSpider )
-		log("spider: httprequest = %s", r.getRequest());
+	char *req = NULL;
+	long reqSize;
+
+	// this returns false and sets g_errno on error
+	if ( ! fullRequest ) {
+		if ( ! r.set ( url , offset , size , ifModifiedSince ,
+			       userAgent , proto , doPost , cookie ,
+			       additionalHeader ) ) return true;
+		reqSize = r.getRequestLen();
+		req = (char *) mdup ( r.getRequest() , reqSize,"HttpServer");
+	}
+	else {
+		// does not contain \0 i guess
+		reqSize = gbstrlen(fullRequest);
+		req = (char *) mdup ( fullRequest , reqSize,"HttpServer");
+	}
+
+	// . get the request from the static buffer and dup it
+	// . return true and set g_errno on error
+	if ( ! req ) return true;
 
 	long  hostLen ;
 	long  port = defPort;
 	char *host = getHostFast ( url , &hostLen , &port );
 	
 
-	// . get the request from the static buffer and dup it
-	// . return true and set g_errno on error
-	long  reqSize = r.getRequestLen();
-	char *req     = (char *) mdup ( r.getRequest() , reqSize,"HttpServer");
-	if ( ! req ) return true;
+	if ( g_conf.m_logDebugSpider )
+		log("spider: httprequest = %s", req );
+
+
 	// do we have an ip to send to? assume not
 	if ( proxyIp ) { ip = proxyIp ; port = proxyPort; }
 	// special NULL case

HttpServer.h

@@ -96,7 +96,9 @@ class HttpServer {
 		      char   *proto = "HTTP/1.0" ,
 		      bool    doPost = false ,
 		      char   *cookie = NULL ,
-		      char *additionalHeader = NULL ); // does not include \r\n
+		      char *additionalHeader = NULL , // does not include \r\n
+		      // specify your own mime and post data here...
+		      char *fullRequest = NULL );
 
 	bool getDoc ( long ip,
 		      long port,

PingServer.cpp

@@ -2879,3 +2879,109 @@ bool gotMxIp ( EmailInfo *ei ) {
 	return true;
 }
 
+
+static void gotMandrillReplyWrapper ( void *state , TcpSocket *s ) {
+	EmailInfo *ei = (EmailInfo *)state;
+	ei->m_callback ( ei->m_state );
+}
+
+
+// mailchimp http mail api
+bool sendEmailThroughMandrill ( class EmailInfo *ei ) {
+
+	// this is often set from XmlDoc.cpp::indexDoc()
+	g_errno = 0;
+
+	SafeBuf sb;
+
+	// then the message to send
+	sb.safePrintf(
+		  "POST /api/1.0/messages/send-template.json"
+		  " HTTP/1.0\r\n"
+		  "Accept: image/gif, image/x-xbitmap, image/jpeg, "
+		  "image/pjpeg, application/x-shockwave-flash, "
+		  "application/msword, */*\r\n"
+		  "Accept-Language: en-us\r\n"
+		  "Content-Type: application/x-www-form-urlencoded\r\n"
+		  "Accept-Encoding: gzip, deflate\r\n"
+		  "User-Agent: Mozilla/4.0 "
+		  "(compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\r\n"
+		  "Host: mandrillapp.com\r\n" // www.t-mobile.com
+		  "Content-Length: xxx\r\n"
+		  //"Connection: Keep-Alive\r\n"
+		  "Connection: close\r\n"
+		  "Cookie: \r\n"
+		  "Cache-Control: no-cache\r\n\r\n"
+		  );
+	//
+	// post data
+	//
+	char *to = ei->m_toAddress.getBufStart();
+	char *from = ei->m_fromAddress.getBufStart();
+
+	SafeBuf ub;
+	sb.safePrintf( "{\"key\":\"GhWT0UpcVBl7kmumrt9dqg\","
+		       "\"template_name\":\"crawl-finished\","
+		       "\"template_content\": [],"
+		       "\"message\": {"
+		       "\"to\": ["
+		       "{"
+		       "\"email\":\"%s\""
+		       "}"
+		       "],"
+
+		       "\"from_email\":\"%s\","
+		       "\"headers\": {"
+		       "\"Reply-To\":\"%s\""
+		       "},"
+		       "\"bcc_address\":\"%s\","
+		       "\"global_merge_vars\":["
+		       "{"
+		       "\"name\":\"CRAWLNAME\","
+		       "\"content\":\"%s\""
+		       "}"
+		       "]"
+		       "}"
+		       "}"
+		       , to
+		       , from
+		       , from
+		       , from
+		       , ei->m_cr->m_coll
+		       );
+	ub.urlEncode();
+	// append the post data to the full request
+	sb.safeMemcpy ( &ub );
+	// make sure ends in \0
+	sb.nullTerm();
+
+	// gotta get the cookie
+	char *uu = "https://mandrillapp.com/";
+	if ( ! g_httpServer.getDoc ( uu,
+				     0, // ip
+				     0                 , // offset
+				     -1                , // size
+				     false             , // m_ifModifiedSince
+				     ei              , // state
+				     gotMandrillReplyWrapper    , // 
+				     60*1000           , // timeout
+				     0                 , // m_proxyIp
+				     0                 , // m_proxyPort
+				     100*1024          , // m_maxTextDocLen
+				     100*1024          , // m_maxOtherDocLen  
+				     NULL,     // user agent
+				     "HTTP/1.0" , //proto
+				     true, // post?
+				     NULL, // cookie
+				     NULL, // additional header
+				     sb.getBufStart() ) ) // full requesst
+		return false;
+	// must have been an error
+	log("net: Got error getting page from mandrill: %s.",
+	    mstrerror(g_errno));
+	// ignore it
+	g_errno = 0;
+	// always call this at the end
+	return true;
+}
+	

PingServer.h

@@ -135,5 +135,8 @@ extern class PingServer g_pingServer;
 // . use this for sending generic emails
 bool sendEmail ( class EmailInfo *ei ) ;
 
+// use mailchimp's mandrill email http api
+bool sendEmailThroughMandrill ( class EmailInfo *ei ) ;
+
 #endif
 

XmlDoc.cpp

@@ -42043,16 +42043,19 @@ bool XmlDoc::sendNotification ( ) {
 		ei->m_toAddress.safeStrcpy ( email );
 		ei->m_toAddress.nullTerm();
 		ei->m_fromAddress.safePrintf("support@diffbot.com");
+		/*
 		ei->m_subject.safePrintf("crawl paused");
 		ei->m_body.safePrintf("Your crawl for collection \"%s\" "
 				      "has been paused because it hit "
 				      "a maxPagesToCrawl or maxPagesToProcess "
 				      "limitation."
 				      , m_cr->m_coll);
+		*/
+		ei->m_cr = m_cr;
 		ei->m_state = this;
 		ei->m_callback = doneSendingNotifyEmailWrapper;
 		// this will usually block, unless error maybe
-		if ( ! sendEmail ( ei ) )
+		if ( ! sendEmailThroughMandrill ( ei ) )
 			m_notifyBlocked++;
 	}
 

XmlDoc.h

@@ -91,6 +91,7 @@ public:
 	SafeBuf m_fromAddress;
 	SafeBuf m_subject;
 	SafeBuf m_body;
+	CollectionRec *m_cr;
 	char *m_dom; // ref into m_toAddress of the domain in email addr
 	SafeBuf m_mxDomain; // just the domain with a "gbmxrec-" prepended
 	void *m_state;