proxy2ch/BBS2chProxyConnection.cpp

#include <pthread.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <curl/curl.h>
#ifdef _WIN32
#include <fcntl.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <mswsock.h>
#define CLOSESOCKET(x) closesocket(x)
#else
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <arpa/inet.h>
#define CLOSESOCKET(x) close(x)
#endif
#include "BBS2chProxyConnection.h"
#include "DataStorage.h"
//#define DEBUG 1

extern char *proxy_server;
extern long proxy_port;
extern long proxy_type;
extern long timeout;
extern char *user_agent;
extern char *api_ua_dat;
extern char *x_2ch_ua_dat;
extern char *appKey;
extern int allow_chunked;
extern int curl_features;
extern unsigned int curl_version_number;
extern bool accept_https;
extern int force_5chnet;
extern int force_5chnet_https;
extern int force_ipv4;
extern char *bbsmenu_url;
extern char *api_server;
extern std::map<std::string, std::string> bbscgi_headers;
extern int gikofix;
extern CURLSH *curl_share;

extern void log_printf(int level, const char *format ...);

#include "utils.h"

void *BBS2chProxyConnection::launch(void *param)
{
	reinterpret_cast<BBS2chProxyConnection *>(param)->connect();
	return (void *)NULL;
}

void BBS2chProxyConnection::run(void)
{
	pthread_t thread;
	pthread_attr_t thread_attr;
	pthread_attr_init(&thread_attr);
	pthread_attr_setdetachstate(&thread_attr , PTHREAD_CREATE_DETACHED);
	if(0 != pthread_create(&thread , &thread_attr , &BBS2chProxyConnection::launch , this))
		perror("pthread_create");
	pthread_attr_destroy(&thread_attr);
}

void *BBS2chProxyConnection::tunnel_c2s(void *param)
{
	char buf[2048];
	fd_set fds;
	int sock_c = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_c;
	int sock_s = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_s;
#ifdef _WIN32
	struct timeval timeout;
	timeout.tv_sec = 0;
	timeout.tv_usec = 10000;
#endif
	
	while(1) {
#ifdef _WIN32
		FD_ZERO(&fds);
		FD_SET(sock_c,&fds);
		if(select(sock_c + 1,&fds,NULL,NULL,&timeout) < 0) break;
		if(FD_ISSET(sock_c,&fds)) {
#endif
			int ret = recv(sock_c, buf, 2048, 0);
			if(ret > 0) send(sock_s, buf, ret, 0);
			else if(ret <= 0) break;
#ifdef _WIN32
		}
#endif
	}
	//fprintf(stderr,"tunnel_c2s end\n");
	CLOSESOCKET(sock_s);
	return NULL;
}

void *BBS2chProxyConnection::tunnel_s2c(void *param)
{
	char buf[2048];
	fd_set fds;
	int sock_c = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_c;
	int sock_s = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_s;
	
	while(1) {
		int ret = recv(sock_s, buf, 2048, 0);
		if(ret > 0) send(sock_c, buf, ret, 0);
		else if(ret <= 0) break;
	}
	//fprintf(stderr,"tunnel_s2c end\n");
	CLOSESOCKET(sock_c);
	return NULL;
}

int BBS2chProxyConnection::tunnel(const char *addr, int port)
{
	struct sockaddr_in server;
	memset(&server, 0, sizeof(server));
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = inet_addr(addr);
	server.sin_port = htons(port);
	if(server.sin_addr.s_addr == 0xffffffff) {
		struct hostent *host;
		 host = gethostbyname(addr);
		if (host == NULL) {
			sendResponse(400, "Bad Request", fpw);
			return -1;
		}
		server.sin_addr.s_addr = *(unsigned int *)host->h_addr_list[0];
	 }
	
	log_printf(1,"Tunneling connection to %s:%d\n",addr,port);
	
	sock_s = socket(AF_INET, SOCK_STREAM, 0);
	if(-1 == ::connect(sock_s, (struct sockaddr *)&server, sizeof(server))) {
		perror("connect");
		sendResponse(400, "Bad Request", fpw);
		return -1;
	}
	send(sock_c, "HTTP/1.1 200 Connection established\r\n\r\n", 39, 0);
	
	pthread_t thread_c2s, thread_s2c;
	if(0 != pthread_create(&thread_c2s, NULL, &BBS2chProxyConnection::tunnel_c2s, this))
		perror("pthread_create");
	if(0 != pthread_create(&thread_s2c, NULL, &BBS2chProxyConnection::tunnel_s2c, this))
		perror("pthread_create");
	pthread_join(thread_c2s, NULL);
	pthread_join(thread_s2c, NULL);
	
	log_printf(1,"Tunneling connection to %s:%d finished\n",addr,port);
	
	return 0;
}

void BBS2chProxyConnection::connect(void)
{
	char method[32], url[1024], protocol[32];
	int i;
	char *buf, *ptr;
	bool html2dat = false;
	bool html2dat_kako = false;
	regmatch_t match[7];
	long statusCode = 0;
	
#ifdef _WIN32
	int sock_osfhandle = _open_osfhandle(sock_c, O_RDONLY);
	fpr = fdopen(sock_osfhandle, "rb");
 	fpw = fdopen(sock_osfhandle, "wb");
#else
	fpr = fdopen(sock_c, "rb");
	fpw = fdopen(sock_c, "wb");
#endif
	if(!fpr || !fpw) {
		log_printf(0, "Error: cannot open file descripter for client\n");
		goto end;
	}
	
	buf = (char *)malloc(16384);
	if(!buf) goto end;
	ptr = buf;
	
	if(!fgets(buf,1024,fpr)) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 32) method[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	method[i] = 0;
	ptr++;
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 1024) url[i++] = *ptr++;
	if(*ptr == 0 || i == 1024) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	url[i] = 0;
	ptr++;
	i=0;
	while(*ptr != '\r' && *ptr != '\n' && *ptr != 0 && i < 32) protocol[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	protocol[i] = 0;
	log_printf(1, "Received %s %s %s\n",method,url,protocol);
	
	if(strcasecmp(method,"GET") && strcasecmp(method,"POST") && strcasecmp(method,"HEAD") && strcasecmp(method,"CONNECT")) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	if(!url[0]) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	if(strncasecmp(protocol,"HTTP",4)) {
		sendResponse(400, "Bad Request", fpw);
		statusCode = 400;
		goto end;
	}
	
	if(!strcasecmp(method,"CONNECT")) {
		if(!accept_https) {
			sendResponse(400, "Bad Request", fpw);
			statusCode = 400;
			goto end;
		}
		while(fgets(buf,16384,fpr)) {
			if(!strcmp("\r\n",buf)) break;
		}
		int port = 443;
		char *ptr = strchr(url, ':');
		if(ptr) {
			*ptr = 0;
			port = atoi(ptr+1);
		}
		tunnel(url, port);
		goto end;
	}
	if(force_5chnet && !strncasecmp(url,"http://",7)) {
		char *ptr = url+8;
		char *end = ptr;
		while(*end != '/' && *end != 0) end++;
		ptr = strstr(ptr,".2ch.net");
		if(ptr && ptr < end && memcmp(ptr-4,"menu.",5)) {
			memcpy(ptr+1,"5ch",3);
			force5ch = true;
			log_printf(1, "Detected *.2ch.net URL, changed target URL to %s\n",url);
		}
	}
	
	if(regexec(&regex, url, 6, match, 0) != REG_NOMATCH) {
		if(appKey || strncasecmp(url+match[1].rm_so,"headline.",9)) html2dat = true;
	}
	else if(regexec(&regex_kako, url, 7, match, 0) != REG_NOMATCH) {
		html2dat_kako = true;
	}
	else if(regexec(&regex_offlaw, url, 5, match, 0) != REG_NOMATCH) {
		char *thread = strstr(url,"key=");
		if(thread) {
			match[6].rm_so = thread+4-url;
			match[6].rm_eo = thread+4-url;
			char *ptr = thread+4;
			while(*ptr != '&' && *ptr != 0) {
				ptr++;
				match[6].rm_eo++;
			}
			if(match[6].rm_so != match[6].rm_eo) html2dat_kako = true;
		}
	}
	
	if(html2dat || html2dat_kako) {
		char tmp[1024];
		regmatch_t *tid = html2dat ? match+5 : match+6;
		strcpy(tmp,url);
		tmp[match[2].rm_eo] = 0;
		tmp[match[4].rm_eo] = 0;
		tmp[tid->rm_eo] = 0;
		if(!appKey || html2dat_kako) {
			log_printf(1, "Retrieving thread via read.cgi...\n");
			sprintf(url,"%s/%s/%s",tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			threadKey = std::string(url);
			sprintf(url,"%s://%s/test/read.cgi/%s/%s/",force_5chnet_https?"https":"http",tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			statusCode = datProxy(url, method);
		}
		else {
			log_printf(1, "Retrieving thread via API...\n");
			tmp[match[1].rm_eo] = 0;
			sprintf(url,"https://%s/v1/%s/%s/%s",api_server,tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			statusCode = datProxyAPI(url, method);
		}
	}
	else {
		char urlMod[1024];
		strcpy(urlMod,url);
		if(force_5chnet_https) {
			const char *host = "5ch.net";
			char *ptr = strstr(url,host);
			if(!ptr) {
				host = "2ch.net";
				ptr = strstr(url,host);
			}
			if(!ptr) {
				host = "bbspink.com";
				ptr = strstr(url,host);
			}
			if(ptr) {
				char *start = url+7;
				char *end = strchr(start,'/');
				int hostLength = strlen(host);
				if(!end) end = url+strlen(url);
				if(ptr >= start && ptr < end && (*(ptr-1) == '.' || *(ptr-1) == '/') && (*(ptr+hostLength) == '/' || *(ptr+hostLength) == ':' || *(ptr+hostLength) == 0)) {
					log_printf(1, "This is %s URL, connecting with HTTPS\n",host);
					if(*(ptr+hostLength) == ':') {
						*(ptr+hostLength) = 0;
						sprintf(urlMod,"https://%s%s",start,end);
						*(ptr+hostLength) = ':';
					}
					else sprintf(urlMod,"https://%s",start);
					log_printf(2, "URL is: %s\n",urlMod);
				}
			}
		}
		
		if(bbsmenu_url && !strcmp(url, bbsmenu_url)) {
			log_printf(1, "Running as a BBS menu proxy...\n");
			statusCode = bbsmenuProxy(urlMod, method);
		}
		else {
			bool bbscgi = false;
			bool isPostRequest = !strcasecmp(method, "POST");
			if(isPostRequest && (strstr(urlMod,".5ch.net") || strstr(urlMod,".bbspink.com")) && strstr(urlMod,"/test/bbs.cgi")) bbscgi = true;
			log_printf(1, "Not a thread request, passthrough...\n");
			struct curl_slist *headers = NULL;
			bool hasExpect = false;
			char *postdata = NULL;
			std::string hostStr;
			bool isFormURLEncoded = false;
			while(fgets(buf,16384,fpr)) {
				if(bbscgi) {
					char *ptr = strchr(buf, ':');
					if(ptr) {
						char *header = (char *)malloc(ptr-buf+1);
						memcpy(header,buf,ptr-buf);
						header[ptr-buf] = 0;
						if(bbscgi_headers.find(header) != bbscgi_headers.end()) {
							log_printf(1, "Ignoring header \"%s\" because alternative value exists\n", header);
							free(header);
							continue;
						}
						free(header);
					}
				}
				//fprintf(fpw,"%s",buf);
				//fprintf(stderr,"%s",buf);
				if(!strcmp("\r\n",buf)) break;
				if(!strncasecmp("Connection:",buf,11)) continue;
				else if(!strncasecmp("Host:",buf,5)) {
					char *ptr;
					ptr = strchr(buf, '\r');
					if(!ptr) ptr = strchr(buf, '\n');
					if(!ptr) continue;
					*ptr = 0;
					ptr = strchr(buf+5, ':');
					if(ptr) *ptr = 0;
					ptr = buf+5;
					while(*ptr == ' ') ptr++;
					if(force_5chnet) {
						char *ptr2 = strstr(ptr, ".2ch.net");
						if(ptr2) {
							*(ptr2+1) = '5';
						}
					}
					hostStr = std::string(ptr);
					continue;
				}
				//else if(!strncasecmp("Authorization:",buf,14)) continue;
				//else if(!strncasecmp("WWW-Authenticate:",buf,17)) continue;
				else if(user_agent && !strncasecmp("User-Agent:",buf,11)) continue;
				else if(bbscgi && !strncasecmp("Content-Length:",buf,15)) {
					content_length = atoi(buf+16);
					continue;
				}
				else {
					char *ptr;
					ptr = strchr(buf, '\r');
					if(!ptr) ptr = strchr(buf, '\n');
					if(!ptr) continue;
					*ptr = 0;
					if(bbscgi && force_5chnet && !strncasecmp("Referer:",buf,8)) {
						char *ptr2 = strstr(buf+8, ".2ch.net");
						if(ptr2) {
							*(ptr2+1) = '5';
						}
					}
					headers = curl_slist_append(headers, buf);
					if(!strncasecmp("Content-Length:",buf,15)) {
						content_length = atoi(buf+16);
					}
					else if(!strncasecmp("Expect:",buf,7)) {
						hasExpect = true;
					}
					else if(!strncasecmp("Content-Type:",buf,13)) {
						ptr = buf + 13;
						while(*ptr == ' ') ptr++;
						if(!strcasecmp("application/x-www-form-urlencoded", ptr)) {
							isFormURLEncoded = true;
						}
					}
				}
			}
			if(!hasExpect) headers = curl_slist_append(headers, "Expect:");
			if(bbscgi && isFormURLEncoded && content_length) {
				postdata = (char *)calloc(content_length+1, 1);
				content_length = fread(postdata,1,content_length,fpr);
				if(gikofix && content_length > 0) {
					char *ptr = postdata+content_length-1;
					while(*ptr == '\r' || *ptr == '\n') {
						*ptr-- = 0;
					}
				}
			}
			if(bbscgi && !bbscgi_headers.empty()) {
				std::map<std::string, std::string> fields;
				if(postdata) {
					const char *ptr = postdata;
					while(1) {
						const char *tmp = ptr;
						while(*tmp != '=' && *tmp != 0) tmp++;
						if(*tmp == 0) break;
						std::string key(ptr, tmp-ptr);
						tmp++;
						ptr = tmp;
						while(*tmp != '&' && *tmp != 0) tmp++;
						std::string value(ptr, tmp-ptr);
						fields.insert(std::make_pair(key, value));
						if(*tmp == 0) break;
						ptr = tmp + 1;
					}
				}
				std::string &boardStr = fields["bbs"];
				std::string &threadStr = fields["key"];
				for(std::map<std::string, std::string>::iterator it = bbscgi_headers.begin(); it!=bbscgi_headers.end(); it++) {
					/* we cannot use a reference here, because the original string shouldn't be replaced */
					std::string value = it->second;
					if(!hostStr.empty()) {
						std::string::size_type pos = value.find("%HOST%");
						while(pos != std::string::npos) {
							value.replace(pos, 6, hostStr);
							pos = value.find("%HOST%", pos+hostStr.length());
						}
					}
					if(!boardStr.empty()) {
						std::string::size_type pos = value.find("%BOARD%");
						while(pos != std::string::npos) {
							value.replace(pos, 7, boardStr);
							pos = value.find("%BOARD%", pos+boardStr.length());
						}
					}
					if(!threadStr.empty()) {
						std::string::size_type pos = value.find("%THREAD%");
						while(pos != std::string::npos) {
							value.replace(pos, 8, threadStr);
							pos = value.find("%THREAD%", pos+threadStr.length());
						}
					}
					sprintf(buf,"%s: %s",it->first.c_str(),value.c_str());
					headers = curl_slist_append(headers, buf);
					log_printf(1, "Appended custom header \"%s\"\n", buf);
				}
			}
			
			CURL *curl = curl_easy_init();
			if(curl) {
				CURLcode res;
				if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
				curl_easy_setopt(curl, CURLOPT_URL, urlMod);
				curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
				curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
				if(postdata) {
					curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postdata);
				} else {
					if(content_length && isPostRequest) {
						/* set Content-Length explicitly via API to work properly with curl >= 7.66.0 */
						curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, content_length);
					}
					curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback_proxy);
					curl_easy_setopt(curl, CURLOPT_READDATA, this);
				}
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
				//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
				if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
				if(!strncasecmp(protocol,"HTTP/1.0",8) && !strncasecmp(urlMod, "http://", 7)) {
					curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
				}
				else {
					/* force use HTTP 1.1 because CURL_HTTP_VERSION_2TLS is used on curl (w/ nghttp2) >= 7.62.0 */
					curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
				}
				curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
				if(user_agent) {
					curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
				}
				if(isPostRequest) {
					curl_easy_setopt(curl, CURLOPT_POST, 1L);
				}
				else if(!strcasecmp(method, "HEAD")) {
					curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
				}
				if(proxy_server) {
					curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
					curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
					curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
				}
				res = curl_easy_perform(curl);
				if(res != CURLE_OK) {
					log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),urlMod);
					if(!status) sendResponse(503, "Service Unavailable", fpw);
					statusCode = 503;
				}
				else {
					if(chunked) {
						fprintf(fpw,"0\r\n\r\n");
					}
					curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
				}
				curl_easy_cleanup(curl);
				fflush(fpw);
			}
			curl_slist_free_all(headers);
			if(postdata) free(postdata);
		}
	}
	
  end:
	if(statusCode) log_printf(1, "Returned status code %d to client\n",statusCode);
	if(buf) free(buf);
	if(fpr) fclose(fpr);
	if(fpw) fclose(fpw);
	close(sock_c);
	delete this;
}

int BBS2chProxyConnection::datProxy(const char *url, const char *method)
{
	DataStorage *html = NULL;
	long statusCode = 0;
	long rangeStart = 0, rangeEnd = 0;
	time_t lastModified = 0;
	time_t ifModifiedSince = 0;
	char *buf = (char *)malloc(16384);
	char userAgentFromHeader[1024] = "";
	if(!buf) goto last;
	
	while(fgets(buf,16384,fpr)) {
		//fprintf(stderr,"%s",buf);
		if(!strcmp("\r\n",buf)) break;
		if(!strncasecmp("Range:",buf,6)) {
			if(strstr(buf+7,"bytes=") && !strchr(buf+7, ',')) {
				char *ptr = buf+13;
				if(*ptr == '-') {
					rangeStart = atoi(ptr);
				}
				else {
					rangeStart = strtol(ptr, &ptr, 10);
					if(*ptr == '-') ptr++;
					if(*ptr && *ptr != '\r') {
						rangeEnd = strtol(ptr, NULL, 10);
						if(rangeEnd && rangeStart > rangeEnd) {
							sendResponse(416, "Requested range not satisfiable", fpw);
							statusCode = 416;
							goto last;
						}
					}
				}
				//fprintf(stderr, "range=%ld-%ld\n",rangeStart,rangeEnd);
			}
			else {
				sendResponse(416, "Requested range not satisfiable", fpw);
				statusCode = 416;
				goto last;
			}
		}
		else if(!strncasecmp("If-Modified-Since:",buf,18)) {
			struct tm time_ = {};
			strptime(buf+19,httpTimestampFmt,&time_);
			ifModifiedSince = mktime(&time_);
		}
		else if(!strncasecmp("User-Agent:",buf,11)) {
			int i=0;
			char *ptr = buf+12;
			while(*ptr != '\r' && *ptr != '\n' && i < 1023) userAgentFromHeader[i++] = *ptr++;
			userAgentFromHeader[i] = 0;
		}
	}
	if(rangeStart > 0) {
		PBBS2chProxyThreadInfo info;
		pthread_mutex_lock(mutex);
		BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
		if(it != threadCache->end()) {
			info = it->second;
		}
		pthread_mutex_unlock(mutex);
		log_printf(5,"range request from %ld bytes\n",rangeStart);
		if(info) {
			int from = info->lastResNum;
			int alreadyRead = info->cachedSize;
			int lastResLength = info->cachedData->length;
			log_printf(5,"hit %s: cached %d bytes, last res size %d\n",threadKey.c_str(),alreadyRead,lastResLength);
			if(rangeStart <= alreadyRead && rangeStart >= alreadyRead - lastResLength) {
				CURL *curl = curl_easy_init();
				if(curl) {
					CURLcode res;
					DataStorage *dat = new DataStorage();
					log_printf(5,"partial access from res num %d\n",from);
					sprintf(buf,"%s%d-n",url,from);
					if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
					curl_easy_setopt(curl, CURLOPT_URL, buf);
					curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
					curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
					curl_easy_setopt(curl, CURLOPT_ENCODING, "");
					curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
					curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
					curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
					if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
					if(proxy_server) {
						curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
						curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
						curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
					}
					if(user_agent) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
					}
					else if(userAgentFromHeader[0]) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
					}
					res = curl_easy_perform(curl);
					if(res == CURLE_OK) {
						curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
						curl_easy_cleanup(curl);
						if(statusCode == 200 && dat->length) {
							DataStorage *updated = html2dat(dat, from, &lastModified, true);
							if(ifModifiedSince && lastModified && updated && updated->length == lastResLength) {
								struct tm time_ = {};
								gmtime_r(&lastModified,&time_);
								time_t tmp = mktime(&time_);
								if(ifModifiedSince >= tmp) {
									sendResponse(304, "Not Modified", fpw);
									log_printf(5,"not modified!\n");
									delete updated;
									delete dat;
									statusCode = 304;
									goto last;
								}
							}
							if(updated && updated->length && updated->length >= lastResLength) {
								html = new DataStorage(alreadyRead - lastResLength);
								html->appendBytes(updated->bytes, updated->length);
								if(!rangeEnd) rangeEnd = html->length - 1;
								if(rangeStart > rangeEnd) {
									sendResponse(416, "Requested range not satisfiable", fpw);
									delete updated;
									delete dat;
									statusCode = 416;
									goto last;
								}
								statusCode = 206;
								log_printf(5,"cache hit; reconstructed data length:%ld\n",(long)html->length);
							}
							else {
								log_printf(5,"cache misshit?\n");
								sendResponse(416, "Requested range not satisfiable", fpw);
								delete updated;
								delete dat;
								statusCode = 416;
								goto last;
							}
							delete updated;
						}
					}
					else {
						log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),buf);
						curl_easy_cleanup(curl);
					}
					delete dat;
					if(html) goto resp;
				}
			}
			else {
				log_printf(5,"invalid cache contents\n");
				pthread_mutex_lock(mutex);
				BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
				if(it != threadCache->end()) {
					threadCache->erase(it);
				}
				pthread_mutex_unlock(mutex);
			}
		}
	}
	
	{
		CURL *curl = curl_easy_init();
		if(curl) {
			CURLcode res;
			DataStorage *dat = new DataStorage();
			if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
			curl_easy_setopt(curl, CURLOPT_URL, url);
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			curl_easy_setopt(curl, CURLOPT_ENCODING, "");
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			if(proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			if(user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			else if(userAgentFromHeader[0]) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
			}
			res = curl_easy_perform(curl);
			if(res != CURLE_OK) {
				log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),url);
				sendResponse(503, "Service Unavailable", fpw);
				curl_easy_cleanup(curl);
				delete dat;
				statusCode = 503;
				goto last;
			}
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			curl_easy_cleanup(curl);
			
			if(statusCode == 200) {
				html = html2dat(dat, 1, &lastModified, false);
			}
			delete dat;
		}
		
		if(!html || !html->length) {
			sendResponse(503, "Service Unavailable", fpw);
			statusCode = 503;
			goto last;
		}
		if((rangeStart || rangeEnd) && html && html->length) {
			if(!rangeEnd) rangeEnd = html->length - 1;
			if(rangeStart < 0) rangeStart = html->length + rangeStart;
			if(rangeStart < html->length && rangeEnd < html->length && rangeStart <= rangeEnd) {
				statusCode = 206;
			}
			else {
				if(ifModifiedSince && lastModified && rangeStart == html->length) {
					struct tm time_ = {};
					gmtime_r(&lastModified,&time_);
					time_t tmp = mktime(&time_);
					if(ifModifiedSince >= tmp) {
						sendResponse(304, "Not Modified", fpw);
						log_printf(5,"not modified!\n");
						statusCode = 304;
						goto last;
					}
				}
				sendResponse(416, "Requested range not satisfiable", fpw);
				statusCode = 416;
				goto last;
			}
		}
	}

resp:
	if(statusCode == 206) sendBasicHeaders(statusCode,"Partial Content",fpw);
	else sendBasicHeaders(statusCode,"OK",fpw);
	if(0 > fprintf(fpw,"Content-Type: text/plain\r\n")) goto last;
	if(0 > fprintf(fpw,"Accept-Ranges: bytes\r\n")) goto last;
	if(statusCode == 206) {
		if(0 > fprintf(fpw,"Content-Range: bytes %ld-%ld/%ld\r\n",rangeStart,rangeEnd,(long)html->length)) goto last;
		//fprintf(stderr,"Content-Length: %ld\r\n",rangeEnd - rangeStart + 1);
		//fprintf(stderr,"Content-Range: bytes %ld-%ld/%ld\r\n",rangeStart,rangeEnd,(long)html->length);
		DataStorage *newHtml = new DataStorage();
		newHtml->appendBytes(html->bytes+rangeStart, rangeEnd - rangeStart + 1);
		delete html;
		html = newHtml;
	}
	if(0 > fprintf(fpw,"Content-Length: %ld\r\n",(long)html->length)) goto last;
	if(lastModified) {
		struct tm time_ = {};
		char date[256];
		gmtime_r(&lastModified,&time_);
		strftime(date,256,httpTimestampFmt,&time_);
		if(0 > fprintf(fpw,"Last-Modified: %s\r\n",date)) goto last;
		//fprintf(stderr,"Last-Modified: %s\r\n",date);
	}
	if(0 > fprintf(fpw,"\r\n")) goto last;
	
	if(html && statusCode >= 200 && statusCode < 300 && strcasecmp(method, "HEAD")) {
		if(html->length > fwrite(html->bytes,1,html->length,fpw)) goto last;
	}
	fflush(fpw);
	
last:
	if(buf) free(buf);
	if(html) delete html;
	return statusCode;
}

DataStorage *BBS2chProxyConnection::html2dat_old(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[32];
	char title[1024];
	int cachedSize = 0;
	bool bbspink = strstr(threadKey.c_str(),"bbspink.com") ? true : false;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<title>", 7);
	if(!ptr) {
		delete txt;
		return NULL;
	}
	ptr += 7;
	while(1) {
		if(*ptr == '<') {
			if(!strncasecmp(ptr,"</title>",8)) {
				ptr += 8;
				break;
			}
			else title[i++] = *ptr++;
		}
		else title[i++] = *ptr++;
	}
	title[i] = 0;
	
	sprintf(signature,"<dt>%d ",res);
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		ptr = strstr(ptr,signature);
		ptr += strlen(signature);
		while(*ptr != '<') ptr++;
		ptr++;
		const char *endStr;
		if(*ptr == 'a' || *ptr == 'A') {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			endStr = "</a>";
		}
		else if(*ptr == 'b') {
			endStr = NULL;
		}
		else {
			endStr = "</font>";
		}
		
		if(endStr) {
			ptr = strstr(ptr,"<b>");
			ptr += 3;
		}
		else {
			ptr = strchr(ptr,'>');
			ptr++;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"</b>",4) && (!endStr || !strncasecmp(ptr+4,endStr,strlen(endStr)))) {
					ptr += 4;
					if(endStr) ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr += 2;
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<dd>",4)) {
					ptr += 4;
					break;
				}
				else if(!strncmp(ptr,"<a href=\"javascript:be(",23)) {
					memcpy(date+i,"BE:",3);
					ptr += 23;
					i += 3;
					while(*ptr != ')') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<br><br>\n",9)) {
					ptr += 9;
					break;
				}
				else if(!strncasecmp(ptr,"<dt>",4) || !strncasecmp(ptr,"</dl>",5)) {
					while(i>0 &&body[i-1] == '\n') i--;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"http",13)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(body+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10) || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!bbspink && !strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else if(!bbspink && *ptr == ' ') {
				if(*(ptr+1) == ' ') ptr++;
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		sprintf(signature,"<dt>%d ",res);
		if(!memmem_priv(ptr, end-ptr+1, signature, strlen(signature))) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				sprintf(formattedDate,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

DataStorage *BBS2chProxyConnection::html2dat(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[64];
	char title[1024];
	int cachedSize = 0;
	char signatureTag[32];
	char closeTag[32];
	int closeTagLen;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<h1 class=\"title\">", 18);
	if(!ptr) {
		delete txt;
		return html2dat_old(html, startResNum, lastModified, useCache);
	}
	else {
		char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, " class=\"post\"", 13);
		if(ptr2) {
			char *tmp = ptr2;
			*ptr2 = 0;
			while(*ptr2 != '<') ptr2--;
			strcpy(signatureTag, ptr2);
			*tmp = ' ';
		}
		else {
			delete txt;
			return NULL;
		}
		/*char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, "<dl class=\"post\"", 16);
		if(ptr2) {
			delete txt;
			return html2dat_pink(html, startResNum, lastModified, useCache);
		}*/
	}
	
	ptr += 18;
	while(1) {
		if(*ptr == '<') {
			if(!strncasecmp(ptr,"</h1>",5)) {
				ptr += 5;
				break;
			}
			else title[i++] = *ptr++;
		}
		else if(*ptr == '\n') break;
		else title[i++] = *ptr++;
	}
	title[i] = 0;
	
	sprintf(signature,"%s class=\"post\" id=\"%d\"",signatureTag,res);
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		ptr = strstr(ptr," class=\"name\"><b>");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			sprintf(closeTag, "</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			ptr = tmp + 17;
		}
		else {
			delete resData;
			break;
		}
		
		char endStr[64];
		if(!strncmp(ptr,"<a href=\"mailto:",16)) {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(1) {
					if(*ptr == '<' && !strncmp(ptr,"<a href=\"",9)) {
						ptr = strchr(ptr,'>');
						ptr++;
						char *link = ptr;
						ptr = strstr(link,"</a>");
						memcpy(mail+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
					else if(*ptr == '"') break;
					else mail[i++] = *ptr++;
				}
				//while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			sprintf(endStr,"</a></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}
		/* we do not have to handle this special case because read.cgi on bbspink doesn't
		   emit font tags anymore and it conflicts with text decorations using "melon point" */
		/*else if(!strncmp(ptr,"<font",5)) {
			sprintf(endStr,"</font></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}*/
		else {
			sprintf(endStr,"</b>%s",closeTag);
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncmp(ptr,endStr,strlen(endStr))) {
					ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"",9)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(name+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr = strstr(ptr," class=\"date\">");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			sprintf(closeTag, "</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			ptr = tmp + 14;
		}
		else {
			delete resData;
			break;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		if(!strncmp(ptr,"<div class=\"uid",15) || !strncmp(ptr,"<span class=\"uid",16)) {
			char *tmp = ptr+1;
			while(*ptr != ' ') ptr++;
			*ptr = 0;
			sprintf(closeTag, "</%s>",tmp);
			closeTagLen = strlen(closeTag);
			ptr += 11;
			while(*ptr != '>') ptr++;
			ptr++;
			date[i++] = ' ';
			while(1) {
				if(*ptr == '<') {
					if(!strncasecmp(ptr,closeTag,closeTagLen)) {
						ptr += closeTagLen;
						break;
					}
					else date[i++] = *ptr++;
				}
				else date[i++] = *ptr++;
			}
		}
		
		if(!strncmp(ptr,"<div class=\"be",14) || !strncmp(ptr,"<span class=\"be",15)) {
			ptr += 14;
			while(*ptr != '>') ptr++;
			ptr++;
			if(!strncmp(ptr,"<a href=\"",9)) {
				ptr += 9;
				while(*ptr != '/' && *ptr != '"') ptr++;
				if(*ptr == '/' && (!strncmp(ptr,"//be.2ch.net/user/",18) || !strncmp(ptr,"//be.5ch.net/user/",18))) {
					memcpy(date+i," BE:",4);
					i += 4;
					ptr += 18;
					while(*ptr != '"') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
			}
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		if(!strcmp(signatureTag,"<div")) {
			ptr = strstr(ptr,"<div class=\"message\">");
			if(!ptr) {
				delete resData;
				break;
			}
			else {
				ptr += 21;
				if(!strncasecmp(ptr,"<span class=\"escaped\">",22)) {
					if(!strncasecmp(ptr+22,"<span class=\"AA\">",17)) {
						strcpy(closeTag,"</span></span></div>");
						closeTagLen = 20;
						ptr += 22+17;
					}
					else {
						strcpy(closeTag,"</span></div>");
						closeTagLen = 13;
						ptr += 22;
					}
				}
				else {
					strcpy(closeTag,"</div>");
					closeTagLen = 6;
				}
			}
		}
		else {
			ptr = strstr(ptr,"<dd class=\"thread_in\">");
			if(!ptr) {
				delete resData;
				break;
			}
			strcpy(closeTag,"</dd>");
			closeTagLen = 5;
			ptr += 22;
		}
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a ",3)) {
					char *tmp = strchr(ptr,'>');
					char *href = (char *)memmem_priv(ptr,tmp-ptr,"href=\"",6);
					char *link = tmp+1;
					if(href && !strncmp(link,"&gt;&gt;",8) && memmem_priv(href,link-href,"test/read.cgi/",14)) {
						while(ptr < link) {
							if(!strncmp(ptr," class=\"",8)) {
								ptr += 8;
								while(*ptr != '"' && *ptr != '>') ptr++;
								if(*ptr == '"') ptr++;
							}
							else body[i++] = *ptr++;
						}
					}
					else {
						ptr = strstr(link,"</a>");
						memcpy(body+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10)  || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		sprintf(signature,"%s class=\"post\" id=\"",signatureTag);
		ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
		if(ptr) {
			int next = atoi(ptr+strlen(signature));
			if(next >= res) {
				while(next > res) {
					txt->appendBytes("broken<><>broken<> broken <>\n", 29);
					res++;
				}
			}
			else ptr = NULL;
		}
		if(!ptr) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				sprintf(formattedDate,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

int BBS2chProxyConnection::datProxyAPI(const char *url, const char *method)
{
	long statusCode = 0;
	std::string postBody = auth->requestBodyForURL(url);
	if(postBody.empty()) {
		sendResponse(401, "Unauthorized", fpw);
		return 401;
	}
	CURL *curl = curl_easy_init();
	char *buf = (char *)malloc(16384);
	if(curl) {
		CURLcode res;
		struct curl_slist *headers = NULL;
		DataStorage *receivedHeader = new DataStorage();
		DataStorage *receivedBody = new DataStorage();
		while(fgets(buf,16384,fpr)) {
			//fprintf(stderr,"%s",buf);
			if(!strcmp("\r\n",buf)) break;
			else if(!strncasecmp("Range:",buf,6)
				 || !strncasecmp("If-Modified-Since:",buf,18)
				 || !strncasecmp("Accept-Encoding:",buf,16)) {
				char *ptr;
				ptr = strchr(buf, '\r');
				if(!ptr) ptr = strchr(buf, '\n');
				if(!ptr) continue;
				*ptr = 0;
				headers = curl_slist_append(headers, buf);
			}
		}
		if(x_2ch_ua_dat) headers = curl_slist_append(headers, x_2ch_ua_dat);
		
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, receivedBody);
		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_download);
		curl_easy_setopt(curl, CURLOPT_HEADERDATA, receivedHeader);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		curl_easy_setopt(curl, CURLOPT_USERAGENT, api_ua_dat?api_ua_dat:"");
		curl_easy_setopt(curl, CURLOPT_POST, 1L);
#if LIBCURL_VERSION_NUM >= 0x071101
		curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, postBody.c_str());
#else
		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postBody.c_str());
#endif
		//return;
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			receivedHeader->appendBytes("",1);
			char *ptr = strstr(receivedHeader->bytes,"\r\n\r\n");
			*(ptr+4) = 0;
			if(strstr(receivedHeader->bytes,"\r\nThread-Status: 1")) {
				if(ptr+4-receivedHeader->bytes > fwrite(receivedHeader->bytes,1,ptr+4-receivedHeader->bytes,fpw)) goto last;
				fflush(fpw);
				if(receivedBody->length > fwrite(receivedBody->bytes,1,receivedBody->length,fpw)) goto last;
				fflush(fpw);
				goto last;
			}
			else if(strstr(receivedHeader->bytes,"\r\nThread-Status: 8")) {
				sendBasicHeaders(302, "Found", fpw);
				if(0 > fprintf(fpw, "Location: http://www2.2ch.net/live.html\r\n")) goto last;
				if(0 > fprintf(fpw, "\r\n")) goto last;
				fflush(fpw);
				statusCode = 302;
				goto last;
			}
			else {
				if (statusCode < 400) {
					sendResponse(401, "Unauthorized", fpw);
					statusCode = 401;
				}
				else {
					sendResponse(503, "Service Unavailable", fpw);
					statusCode = 503;
				}
				receivedBody->appendBytes("",1);
				if(!strncasecmp(receivedBody->bytes,"ng (",4)) {
					log_printf(0,"API gateway returned error: %s\n",receivedBody->bytes);
				}
			}
			//fprintf(stderr,"%ld\n",statusCode);
		}
		else {
			log_printf(0,"curl error: %s\n",curl_easy_strerror(res));
			sendResponse(503, "Service Unavailable", fpw);
			statusCode = 503;
		}
last:
		curl_easy_cleanup(curl);
		curl_slist_free_all(headers);
		delete receivedBody;
		delete receivedHeader;
	}
	free(buf);
	return statusCode;
}

int BBS2chProxyConnection::bbsmenuProxy(const char *url, const char *method)
{
	long statusCode = 0;
	DataStorage *dat = new DataStorage();
	DataStorage *outHTML = new DataStorage();
	char *buf = (char *)malloc(16384);
	char userAgentFromHeader[1024] = "";
	CURL *curl = NULL;
	if(!buf) goto last;
	
	while(fgets(buf,16384,fpr)) {
		//fprintf(stderr,"%s",buf);
		if(!strcmp("\r\n",buf)) break;
		else if(!strncasecmp("User-Agent:",buf,11)) {
			int i=0;
			char *ptr = buf+12;
			while(*ptr != '\r' && *ptr != '\n' && i < 1023) userAgentFromHeader[i++] = *ptr++;
			userAgentFromHeader[i] = 0;
		}
	}
	curl = curl_easy_init();
	if(curl) {
		CURLcode res;
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_ENCODING, "");
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if(user_agent) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
		}
		else if(userAgentFromHeader[0]) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
		}
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			if(statusCode == 200 && dat->length) {
				dat->appendBytes("",1);
				dat->length--;
				char *ptr = dat->bytes;
				while(*ptr) {
					if(!strncasecmp(ptr,"<a href=",8)) {
						char *start = ptr;
						char *end = strchr(ptr+8,'>');
						ptr = strstr(ptr+8,"://");
						if(ptr && ptr < end) {
							char *protocol = ptr;
							char *end2 = strchr(ptr+3,'/');
							ptr = strstr(protocol+3,"5ch.net");
							if(ptr && ptr < end2 && (*(ptr-1)=='.' || *(ptr-1)=='/')) {
								memcpy(ptr,"2ch",3);
								if(*(protocol-1) == 's') outHTML->appendBytes(start, protocol-start-1);
								else outHTML->appendBytes(start, protocol-start);
								outHTML->appendBytes(protocol, end-protocol);
								ptr = end;
								continue;
							}
							ptr = strstr(protocol+3,"bbspink.com");
							if(ptr && ptr < end2 && (*(ptr-1)=='.' || *(ptr-1)=='/')) {
								if(*(protocol-1) == 's') outHTML->appendBytes(start, protocol-start-1);
								else outHTML->appendBytes(start, protocol-start);
								outHTML->appendBytes(protocol, end-protocol);
								ptr = end;
								continue;
							}
						}
						ptr = start;
					}
					outHTML->appendBytes(ptr++, 1);
				}
			}
		}
		else {
			log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),buf);
			statusCode = 503;
		}
	}
	if(statusCode == 200) {
		sendBasicHeaders(statusCode,"OK",fpw);
		if(0 > fprintf(fpw,"Content-Type: text/html\r\n")) goto last;
		if(0 > fprintf(fpw,"Content-Length: %ld\r\n",(long)outHTML->length)) goto last;
		if(0 > fprintf(fpw,"\r\n")) goto last;
		if(strcasecmp(method, "HEAD")) {
			if(outHTML->length > fwrite(outHTML->bytes,1,outHTML->length,fpw)) goto last;
		}
		fflush(fpw);
	}
	else {
		sendResponse(503, "Service Unavailable", fpw);
		statusCode = 503;
	}
	
last:
	if(curl) curl_easy_cleanup(curl);
	if(buf) free(buf);
	if(dat) delete dat;
	if(outHTML) delete outHTML;
	return statusCode;
}