proxy2ch/BBS2chProxyConnection.cpp

#include <string>
#include <vector>
#include <map>
#include <set>
#include <sstream>
#include <stdexcept>
#include <pthread.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <curl/curl.h>
#ifdef USE_LUA
#include <lua.hpp>
#endif
#ifdef _WIN32
#include <fcntl.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <mswsock.h>
#define CLOSESOCKET(x) closesocket(x)
#else
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <arpa/inet.h>
#define CLOSESOCKET(x) close(x)
#endif
#include "BBS2chProxyConnection.h"
#include "DataStorage.h"
#include "hmac.h"
#include "stringEncodingConverter.h"
#include "BBS2chProxyRawSocket.h"
#ifdef USE_MITM
#include "BBS2chProxySecureSocket.h"
#endif
//#define DEBUG 1

extern char *proxy_server;
extern long proxy_port;
extern long proxy_type;
extern long timeout;
extern char *user_agent;
extern char *api_ua_dat;
extern char *x_2ch_ua_dat;
extern char *appKey;
extern char *hmacKey;
extern int allow_chunked;
extern int curl_features;
extern unsigned int curl_version_number;
extern bool accept_https;
extern int force_5chnet;
extern int force_5chnet_https;
extern int force_ipv4;
extern char *bbsmenu_url;
extern char *api_server;
extern std::map<std::string, std::string> bbscgi_headers;
extern int gikofix;
extern CURLSH *curl_share;
extern char *lua_script;
extern unsigned int api_mode;
extern std::vector<std::string> bbscgi_postorder;
#ifdef USE_MITM
extern unsigned int mitm_mode;
#endif

extern void log_printf(int level, const char *format ...);

#include "utils.h"

static std::string monaKey;
static std::map<std::string, double> monaKeyIssueTime;
static std::set<std::string> expiredMonaKeys;

#ifdef USE_LUA
extern "C" {
	static int lua_hmacSHA256(lua_State *l)
	{
		static const char *table = "0123456789abcdef";
		size_t keyLength, dataLength;
		const char *key = luaL_checklstring(l, 1, &keyLength);
		const char *data = luaL_checklstring(l, 2, &dataLength);
		if (!key || !data) return 0;
		unsigned char digest[32];
		char digestStr[65];
		proxy2ch_HMAC_SHA256(key, keyLength, data, dataLength, digest);
		for (int i=0; i<32; i++) {
			unsigned char c = digest[i];
			unsigned char upper = (c >> 4) & 0xf;
			unsigned char lower = c & 0xf;
			digestStr[i*2] = table[upper];
			digestStr[i*2+1] = table[lower];
		}
		digestStr[64] = 0;
		lua_pushstring(l, digestStr);
		return 1;
	}
	static int lua_decodeURIComponent(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		bool decodePlus = true;
		if (!lua_isnoneornil(l, 2)) {
			decodePlus = (lua_toboolean(l, 2));
		}

		std::string output = decodeURIComponent(input, length, decodePlus);
		lua_pushstring(l, output.c_str());
		return 1;
	}
	static int lua_encodeURIComponent(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		bool spaceAsPlus = true;
		if (!lua_isnoneornil(l, 2)) {
			spaceAsPlus = (lua_toboolean(l, 2));
		}

		std::string output = encodeURIComponent(input, length, spaceAsPlus);
		lua_pushstring(l, output.c_str());
		return 1;
	}
	static int lua_convertShiftJISToUTF8(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		if (length > 0) {
			char *output = convertShiftJISToUTF8(input, length);
			if (!output) lua_pushnil(l);
			else {
				lua_pushstring(l, output);
				free(output);
			}
		}
		else lua_pushstring(l, "");
		return 1;
	}
	static int lua_isExpiredKey(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		if (expiredMonaKeys.count(input)) {
			lua_pushboolean(l, 1);
		}
		else lua_pushboolean(l, 0);
		return 1;
	}
}
#endif
void BBS2chProxyConnection::setMonaKey(const std::string &key, int reason) {
	pthread_mutex_lock(mutex);
	if (key.empty()) {
		if ((reason >= 3320 && reason <= 3324) || (reason >= 3390 && reason <= 3392)) {
			expiredMonaKeys.insert(monaKeyForRequest);
		}
		log_printf(1, "Reset MonaKey\n");
	}
	else {
		monaKeyIssueTime[key] = getCurrentTime();
		log_printf(1, "Updated MonaKey: %s\n", key.c_str());
	}
	monaKey = key;
	pthread_mutex_unlock(mutex);
}
std::string BBS2chProxyConnection::getMonaKey() {
	if (monaKey.empty())
		return "00000000-0000-0000-0000-000000000000";
	return monaKey;
}

void *BBS2chProxyConnection::launch(void *param)
{
	reinterpret_cast<BBS2chProxyConnection *>(param)->connect();
	return (void *)NULL;
}

void BBS2chProxyConnection::run(void)
{
	pthread_t thread;
	pthread_attr_t thread_attr;
	pthread_attr_init(&thread_attr);
	pthread_attr_setdetachstate(&thread_attr , PTHREAD_CREATE_DETACHED);
	if(0 != pthread_create(&thread , &thread_attr , &BBS2chProxyConnection::launch , this))
		perror("pthread_create");
	pthread_attr_destroy(&thread_attr);
}

void *BBS2chProxyConnection::tunnel_c2s(void *param)
{
	char buf[2048];
	fd_set fds;
	int sock_c = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_c;
	int sock_s = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_s;
#ifdef _WIN32
	struct timeval timeout;
	timeout.tv_sec = 0;
	timeout.tv_usec = 10000;
#endif
	
	while(1) {
#ifdef _WIN32
		FD_ZERO(&fds);
		FD_SET(sock_c,&fds);
		if(select(sock_c + 1,&fds,NULL,NULL,&timeout) < 0) break;
		if(FD_ISSET(sock_c,&fds)) {
#endif
			int ret = recv(sock_c, buf, 2048, 0);
			if(ret > 0) send(sock_s, buf, ret, 0);
			else if(ret <= 0) break;
#ifdef _WIN32
		}
#endif
	}
	//fprintf(stderr,"tunnel_c2s end\n");
	CLOSESOCKET(sock_s);
	return NULL;
}

void *BBS2chProxyConnection::tunnel_s2c(void *param)
{
	char buf[2048];
	fd_set fds;
	int sock_c = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_c;
	int sock_s = reinterpret_cast<BBS2chProxyConnection *>(param)->sock_s;
	
	while(1) {
		int ret = recv(sock_s, buf, 2048, 0);
		if(ret > 0) send(sock_c, buf, ret, 0);
		else if(ret <= 0) break;
	}
	//fprintf(stderr,"tunnel_s2c end\n");
	CLOSESOCKET(sock_c);
	return NULL;
}

int BBS2chProxyConnection::tunnel(const char *addr, int port)
{
	struct sockaddr_in server;
	memset(&server, 0, sizeof(server));
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = inet_addr(addr);
	server.sin_port = htons(port);
	if(server.sin_addr.s_addr == 0xffffffff) {
		struct hostent *host;
		 host = gethostbyname(addr);
		if (host == NULL) {
			sendResponse(400, "Bad Request", socketToClient);
			return -1;
		}
		server.sin_addr.s_addr = *(unsigned int *)host->h_addr_list[0];
	 }
	
	log_printf(1,"Tunneling connection to %s:%d\n",addr,port);
	
	sock_s = socket(AF_INET, SOCK_STREAM, 0);
	if(-1 == ::connect(sock_s, (struct sockaddr *)&server, sizeof(server))) {
		perror("connect");
		sendResponse(400, "Bad Request", socketToClient);
		return -1;
	}
	send(sock_c, "HTTP/1.1 200 Connection established\r\n\r\n", 39, 0);
	
	pthread_t thread_c2s, thread_s2c;
	if(0 != pthread_create(&thread_c2s, NULL, &BBS2chProxyConnection::tunnel_c2s, this))
		perror("pthread_create");
	if(0 != pthread_create(&thread_s2c, NULL, &BBS2chProxyConnection::tunnel_s2c, this))
		perror("pthread_create");
	pthread_join(thread_c2s, NULL);
	pthread_join(thread_s2c, NULL);
	
	log_printf(1,"Tunneling connection to %s:%d finished\n",addr,port);
	
	return 0;
}

void BBS2chProxyConnection::connect(void)
{
	char method[32], url[1024], protocol[32];
	int i;
	char *buf, *ptr;
	bool html2dat = false;
	bool html2dat_kako = false;
	regmatch_t match[7];
	long statusCode = 0;
	std::string baseURL;
	
	socketToClient = new BBS2chProxyRawSocket(sock_c);
	
	buf = (char *)malloc(16384);
	if(!buf) goto end;
beginHandleRequest:
	ptr = buf;
	if(!socketToClient->readLine(buf, 1024)) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 32) method[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	method[i] = 0;
	ptr++;
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 1024) url[i++] = *ptr++;
	if(*ptr == 0 || i == 1024) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	url[i] = 0;
	ptr++;
	i=0;
	while(*ptr != '\r' && *ptr != '\n' && *ptr != 0 && i < 32) protocol[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	protocol[i] = 0;
	log_printf(1, "Received %s %s %s\n",method,url,protocol);
	
	if(strcasecmp(method,"GET") && strcasecmp(method,"POST") && strcasecmp(method,"HEAD") && strcasecmp(method,"CONNECT") && strcasecmp(method,"PUT") && strcasecmp(method, "OPTIONS")) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	if(!url[0]) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	if(strncasecmp(protocol,"HTTP",4)) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	
	if(!strcasecmp(method,"CONNECT")) {
		if(!accept_https || !baseURL.empty()) {
			sendResponse(400, "Bad Request", socketToClient);
			statusCode = 400;
			goto end;
		}
		while(socketToClient->readLine(buf, 16384)) {
			if(!strcmp("\r\n",buf)) break;
		}
		int port = 443;
		char *ptr = strchr(url, ':');
		if(ptr) {
			*ptr = 0;
			port = atoi(ptr+1);
		}
#ifdef USE_MITM
		bool useMITM = false;
		if (mitm_mode) {
			if (mitm_mode == 2) useMITM = true;
			else if (mitm_mode == 1) {
				int length = strlen(url);
				if (length >= strlen("5ch.net") && !strcasecmp(url+length-strlen("5ch.net"), "5ch.net")) {
					int pos = length-strlen("5ch.net");
					if (pos == 0) useMITM = true;
					else useMITM = (url[pos-1] == '.');
				}
				else if (length >= strlen("2ch.net") && !strcasecmp(url+length-strlen("2ch.net"), "2ch.net")) {
					int pos = length-strlen("2ch.net");
					if (pos == 0) useMITM = true;
					else useMITM = (url[pos-1] == '.');
				}
				else if (length >= strlen("bbspink.com") && !strcasecmp(url+length-strlen("bbspink.com"), "bbspink.com")) {
					int pos = length-strlen("bbspink.com");
					if (pos == 0) useMITM = true;
					else useMITM = (url[pos-1] == '.');
				}
			}
		}
		if (useMITM) {
			socketToClient->writeString("HTTP/1.1 200 Connection established\r\n\r\n");
			if (port == 80) {
				baseURL = "http://";
				baseURL += url;
				goto beginHandleRequest;
			}
			else {
				try {
					BBS2chProxySecureSocket *secureSocket = new BBS2chProxySecureSocket(sock_c, url);
					delete socketToClient;
					socketToClient = secureSocket;
					baseURL = "https://";
					baseURL += url;
					if (port != 443) {
						snprintf(buf, 16384, "%d", port);
						baseURL += ":";
						baseURL += buf;
					}
					goto beginHandleRequest;
				} catch (const std::runtime_error& e) {
					log_printf(0, "%s\n", e.what());
					sendResponse(400, "Bad Request", socketToClient);
					statusCode = 400;
					goto end;
				}
			}
		}
		else
#endif
		{
			delete socketToClient;
			socketToClient = NULL;
			tunnel(url, port);
			goto end;
		}
	}
#if USE_MITM
	if(!baseURL.empty()) {
		std::string completeURL = baseURL + url;
		strcpy(url, completeURL.c_str());
		log_printf(1, "Running as MITM proxy for %s\n", url);
	}
#endif
	if(force_5chnet && !strncasecmp(url, "http", 4)) {
		char *ptr = strstr(url, "://");
		if (ptr) {
			ptr += 3;
			char *end = ptr;
			while(*end != '/' && *end != 0) end++;
			ptr = strstr(ptr,".2ch.net");
			if(ptr && ptr < end && memcmp(ptr-4,"menu.",5)) {
				memcpy(ptr+1,"5ch",3);
				force5ch = true;
				log_printf(1, "Detected *.2ch.net URL, changed target URL to %s\n",url);
			}
		}
	}
	
	if(regexec(&regex, url, 6, match, 0) != REG_NOMATCH) {
		if((appKey && (api_mode & 1)) || strncasecmp(url+match[1].rm_so,"headline.",9)) html2dat = true;
	}
	else if(regexec(&regex_kako, url, 7, match, 0) != REG_NOMATCH) {
		html2dat_kako = true;
	}
	else if(regexec(&regex_offlaw, url, 5, match, 0) != REG_NOMATCH) {
		char *thread = strstr(url,"key=");
		if(thread) {
			match[6].rm_so = thread+4-url;
			match[6].rm_eo = thread+4-url;
			char *ptr = thread+4;
			while(*ptr != '&' && *ptr != 0) {
				ptr++;
				match[6].rm_eo++;
			}
			if(match[6].rm_so != match[6].rm_eo) html2dat_kako = true;
		}
	}
	
	if(html2dat || html2dat_kako) {
		char tmp[1024];
		regmatch_t *tid = html2dat ? match+5 : match+6;
		strcpy(tmp,url);
		tmp[match[2].rm_eo] = 0;
		tmp[match[4].rm_eo] = 0;
		tmp[tid->rm_eo] = 0;
		if(!appKey || !(api_mode & 1) || html2dat_kako) {
			log_printf(1, "Retrieving thread via read.cgi...\n");
			snprintf(url,1024,"%s/%s/%s",tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			threadKey = std::string(url);
			snprintf(url,1024,"%s://%s/test/read.cgi/%s/%s/",force_5chnet_https?"https":"http",tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			statusCode = datProxy(url, method);
		}
		else {
			log_printf(1, "Retrieving thread via API...\n");
			tmp[match[1].rm_eo] = 0;
			snprintf(url,1024,"https://%s/v1/%s/%s/%s",api_server,tmp+match[1].rm_so,tmp+match[4].rm_so,tmp+tid->rm_so);
			statusCode = datProxyAPI(url, method);
		}
	}
	else {
		char urlMod[1024];
		strcpy(urlMod,url);
		if(force_5chnet_https) {
			const char *host = "5ch.net";
			char *ptr = strstr(url,host);
			if(!ptr) {
				host = "2ch.net";
				ptr = strstr(url,host);
			}
			if(!ptr) {
				host = "bbspink.com";
				ptr = strstr(url,host);
			}
			if(ptr) {
				char *start = url+7;
				char *end = strchr(start,'/');
				int hostLength = strlen(host);
				if(!end) end = url+strlen(url);
				if(ptr >= start && ptr < end && (*(ptr-1) == '.' || *(ptr-1) == '/') && (*(ptr+hostLength) == '/' || *(ptr+hostLength) == ':' || *(ptr+hostLength) == 0)) {
					log_printf(1, "This is %s URL, connecting with HTTPS\n",host);
					if(*(ptr+hostLength) == ':') {
						*(ptr+hostLength) = 0;
						snprintf(urlMod,1024,"https://%s%s",start,end);
						*(ptr+hostLength) = ':';
					}
					else snprintf(urlMod,1024,"https://%s",start);
					log_printf(2, "URL is: %s\n",urlMod);
				}
			}
		}
		
		if(bbsmenu_url && !strcmp(url, bbsmenu_url)) {
			log_printf(1, "Running as a BBS menu proxy...\n");
			statusCode = bbsmenuProxy(urlMod, method);
		}
		else {
			bool isPostRequest = !strcasecmp(method, "POST");
			bool isPutRequest = !strcasecmp(method, "PUT");
			if(isPostRequest && (strstr(urlMod,".5ch.net") || strstr(urlMod,".bbspink.com")) && strstr(urlMod,"/test/bbs.cgi")) bbscgi = true;
			log_printf(1, "Not a thread request, passthrough...\n");
			struct curl_slist *headers = NULL;
			bool hasExpect = false;
			std::string hostStr;
			bool isNotFormURLEncoded = false;
#ifdef USE_LUA
			std::map<std::string, std::string> headersForLua;
#endif
			while(socketToClient->readLine(buf, 16384)) {
				if(bbscgi) {
					char *ptr = strchr(buf, ':');
					if(ptr) {
						std::string header(buf, ptr-buf);
						if(bbscgi_headers.find(header) != bbscgi_headers.end()) {
							log_printf(1, "Ignoring header \"%s\" because alternative value exists\n", header.c_str());
							continue;
						}
					}
				}
				//fprintf(fpw,"%s",buf);
				//fprintf(stderr,"%s",buf);
				if(!strcmp("\r\n",buf)) break;
				if(!strncasecmp("Connection:",buf,11)) continue;
				else if(!strncasecmp("Host:",buf,5)) {
					char *ptr;
					ptr = strchr(buf, '\r');
					if(!ptr) ptr = strchr(buf, '\n');
					if(!ptr) continue;
					*ptr = 0;
					ptr = strchr(buf+5, ':');
					if(ptr) *ptr = 0;
					ptr = buf+5;
					while(*ptr == ' ') ptr++;
					if(force_5chnet) {
						char *ptr2 = strstr(ptr, ".2ch.net");
						if(ptr2) {
							*(ptr2+1) = '5';
						}
					}
					hostStr = std::string(ptr);
					continue;
				}
				//else if(!strncasecmp("Authorization:",buf,14)) continue;
				//else if(!strncasecmp("WWW-Authenticate:",buf,17)) continue;
				else if(user_agent && !strncasecmp("User-Agent:",buf,11)) continue;
				else if(bbscgi && !strncasecmp("Content-Length:",buf,15)) {
					char *ptr = buf + 15;
					while(*ptr == ' ') ptr++;
					content_length = atoi(ptr);
					continue;
				}
				else {
					char *ptr;
					ptr = strchr(buf, '\r');
					if(!ptr) ptr = strchr(buf, '\n');
					if(!ptr) continue;
					*ptr = 0;
					if(bbscgi && force_5chnet && !strncasecmp("Referer:",buf,8)) {
						char *ptr2 = strstr(buf+8, ".2ch.net");
						if(ptr2) {
							*(ptr2+1) = '5';
						}
					}
					headers = curl_slist_append(headers, buf);
					if(!strncasecmp("Content-Length:",buf,15)) {
						char *ptr = buf + 15;
						while(*ptr == ' ') ptr++;
						content_length = atoi(ptr);
					}
					else if(!strncasecmp("Expect:",buf,7)) {
						hasExpect = true;
					}
					else if(!strncasecmp("Content-Type:",buf,13)) {
						ptr = buf + 13;
						while(*ptr == ' ') ptr++;
						if(strncasecmp("application/x-www-form-urlencoded", ptr, 33)) {
							isNotFormURLEncoded = true;
						}
					}
				}
			}
			if(bbscgi && !isNotFormURLEncoded && content_length) {
				char *postdata = (char *)calloc(content_length+1, 1);
				content_length = socketToClient->read(postdata, content_length);
				if(gikofix && content_length > 0) {
					char *ptr = postdata+content_length-1;
					while(*ptr == '\r' || *ptr == '\n') {
						*ptr-- = 0;
					}
				}
				statusCode = bbsCgiProxy(urlMod, protocol, hostStr, headers, postdata);
				free(postdata);
				curl_slist_free_all(headers);
				goto end;
			}
			if(!hasExpect) headers = curl_slist_append(headers, "Expect:");
			
			CURL *curl = curl_easy_init();
			if(curl) {
				CURLcode res;
				if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
				curl_easy_setopt(curl, CURLOPT_URL, urlMod);
				curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
				curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
				if(content_length) {
					/* set Content-Length explicitly via API to work properly with curl >= 7.66.0 */
					if(isPostRequest)
						curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, content_length);
					else if(isPutRequest)
						curl_easy_setopt(curl, CURLOPT_INFILESIZE, content_length);
				}
				curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_READDATA, this);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
				//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
				if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
				if(!strncasecmp(protocol,"HTTP/1.0",8) && !strncasecmp(urlMod, "http://", 7)) {
					curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
				}
				else {
					/* force use HTTP 1.1 because CURL_HTTP_VERSION_2TLS is used on curl (w/ nghttp2) >= 7.62.0 */
					curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
				}
				curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
				if(user_agent) {
					curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
				}
				if(isPostRequest) {
					curl_easy_setopt(curl, CURLOPT_POST, 1L);
				}
				else if(isPutRequest) {
					curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
				}
				else if(!strcasecmp(method, "HEAD")) {
					curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
				}
				else if(!strcasecmp(method, "OPTIONS")) {
					curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "OPTIONS");
				}
				if(proxy_server) {
					curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
					curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
					curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
				}
				res = curl_easy_perform(curl);
				if(res != CURLE_OK) {
					log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),urlMod);
					if(!status) sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
				else {
					if(chunked) {
						socketToClient->writeString("0\r\n\r\n");
					}
					curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
				}
				curl_easy_cleanup(curl);
			}
			curl_slist_free_all(headers);
		}
	}
	
  end:
	if(statusCode) log_printf(1, "Returned status code %d to client\n",statusCode);
	if(buf) free(buf);
	if(socketToClient) socketToClient->close();
	delete this;
}

int BBS2chProxyConnection::datProxy(const char *url, const char *method)
{
	DataStorage *html = NULL;
	long statusCode = 0;
	long rangeStart = 0, rangeEnd = 0;
	time_t lastModified = 0;
	time_t ifModifiedSince = 0;
	char *buf = (char *)malloc(16384);
	char userAgentFromHeader[1024] = "";
	if(!buf) goto last;
	
	while(socketToClient->readLine(buf, 16384)) {
		//fprintf(stderr,"%s",buf);
		if(!strcmp("\r\n",buf)) break;
		if(!strncasecmp("Range:",buf,6)) {
			if(strstr(buf+7,"bytes=") && !strchr(buf+7, ',')) {
				char *ptr = buf+13;
				if(*ptr == '-') {
					rangeStart = atoi(ptr);
				}
				else {
					rangeStart = strtol(ptr, &ptr, 10);
					if(*ptr == '-') ptr++;
					if(*ptr && *ptr != '\r') {
						rangeEnd = strtol(ptr, NULL, 10);
						if(rangeEnd && rangeStart > rangeEnd) {
							sendResponse(416, "Requested range not satisfiable", socketToClient);
							statusCode = 416;
							goto last;
						}
					}
				}
				//fprintf(stderr, "range=%ld-%ld\n",rangeStart,rangeEnd);
			}
			else {
				sendResponse(416, "Requested range not satisfiable", socketToClient);
				statusCode = 416;
				goto last;
			}
		}
		else if(!strncasecmp("If-Modified-Since:",buf,18)) {
			struct tm time_ = {};
			strptime(buf+19,httpTimestampFmt,&time_);
			ifModifiedSince = mktime(&time_);
		}
		else if(!strncasecmp("User-Agent:",buf,11)) {
			int i=0;
			char *ptr = buf+12;
			while(*ptr != '\r' && *ptr != '\n' && i < 1023) userAgentFromHeader[i++] = *ptr++;
			userAgentFromHeader[i] = 0;
		}
	}
	if(rangeStart > 0) {
		PBBS2chProxyThreadInfo info;
		pthread_mutex_lock(mutex);
		BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
		if(it != threadCache->end()) {
			info = it->second;
		}
		pthread_mutex_unlock(mutex);
		log_printf(5,"range request from %ld bytes\n",rangeStart);
		if(info) {
			int from = info->lastResNum;
			int alreadyRead = info->cachedSize;
			int lastResLength = info->cachedData->length;
			log_printf(5,"hit %s: cached %d bytes, last res size %d\n",threadKey.c_str(),alreadyRead,lastResLength);
			if(rangeStart <= alreadyRead && rangeStart >= alreadyRead - lastResLength) {
				CURL *curl = curl_easy_init();
				if(curl) {
					CURLcode res;
					DataStorage *dat = new DataStorage();
					log_printf(5,"partial access from res num %d\n",from);
					snprintf(buf,16384,"%s%d-n",url,from);
					if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
					curl_easy_setopt(curl, CURLOPT_URL, buf);
					curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
					curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
					curl_easy_setopt(curl, CURLOPT_ENCODING, "");
					curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
					curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
					curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
					if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
					if(proxy_server) {
						curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
						curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
						curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
					}
					if(user_agent) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
					}
					else if(userAgentFromHeader[0]) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
					}
					res = curl_easy_perform(curl);
					if(res == CURLE_OK) {
						curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
						curl_easy_cleanup(curl);
						if(statusCode == 200 && dat->length) {
							DataStorage *updated = html2dat(dat, from, &lastModified, true);
							if(ifModifiedSince && lastModified && updated && updated->length == lastResLength) {
								struct tm time_ = {};
								gmtime_r(&lastModified,&time_);
								time_t tmp = mktime(&time_);
								if(ifModifiedSince >= tmp) {
									sendResponse(304, "Not Modified", socketToClient);
									log_printf(5,"not modified!\n");
									delete updated;
									delete dat;
									statusCode = 304;
									goto last;
								}
							}
							if(updated && updated->length && updated->length >= lastResLength) {
								html = new DataStorage(alreadyRead - lastResLength);
								html->appendBytes(updated->bytes, updated->length);
								if(!rangeEnd) rangeEnd = html->length - 1;
								if(rangeStart > rangeEnd) {
									sendResponse(416, "Requested range not satisfiable", socketToClient);
									delete updated;
									delete dat;
									statusCode = 416;
									goto last;
								}
								statusCode = 206;
								log_printf(5,"cache hit; reconstructed data length:%ld\n",(long)html->length);
							}
							else {
								log_printf(5,"cache misshit?\n");
								sendResponse(416, "Requested range not satisfiable", socketToClient);
								delete updated;
								delete dat;
								statusCode = 416;
								goto last;
							}
							delete updated;
						}
					}
					else {
						log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),buf);
						curl_easy_cleanup(curl);
					}
					delete dat;
					if(html) goto resp;
				}
			}
			else {
				log_printf(5,"invalid cache contents\n");
				pthread_mutex_lock(mutex);
				BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
				if(it != threadCache->end()) {
					threadCache->erase(it);
				}
				pthread_mutex_unlock(mutex);
			}
		}
	}
	
	{
		CURL *curl = curl_easy_init();
		if(curl) {
			CURLcode res;
			DataStorage *dat = new DataStorage();
			if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
			curl_easy_setopt(curl, CURLOPT_URL, url);
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			curl_easy_setopt(curl, CURLOPT_ENCODING, "");
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			if(proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			if(user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			else if(userAgentFromHeader[0]) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
			}
			res = curl_easy_perform(curl);
			if(res != CURLE_OK) {
				log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),url);
				sendResponse(503, "Service Unavailable", socketToClient);
				curl_easy_cleanup(curl);
				delete dat;
				statusCode = 503;
				goto last;
			}
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			curl_easy_cleanup(curl);
			
			if(statusCode == 200) {
				html = html2dat(dat, 1, &lastModified, false);
			}
			delete dat;
		}
		
		if(!html || !html->length) {
			sendResponse(503, "Service Unavailable", socketToClient);
			statusCode = 503;
			goto last;
		}
		if((rangeStart || rangeEnd) && html && html->length) {
			if(!rangeEnd) rangeEnd = html->length - 1;
			if(rangeStart < 0) rangeStart = html->length + rangeStart;
			if(rangeStart < html->length && rangeEnd < html->length && rangeStart <= rangeEnd) {
				statusCode = 206;
			}
			else {
				if(ifModifiedSince && lastModified && rangeStart == html->length) {
					struct tm time_ = {};
					gmtime_r(&lastModified,&time_);
					time_t tmp = mktime(&time_);
					if(ifModifiedSince >= tmp) {
						sendResponse(304, "Not Modified", socketToClient);
						log_printf(5,"not modified!\n");
						statusCode = 304;
						goto last;
					}
				}
				sendResponse(416, "Requested range not satisfiable", socketToClient);
				statusCode = 416;
				goto last;
			}
		}
	}

resp:
	if(statusCode == 206) sendBasicHeaders(statusCode,"Partial Content",socketToClient);
	else sendBasicHeaders(statusCode,"OK",socketToClient);
	if(0 >= socketToClient->writeString("Content-Type: text/plain\r\n")) goto last;
	if(0 >= socketToClient->writeString("Accept-Ranges: bytes\r\n")) goto last;
	if(statusCode == 206) {
		std::ostringstream ss;
		ss << "Content-Range: bytes " << rangeStart << "-" << rangeEnd << "/" << html->length << "\r\n";
		if (0 >= socketToClient->writeString(ss.str())) goto last;
		//fprintf(stderr,"Content-Length: %ld\r\n",rangeEnd - rangeStart + 1);
		//fprintf(stderr,"Content-Range: bytes %ld-%ld/%ld\r\n",rangeStart,rangeEnd,(long)html->length);
		DataStorage *newHtml = new DataStorage();
		newHtml->appendBytes(html->bytes+rangeStart, rangeEnd - rangeStart + 1);
		delete html;
		html = newHtml;
	}
	{
		std::ostringstream ss;
		ss << "Content-Length: " << html->length << "\r\n";
		if(0 >= socketToClient->writeString(ss.str())) goto last;
	}
	if(lastModified) {
		struct tm time_ = {};
		char date[256];
		gmtime_r(&lastModified,&time_);
		strftime(date,256,httpTimestampFmt,&time_);
		std::string header = "Last-Modified: ";
		header += date;
		header += "\r\n";
		if(0 >= socketToClient->writeString(header)) goto last;
		//fprintf(stderr,"Last-Modified: %s\r\n",date);
	}
	if(0 > socketToClient->writeString("\r\n")) goto last;
	
	if(html && statusCode >= 200 && statusCode < 300 && strcasecmp(method, "HEAD")) {
		if(html->length > socketToClient->write(html->bytes, html->length)) goto last;
	}
	
last:
	if(buf) free(buf);
	if(html) delete html;
	return statusCode;
}

DataStorage *BBS2chProxyConnection::html2dat_old(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[32];
	char title[1024];
	int cachedSize = 0;
	bool bbspink = strstr(threadKey.c_str(),"bbspink.com") ? true : false;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<title>", 7);
	if(!ptr) {
		delete txt;
		return NULL;
	}
	ptr += 7;
	while(1) {
		if(*ptr == '<') {
			if(!strncasecmp(ptr,"</title>",8)) {
				ptr += 8;
				break;
			}
			else title[i++] = *ptr++;
		}
		else title[i++] = *ptr++;
	}
	title[i] = 0;
	
	snprintf(signature,32,"<dt>%d ",res);
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		ptr = strstr(ptr,signature);
		ptr += strlen(signature);
		while(*ptr != '<') ptr++;
		ptr++;
		const char *endStr;
		if(*ptr == 'a' || *ptr == 'A') {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			endStr = "</a>";
		}
		else if(*ptr == 'b') {
			endStr = NULL;
		}
		else {
			endStr = "</font>";
		}
		
		if(endStr) {
			ptr = strstr(ptr,"<b>");
			ptr += 3;
		}
		else {
			ptr = strchr(ptr,'>');
			ptr++;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"</b>",4) && (!endStr || !strncasecmp(ptr+4,endStr,strlen(endStr)))) {
					ptr += 4;
					if(endStr) ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr += 2;
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<dd>",4)) {
					ptr += 4;
					break;
				}
				else if(!strncmp(ptr,"<a href=\"javascript:be(",23)) {
					memcpy(date+i,"BE:",3);
					ptr += 23;
					i += 3;
					while(*ptr != ')') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<br><br>\n",9)) {
					ptr += 9;
					break;
				}
				else if(!strncasecmp(ptr,"<dt>",4) || !strncasecmp(ptr,"</dl>",5)) {
					while(i>0 &&body[i-1] == '\n') i--;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"http",13)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(body+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10) || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!bbspink && !strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else if(!bbspink && *ptr == ' ') {
				if(*(ptr+1) == ' ') ptr++;
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		snprintf(signature,32,"<dt>%d ",res);
		if(!memmem_priv(ptr, end-ptr+1, signature, strlen(signature))) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				snprintf(formattedDate,256,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

DataStorage *BBS2chProxyConnection::html2dat(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[64];
	char title[1024];
	int cachedSize = 0;
	char signatureTag[32];
	char closeTag[32];
	int closeTagLen;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<h1 class=\"title\">", 18);
	if(!ptr) {
		delete txt;
		return html2dat_old(html, startResNum, lastModified, useCache);
	}
	else {
		char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, " class=\"post\"", 13);
		if(ptr2) {
			char *tmp = ptr2;
			*ptr2 = 0;
			while(*ptr2 != '<') ptr2--;
			strcpy(signatureTag, ptr2);
			*tmp = ' ';
		}
		else {
			delete txt;
			return NULL;
		}
		/*char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, "<dl class=\"post\"", 16);
		if(ptr2) {
			delete txt;
			return html2dat_pink(html, startResNum, lastModified, useCache);
		}*/
	}
	
	ptr += 18;
	while(1) {
		if(*ptr == '<') {
			if(!strncasecmp(ptr,"</h1>",5)) {
				ptr += 5;
				break;
			}
			else title[i++] = *ptr++;
		}
		else if(*ptr == '\n') break;
		else title[i++] = *ptr++;
	}
	title[i] = 0;
	
	snprintf(signature,32,"%s class=\"post\" id=\"%d\"",signatureTag,res);
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		ptr = strstr(ptr," class=\"name\"><b>");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			snprintf(closeTag,32,"</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			ptr = tmp + 17;
		}
		else {
			delete resData;
			break;
		}
		
		char endStr[64];
		if(!strncmp(ptr,"<a href=\"mailto:",16)) {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(1) {
					if(*ptr == '<' && !strncmp(ptr,"<a href=\"",9)) {
						ptr = strchr(ptr,'>');
						ptr++;
						char *link = ptr;
						ptr = strstr(link,"</a>");
						memcpy(mail+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
					else if(*ptr == '"') break;
					else mail[i++] = *ptr++;
				}
				//while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			snprintf(endStr,64,"</a></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}
		/* we do not have to handle this special case because read.cgi on bbspink doesn't
		   emit font tags anymore and it conflicts with text decorations using "melon point" */
		/*else if(!strncmp(ptr,"<font",5)) {
			snprintf(endStr,64,"</font></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}*/
		else {
			snprintf(endStr,64,"</b>%s",closeTag);
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncmp(ptr,endStr,strlen(endStr))) {
					ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"",9)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(name+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr = strstr(ptr," class=\"date\">");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			snprintf(closeTag,32,"</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			ptr = tmp + 14;
		}
		else {
			delete resData;
			break;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		if(!strncmp(ptr,"<div class=\"uid",15) || !strncmp(ptr,"<span class=\"uid",16)) {
			char *tmp = ptr+1;
			while(*ptr != ' ') ptr++;
			*ptr = 0;
			snprintf(closeTag,32,"</%s>",tmp);
			closeTagLen = strlen(closeTag);
			ptr += 11;
			while(*ptr != '>') ptr++;
			ptr++;
			date[i++] = ' ';
			while(1) {
				if(*ptr == '<') {
					if(!strncasecmp(ptr,closeTag,closeTagLen)) {
						ptr += closeTagLen;
						break;
					}
					else date[i++] = *ptr++;
				}
				else date[i++] = *ptr++;
			}
		}
		
		if(!strncmp(ptr,"<div class=\"be",14) || !strncmp(ptr,"<span class=\"be",15)) {
			ptr += 14;
			while(*ptr != '>') ptr++;
			ptr++;
			if(!strncmp(ptr,"<a href=\"",9)) {
				ptr += 9;
				while(*ptr != '/' && *ptr != '"') ptr++;
				if(*ptr == '/' && (!strncmp(ptr,"//be.2ch.net/user/",18) || !strncmp(ptr,"//be.5ch.net/user/",18))) {
					memcpy(date+i," BE:",4);
					i += 4;
					ptr += 18;
					while(*ptr != '"') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
			}
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		if(!strcmp(signatureTag,"<div")) {
			ptr = strstr(ptr,"<div class=\"message\">");
			if(!ptr) {
				delete resData;
				break;
			}
			else {
				ptr += 21;
				if(!strncasecmp(ptr,"<span class=\"escaped\">",22)) {
					if(!strncasecmp(ptr+22,"<span class=\"AA\">",17)) {
						strcpy(closeTag,"</span></span></div>");
						closeTagLen = 20;
						ptr += 22+17;
					}
					else {
						strcpy(closeTag,"</span></div>");
						closeTagLen = 13;
						ptr += 22;
					}
				}
				else {
					strcpy(closeTag,"</div>");
					closeTagLen = 6;
				}
			}
		}
		else {
			ptr = strstr(ptr,"<dd class=\"thread_in\">");
			if(!ptr) {
				delete resData;
				break;
			}
			strcpy(closeTag,"</dd>");
			closeTagLen = 5;
			ptr += 22;
		}
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a ",3)) {
					char *tmp = strchr(ptr,'>');
					char *href = (char *)memmem_priv(ptr,tmp-ptr,"href=\"",6);
					char *link = tmp+1;
					if(href && !strncmp(link,"&gt;&gt;",8) && memmem_priv(href,link-href,"test/read.cgi/",14)) {
						while(ptr < link) {
							if(!strncmp(ptr," class=\"",8)) {
								ptr += 8;
								while(*ptr != '"' && *ptr != '>') ptr++;
								if(*ptr == '"') ptr++;
							}
							else body[i++] = *ptr++;
						}
					}
					else {
						ptr = strstr(link,"</a>");
						memcpy(body+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10)  || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		snprintf(signature,64,"%s class=\"post\" id=\"",signatureTag);
		ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
		if(ptr) {
			int next = atoi(ptr+strlen(signature));
			if(next >= res) {
				while(next > res) {
					txt->appendBytes("broken<><>broken<> broken <>\n", 29);
					res++;
				}
			}
			else ptr = NULL;
		}
		if(!ptr) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				snprintf(formattedDate,256,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

int BBS2chProxyConnection::datProxyAPI(const char *url, const char *method)
{
	long statusCode = 0;
	std::string postBody = auth->requestBodyForURL(url);
	if(postBody.empty()) {
		sendResponse(401, "Unauthorized", socketToClient);
		return 401;
	}
	CURL *curl = curl_easy_init();
	char *buf = (char *)malloc(16384);
	if(curl) {
		CURLcode res;
		struct curl_slist *headers = NULL;
		DataStorage *receivedHeader = new DataStorage();
		DataStorage *receivedBody = new DataStorage();
		while(socketToClient->readLine(buf, 16384)) {
			//fprintf(stderr,"%s",buf);
			if(!strcmp("\r\n",buf)) break;
			else if(!strncasecmp("Range:",buf,6)
				 || !strncasecmp("If-Modified-Since:",buf,18)
				 || !strncasecmp("Accept-Encoding:",buf,16)) {
				char *ptr;
				ptr = strchr(buf, '\r');
				if(!ptr) ptr = strchr(buf, '\n');
				if(!ptr) continue;
				*ptr = 0;
				headers = curl_slist_append(headers, buf);
			}
		}
		if(x_2ch_ua_dat) headers = curl_slist_append(headers, x_2ch_ua_dat);
		
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, receivedBody);
		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_download);
		curl_easy_setopt(curl, CURLOPT_HEADERDATA, receivedHeader);
		curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		curl_easy_setopt(curl, CURLOPT_USERAGENT, api_ua_dat?api_ua_dat:"");
		curl_easy_setopt(curl, CURLOPT_POST, 1L);
#if LIBCURL_VERSION_NUM >= 0x071101
		curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, postBody.c_str());
#else
		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postBody.c_str());
#endif
		//return;
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			receivedHeader->appendBytes("",1);
			char *ptr = strstr(receivedHeader->bytes,"\r\n\r\n");
			*(ptr+4) = 0;
			int threadStatus = 0;
			for(char *ptr2 = receivedHeader->bytes; ptr2<ptr;) {
				if(!strncasecmp(ptr2, "Thread-Status:", strlen("Thread-Status:"))) {
					ptr2 += strlen("Thread-Status:");
					while(*ptr2 == ' ') ptr2++;
					threadStatus = atoi(ptr2);
					break;
				}
				ptr2 = strstr(ptr2, "\r\n") + 2;
			}
			if(threadStatus == 1) {
				if(ptr+4-receivedHeader->bytes > socketToClient->write(receivedHeader->bytes, ptr+4-receivedHeader->bytes)) goto last;
				if(receivedBody->length > socketToClient->write(receivedBody->bytes, receivedBody->length)) goto last;
				goto last;
			}
			else if(threadStatus == 8) {
				sendBasicHeaders(302, "Found", socketToClient);
				if(0 >= socketToClient->writeString("Location: http://www2.2ch.net/live.html\r\n")) goto last;
				if(0 >= socketToClient->writeString("\r\n")) goto last;
				statusCode = 302;
				goto last;
			}
			else {
				if (statusCode < 400) {
					sendResponse(401, "Unauthorized", socketToClient);
					statusCode = 401;
				}
				else {
					sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
				receivedBody->appendBytes("",1);
				if(!strncasecmp(receivedBody->bytes,"ng (",4)) {
					log_printf(0,"API gateway returned error: %s\n",receivedBody->bytes);
				}
			}
			//fprintf(stderr,"%ld\n",statusCode);
		}
		else {
			log_printf(0,"curl error: %s\n",curl_easy_strerror(res));
			sendResponse(503, "Service Unavailable", socketToClient);
			statusCode = 503;
		}
last:
		curl_easy_cleanup(curl);
		curl_slist_free_all(headers);
		delete receivedBody;
		delete receivedHeader;
	}
	free(buf);
	return statusCode;
}

int BBS2chProxyConnection::bbsmenuProxy(const char *url, const char *method)
{
	long statusCode = 0;
	DataStorage *dat = new DataStorage();
	DataStorage *outHTML = new DataStorage();
	char *buf = (char *)malloc(16384);
	char userAgentFromHeader[1024] = "";
	CURL *curl = NULL;
	if(!buf) goto last;
	
	while(socketToClient->readLine(buf, 16384)) {
		//fprintf(stderr,"%s",buf);
		if(!strcmp("\r\n",buf)) break;
		else if(!strncasecmp("User-Agent:",buf,11)) {
			int i=0;
			char *ptr = buf+12;
			while(*ptr != '\r' && *ptr != '\n' && i < 1023) userAgentFromHeader[i++] = *ptr++;
			userAgentFromHeader[i] = 0;
		}
	}
	curl = curl_easy_init();
	if(curl) {
		CURLcode res;
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_ENCODING, "");
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if(user_agent) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
		}
		else if(userAgentFromHeader[0]) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgentFromHeader);
		}
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			if(statusCode == 200 && dat->length) {
				dat->appendBytes("",1);
				dat->length--;
				char *ptr = dat->bytes;
				while(*ptr) {
					if(!strncasecmp(ptr,"<a href=",8)) {
						char *start = ptr;
						char *end = strchr(ptr+8,'>');
						ptr = strstr(ptr+8,"://");
						if(ptr && ptr < end) {
							char *protocol = ptr;
							char *end2 = strchr(ptr+3,'/');
							ptr = strstr(protocol+3,"5ch.net");
							if(ptr && ptr < end2 && (*(ptr-1)=='.' || *(ptr-1)=='/')) {
								memcpy(ptr,"2ch",3);
								if(*(protocol-1) == 's') outHTML->appendBytes(start, protocol-start-1);
								else outHTML->appendBytes(start, protocol-start);
								outHTML->appendBytes(protocol, end-protocol);
								ptr = end;
								continue;
							}
							ptr = strstr(protocol+3,"bbspink.com");
							if(ptr && ptr < end2 && (*(ptr-1)=='.' || *(ptr-1)=='/')) {
								if(*(protocol-1) == 's') outHTML->appendBytes(start, protocol-start-1);
								else outHTML->appendBytes(start, protocol-start);
								outHTML->appendBytes(protocol, end-protocol);
								ptr = end;
								continue;
							}
						}
						ptr = start;
					}
					outHTML->appendBytes(ptr++, 1);
				}
			}
		}
		else {
			log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),buf);
			statusCode = 503;
		}
	}
	if(statusCode == 200) {
		std::ostringstream ss;
		ss << "Content-Length: " << outHTML->length << "\r\n";
		sendBasicHeaders(statusCode,"OK",socketToClient);
		if(0 >= socketToClient->writeString("Content-Type: text/html\r\n")) goto last;
		if(0 >= socketToClient->writeString(ss.str())) goto last;
		if(0 >= socketToClient->writeString("\r\n")) goto last;
		if(strcasecmp(method, "HEAD")) {
			if(outHTML->length > socketToClient->write(outHTML->bytes, outHTML->length)) goto last;
		}
	}
	else {
		sendResponse(503, "Service Unavailable", socketToClient);
		statusCode = 503;
	}
	
last:
	if(curl) curl_easy_cleanup(curl);
	if(buf) free(buf);
	if(dat) delete dat;
	if(outHTML) delete outHTML;
	return statusCode;
}

int BBS2chProxyConnection::bbsCgiProxy(const char *url, const char *protocol, std::string &hostStr, void *requestHeaders, const char *requestBody)
{
	CURL *curl = curl_easy_init();
	long statusCode = 0;
	curl_slist *headers = (curl_slist *)requestHeaders;
	std::string boardStr;
	std::string threadStr;
#ifdef USE_LUA
	std::map<std::string, std::string> headersForLua;
	if (lua_script) {
		curl_slist *current = headers;
		while (current) {
			const char *delim = strchr(current->data, ':');
			if (delim) {
				const char *ptr = current->data;
				while (*ptr != ' ' && ptr < delim) ptr++;
				std::string name(current->data, ptr-current->data);
				ptr = delim + 1;
				while (*ptr == ' ') ptr++;
				headersForLua[name] = std::string(ptr);
			}
			current = current->next;
		}
		if (user_agent) {
			headersForLua["User-Agent"] = std::string(user_agent);
		}
	}
#endif

	if (requestBody && (lua_script || !bbscgi_headers.empty() || !bbscgi_postorder.empty())) {
		std::map<std::string, std::string> fields;
		const char *ptr = requestBody;
		size_t bodyLength = 0;
		while (1) {
			const char *tmp = ptr;
			while (*tmp != '=' && *tmp != 0) tmp++;
			if (*tmp == 0) {
				bodyLength = tmp - requestBody;
				break;
			}
			std::string key(ptr, tmp-ptr);
			tmp++;
			ptr = tmp;
			while (*tmp != '&' && *tmp != 0) tmp++;
			std::string value(ptr, tmp-ptr);
			fields.insert(std::make_pair(key, value));
			if (*tmp == 0) {
				bodyLength = tmp - requestBody;
				break;
			}
			ptr = tmp + 1;
		}
		if (fields.count("bbs")) boardStr = fields["bbs"];
		if (fields.count("key")) threadStr = fields["key"];
		if (!bbscgi_postorder.empty()) {
			std::string newBody;
			for (std::vector<std::string>::iterator it = bbscgi_postorder.begin(); it != bbscgi_postorder.end(); it++) {
				std::string &name = *it;
				if (fields.count(name)) {
					if (!newBody.empty()) newBody.append("&");
					newBody.append(name);
					newBody.append("=");
					newBody.append(fields[name]);
					fields.erase(name);
				}
			}
			for (std::map<std::string, std::string>::iterator it = fields.begin(); it != fields.end(); it++) {
				if (!newBody.empty()) newBody.append("&");
				newBody.append(it->first);
				newBody.append("=");
				newBody.append(it->second);
			}
			if (bodyLength == newBody.length()) {
				strcpy((char *)requestBody, newBody.c_str());
				log_printf(1, "Reordered request body is: %s\n", requestBody);
			}
			else {
				log_printf(0, "Error occured while reordering the request body - skipping\n");
			}
		}
	}
	if (!bbscgi_headers.empty() || lua_script) {
		char buf[1024];
		for (std::map<std::string, std::string>::iterator it = bbscgi_headers.begin(); it!=bbscgi_headers.end(); it++) {
			/* we cannot use a reference here, because the original string shouldn't be replaced */
			std::string value = it->second;
			if (!hostStr.empty()) {
				std::string::size_type pos = value.find("%HOST%");
				while (pos != std::string::npos) {
					value.replace(pos, 6, hostStr);
					pos = value.find("%HOST%", pos+hostStr.length());
				}
			}
			if (!boardStr.empty()) {
				std::string::size_type pos = value.find("%BOARD%");
				while (pos != std::string::npos) {
					value.replace(pos, 7, boardStr);
					pos = value.find("%BOARD%", pos+boardStr.length());
				}
			}
			if (!threadStr.empty()) {
				std::string::size_type pos = value.find("%THREAD%");
				while (pos != std::string::npos) {
					value.replace(pos, 8, threadStr);
					pos = value.find("%THREAD%", pos+threadStr.length());
				}
			}
			snprintf(buf,1024,"%s: %s",it->first.c_str(),value.c_str());
			headers = curl_slist_append(headers, buf);
			log_printf(1, "Appended custom header \"%s\"\n", buf);
#ifdef USE_LUA
			if (lua_script) headersForLua[it->first] = value;
#endif
		}
	}

	for (int run=0; run<2; run++) {
		curl_slist *_headers = headers;
		char *_body = (char *)requestBody;
		status = 0;
		monaKeyForRequest = "";
#ifdef USE_LUA
		if (lua_script) {
			lua_State* l = luaL_newstate();
			luaL_openlibs(l);
			if (luaL_loadfile(l, lua_script) != LUA_OK) {
				log_printf(0, "Lua: Failed to open script %s:\n  %s\n", lua_script, lua_tostring(l, -1));
				goto lua_end;
			}
			lua_newtable(l);
			lua_pushstring(l, "hmacSHA256");
			lua_pushcfunction(l, lua_hmacSHA256);
			lua_settable(l, -3);
			lua_pushstring(l, "decodeURIComponent");
			lua_pushcfunction(l, lua_decodeURIComponent);
			lua_settable(l, -3);
			lua_pushstring(l, "encodeURIComponent");
			lua_pushcfunction(l, lua_encodeURIComponent);
			lua_settable(l, -3);
			lua_pushstring(l, "convertShiftJISToUTF8");
			lua_pushcfunction(l, lua_convertShiftJISToUTF8);
			lua_settable(l, -3);
			lua_pushstring(l, "isExpiredKey");
			lua_pushcfunction(l, lua_isExpiredKey);
			lua_settable(l, -3);
			lua_pushstring(l, "monaKey");
			lua_pushstring(l, getMonaKey().c_str());
			lua_settable(l, -3);
			lua_setglobal(l, "proxy2ch");
			if (lua_pcall(l, 0, 0, 0) != LUA_OK) {
				log_printf(0, "Lua: Failed to run script %s:\n  %s\n", lua_script, lua_tostring(l, -1));
				goto lua_end;
			}
			lua_getglobal(l, "willSendRequestToBbsCgi");
			if (!lua_isfunction(l, -1)) {
				log_printf(0, "Lua: willSendRequestToBbsCgi function does not exist in the script\n");
				goto lua_end;
			}
			lua_newtable(l);
			lua_pushstring(l, "headers");
			lua_newtable(l);
			for (std::map<std::string, std::string>::iterator it = headersForLua.begin(); it!=headersForLua.end(); it++) {
				lua_pushstring(l, it->first.c_str());
				lua_pushstring(l, it->second.c_str());
				lua_settable(l, -3);
			}
			lua_settable(l, -3);
			lua_pushstring(l, "body");
			lua_pushstring(l, _body);
			lua_settable(l, -3);
			lua_pushstring(l, hostStr.c_str());
			lua_pushstring(l, boardStr.c_str());
			lua_pushstring(l, threadStr.c_str());
			if (lua_pcall(l, 4, 1, 0) != LUA_OK) {
				log_printf(0, "Lua: Failed to call willSendRequestToBbsCgi function:\n  %s\n", lua_tostring(l, -1));
				goto lua_end;
			}
			if (!lua_istable(l, -1)) {
				log_printf(0, "Lua: A return type of willSendRequestToBbsCgi function should be a table\n");
				goto lua_end;
			}
			lua_pushstring(l, "body");
			lua_rawget(l, -2);
			if (lua_isstring(l, -1)) {
				const char *newBody = lua_tostring(l, -1);
				_body = strdup(newBody);
				log_printf(1, "Lua: Set request body \"%s\"\n", newBody);
			}
			lua_pop(l, 1);
			lua_pushstring(l, "headers");
			lua_rawget(l, -2);
			if (lua_istable(l, -1)) {
				_headers = NULL;
				lua_pushnil(l);
				while (lua_next(l, -2)) {
					if (lua_isstring(l, -1) && lua_isstring(l, -2)) {
						std::string header = lua_tostring(l, -2);
						header += ": ";
						header += lua_tostring(l, -1);
						_headers = curl_slist_append(_headers, header.c_str());
						log_printf(1, "Lua: Set request header \"%s\"\n", header.c_str());
					}
					lua_pop(l, 1);
				}
			}
lua_end:
			lua_close(l);
		}
#endif
		do {
			bool isPink = hostStr.find("bbspink.com") != std::string::npos;
			bool shouldSign = appKey && (((api_mode & 2) && !isPink) || (api_mode & 4));
			bool hasSignature = false;
			bool hasExpect = false;
			bool hasAccept = false;
			std::string userAgent = user_agent ? user_agent : "";
			curl_slist *current = _headers;
			while (current) {
				if (!strncasecmp(current->data, "X-MonaKey", 9)) {
					char *ptr = current->data + 9;
					while (*ptr == ' ') ptr++;
					if (*ptr++ != ':') goto next;
					while (*ptr == ' ') ptr++;
					monaKeyForRequest = std::string(ptr);
				}
				else if (shouldSign && !strncasecmp(current->data, "X-PostSig", 9)) {
					char *ptr = current->data + 9;
					while (*ptr == ' ') ptr++;
					if (*ptr++ != ':') goto next;
					hasSignature = true;
				}
				else if (shouldSign && !strncasecmp(current->data, "User-Agent", 10)) {
					char *ptr = current->data + 10;
					while (*ptr == ' ') ptr++;
					if (*ptr++ != ':') goto next;
					while (*ptr == ' ') ptr++;
					userAgent = std::string(ptr);
				}
				else if (!strncasecmp(current->data, "Expect", 6)) {
					char *ptr = current->data + 6;
					while (*ptr == ' ') ptr++;
					if (*ptr++ != ':') goto next;
					hasExpect = true;
				}
				else if (!strncasecmp(current->data, "Accept", 6)) {
					char *ptr = current->data + 6;
					while (*ptr == ' ') ptr++;
					if (*ptr++ != ':') goto next;
					hasAccept = true;
				}
next:
				current = current->next;
			}
			if (shouldSign && !hasSignature) {
				if (!userAgent.empty()) {
					monaKeyForRequest = getMonaKey();
					appendPostSignature(_body, userAgent, monaKeyForRequest, &_headers);
				} else {
					log_printf(0, "API: User-Agent muse be set explicitly to post with API.\n");
				}
			}
			if (!monaKeyForRequest.empty() && monaKeyIssueTime.count(monaKeyForRequest)) {
				double issued = monaKeyIssueTime[monaKeyForRequest];
				double now = getCurrentTime();
				double wait = 4.0 - (now - issued);
				if (wait > 0) {
					log_printf(1, "Sleeping for %.1f seconds to avoid posting too fast...\n", wait);
#ifdef _WIN32
					Sleep(wait * 1e+3);
#else
					usleep(wait * 1e+6);
#endif
				}
				monaKeyIssueTime.erase(monaKeyForRequest);
			}
			if (!hasExpect) _headers = curl_slist_append(_headers, "Expect:");
			if (!hasAccept) _headers = curl_slist_append(_headers, "Accept:");
		} while (0);

		if (curl) {
			CURLcode res;
			if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
			curl_easy_setopt(curl, CURLOPT_URL, url);
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			if (run == 0)
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_bbscgi);
			else
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
			curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
			curl_easy_setopt(curl, CURLOPT_POST, 1L);
			curl_easy_setopt(curl, CURLOPT_POSTFIELDS, _body);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
			if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			if (!strncasecmp(protocol,"HTTP/1.0",8) && !strncasecmp(url, "http://", 7)) {
				curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
			}
			else {
				/* force use HTTP 1.1 because CURL_HTTP_VERSION_2TLS is used on curl (w/ nghttp2) >= 7.62.0 */
				curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
			}
			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _headers);
			if (user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			if (proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			res = curl_easy_perform(curl);
			if (res != CURLE_OK) {
				if (res == CURLE_WRITE_ERROR && status == 2) {
					log_printf(1, "MonaKey should be reset. Sending the same request automatically...\n");
					curl_easy_reset(curl);
					if (_headers != headers) curl_slist_free_all(_headers);
					if (_body != requestBody) free(_body);
					continue;
				}
				else {
					log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),url);
					if (!status) sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
			}
			else {
				if (chunked) {
					socketToClient->writeString("0\r\n\r\n");
				}
				curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
			}
			curl_easy_cleanup(curl);
		}
		if (_headers != headers) curl_slist_free_all(_headers);
		if (_body != requestBody) free(_body);
		break;
	}
	if (requestHeaders == NULL && headers != NULL) curl_slist_free_all(headers);
	return statusCode;
}