proxy2ch/BBS2chProxyConnection.cpp

#include <string>
#include <vector>
#include <map>
#include <set>
#include <sstream>
#include <stdexcept>
#include <algorithm>
#include <pthread.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#ifdef USE_LUA
#include <lua.hpp>
#endif
#ifdef _WIN32
#include <fcntl.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <mswsock.h>
#define CLOSESOCKET(x) closesocket(x)
#define gmtime_r(a, b) gmtime_s(b, a)
#else
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <poll.h>
#define CLOSESOCKET(x) close(x)
#endif
#include "BBS2chProxyConnection.h"
#include "DataStorage.h"
#include "hmac.h"
#include "stringEncodingConverter.h"
#include "BBS2chProxyRawSocket.h"
#ifdef USE_MITM
#include "BBS2chProxySecureSocket.h"
#endif
//#define DEBUG 1

extern char *proxy_server;
extern long proxy_port;
extern long proxy_type;
extern long timeout;
extern char *user_agent;
extern char *api_ua_dat;
extern char *x_2ch_ua_dat;
extern char *appKey;
extern char *hmacKey;
extern int allow_chunked;
extern int curl_features;
extern unsigned int curl_version_number;
extern bool accept_https;
extern int force_5chnet;
extern int force_5chnet_https;
extern int force_ipv4;
extern char *bbsmenu_url;
extern char *api_server;
extern std::map<std::string, std::string> bbscgi_headers;
extern int gikofix;
extern CURLSH *curl_share;
extern char *lua_script;
extern unsigned int api_mode;
extern std::vector<std::string> bbscgi_postorder;
extern unsigned int bbscgi_utf8;
extern int api_override;
#ifdef USE_MITM
extern unsigned int mitm_mode;
#endif

extern void log_printf(int level, const char *format ...);

#include "utils.h"

BBS2chProxyKeyManager BBS2chProxyConnection::keyManager;
BBS2chProxyAuth BBS2chProxyConnection::auth;

static regex_t regex;
static regex_t regex_kako;
static regex_t regex_offlaw;
static regex_t regex_api;
static regex_t regex_api_auth;

#ifdef USE_LUA
extern "C" {
	static int lua_hmacSHA256(lua_State *l)
	{
		static const char *table = "0123456789abcdef";
		size_t keyLength, dataLength;
		const char *key = luaL_checklstring(l, 1, &keyLength);
		const char *data = luaL_checklstring(l, 2, &dataLength);
		if (!key || !data) return 0;
		unsigned char digest[32];
		char digestStr[65];
		proxy2ch_HMAC_SHA256(key, keyLength, data, dataLength, digest);
		for (int i=0; i<32; i++) {
			unsigned char c = digest[i];
			unsigned char upper = (c >> 4) & 0xf;
			unsigned char lower = c & 0xf;
			digestStr[i*2] = table[upper];
			digestStr[i*2+1] = table[lower];
		}
		digestStr[64] = 0;
		lua_pushstring(l, digestStr);
		return 1;
	}
	static int lua_decodeURIComponent(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		bool decodePlus = true;
		if (!lua_isnoneornil(l, 2)) {
			decodePlus = (lua_toboolean(l, 2));
		}

		std::string output = decodeURIComponent(input, length, decodePlus);
		lua_pushstring(l, output.c_str());
		return 1;
	}
	static int lua_encodeURIComponent(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		bool spaceAsPlus = true;
		if (!lua_isnoneornil(l, 2)) {
			spaceAsPlus = (lua_toboolean(l, 2));
		}

		std::string output = encodeURIComponent(input, length, spaceAsPlus);
		lua_pushstring(l, output.c_str());
		return 1;
	}
	static int lua_convertShiftJISToUTF8(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		if (length > 0) {
			char *output = convertShiftJISToUTF8(input, length);
			if (!output) lua_pushnil(l);
			else {
				lua_pushstring(l, output);
				free(output);
			}
		}
		else lua_pushstring(l, "");
		return 1;
	}
	static int lua_isExpiredKey(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		if (BBS2chProxyConnection::keyManager.isExpired(input)) {
			lua_pushboolean(l, 1);
		}
		else lua_pushboolean(l, 0);
		return 1;
	}
	static int lua_isValidAsUTF8(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		lua_pushboolean(l, isValidAsUTF8(input, length));
		return 1;
	}
	static int lua_getMonaKey(lua_State *l)
	{
		size_t length;
		const char *input = luaL_checklstring(l, 1, &length);
		if (!input) return 0;
		const std::string &key = BBS2chProxyConnection::keyManager.getKey(input);
		lua_pushstring(l, key.c_str());
		return 1;
	}
}
#endif

void BBS2chProxyConnection::run(void * (*func)(void *))
{
	pthread_t thread;
	pthread_attr_t thread_attr;
	pthread_attr_init(&thread_attr);
	pthread_attr_setdetachstate(&thread_attr , PTHREAD_CREATE_DETACHED);
	if(0 != pthread_create(&thread , &thread_attr , func , this))
		perror("pthread_create");
	pthread_attr_destroy(&thread_attr);
}

struct TunnelSockets {
	int sock_c;
	int sock_s;
	std::string addr;
	int port;
};

static void *tunnelMain(void *param)
{
	TunnelSockets *sockets = (TunnelSockets *)param;
	char *buf = new char[16384];
#ifdef _WIN32
	fd_set fds;
	int nfds = sockets->sock_c > sockets->sock_s ? sockets->sock_c + 1 : sockets->sock_s + 1;
#else
	struct pollfd fds[2];
	memset(fds, 0, sizeof(fds));
	fds[0].fd = sockets->sock_c;
	fds[0].events = POLLIN;
	fds[1].fd = sockets->sock_s;
	fds[1].events = POLLIN;
#endif
	while (1) {
#ifdef _WIN32
		FD_ZERO(&fds);
		FD_SET(sockets->sock_c, &fds);
		FD_SET(sockets->sock_s, &fds);
		if (select(nfds, &fds, NULL, NULL, NULL) < 0) break;
		if (FD_ISSET(sockets->sock_c, &fds)) {
			int ret = recv(sockets->sock_c, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_s, buf, ret, 0);
			else if (ret <= 0) break;
		}
		if (FD_ISSET(sockets->sock_s, &fds)) {
			int ret = recv(sockets->sock_s, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_c, buf, ret, 0);
			else if (ret <= 0) break;
		}
#else
		if (poll(fds, 2, -1) < 0) break;
		if (fds[0].revents & POLLIN) {
			int ret = recv(sockets->sock_c, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_s, buf, ret, 0);
			else if (ret <= 0) break;
		}
		else if (fds[0].revents != 0) break;
		if (fds[1].revents & POLLIN) {
			int ret = recv(sockets->sock_s, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_c, buf, ret, 0);
			else if (ret <= 0) break;
		}
		else if (fds[1].revents != 0) break;
#endif
	}
	CLOSESOCKET(sockets->sock_c);
	CLOSESOCKET(sockets->sock_s);
	log_printf(1, "Finished tunneling to %s:%d\n", sockets->addr.c_str(), sockets->port);
	delete sockets;
	delete[] buf;
	return NULL;
}

int BBS2chProxyConnection::tunnel(const char *addr, int port)
{
	struct sockaddr_in server;
	memset(&server, 0, sizeof(server));
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = inet_addr(addr);
	server.sin_port = htons(port);
	if(server.sin_addr.s_addr == 0xffffffff) {
		struct hostent *host;
		host = gethostbyname(addr);
		if (host == NULL) {
			log_printf(0, "Failed to lookup hostname %s\n", addr);
			sendResponse(400, "Bad Request", socketToClient);
			return 400;
		}
		server.sin_addr.s_addr = *(unsigned int *)host->h_addr_list[0];
	 }
	
	log_printf(1,"Tunneling connection to %s:%d\n",addr,port);
	
	int sock_s = socket(AF_INET, SOCK_STREAM, 0);
	if(-1 == ::connect(sock_s, (struct sockaddr *)&server, sizeof(server))) {
		perror("connect");
		sendResponse(400, "Bad Request", socketToClient);
		return 400;
	}
	send(sock_c, "HTTP/1.1 200 Connection established\r\n\r\n", 39, 0);

	TunnelSockets *sockets = new TunnelSockets();
	sockets->sock_c = sock_c;
	sockets->sock_s = sock_s;
	sockets->addr = addr;
	sockets->port = port;
	pthread_t thread;
	if(0 != pthread_create(&thread, NULL, tunnelMain, sockets))
		perror("pthread_create");
	pthread_detach(thread);
	
	return 0;
}

void BBS2chProxyConnection::connect(void)
{
	char method[32], url[1024], protocol[32];
	int i;
	char *buf, *ptr;
	unsigned int datProxyMode = 0; // 0: no dat, 1: read.cgi or API, 2: force API, 3: kakolog
	regmatch_t match[7];
	long statusCode = 0;
	BBS2chProxyURL baseURL;
	BBS2chProxyHttpHeaders requestHeaders;
	
	socketToClient = new BBS2chProxyRawSocket(sock_c);
	
	buf = (char *)malloc(16384);
	if(!buf) goto end;
beginHandleRequest:
	ptr = buf;
	if(!socketToClient->readLine(buf, 1024)) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 32) method[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	method[i] = 0;
	ptr++;
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 1024) url[i++] = *ptr++;
	if(*ptr == 0 || i == 1024) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	url[i] = 0;
	ptr++;
	i=0;
	while(*ptr != '\r' && *ptr != '\n' && *ptr != 0 && i < 32) protocol[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	protocol[i] = 0;
	if(!strncasecmp(protocol,"HTTP/1.0",8)) {
		isClientHttp1_0 = true;
	}
	else isClientHttp1_0 = false;
	log_printf(1, "Received %s %s %s\n",method,url,protocol);
	
	if(strcasecmp(method,"GET") && strcasecmp(method,"POST") && strcasecmp(method,"HEAD") && strcasecmp(method,"CONNECT") && strcasecmp(method,"PUT") && strcasecmp(method, "OPTIONS")) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	if(!url[0]) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	if(strncasecmp(protocol,"HTTP",4)) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	
	if(!strcasecmp(method,"CONNECT")) {
		if(!accept_https || baseURL.isValid()) {
			sendResponse(400, "Bad Request", socketToClient);
			statusCode = 400;
			goto end;
		}
		while(socketToClient->readLine(buf, 16384)) {
			if(!strcmp("\r\n",buf)) break;
		}
		int port = 443;
		char *ptr = strchr(url, ':');
		if(ptr) {
			*ptr = 0;
			port = atoi(ptr+1);
		}
#ifdef USE_MITM
		bool useMITM = false;
		if (mitm_mode) {
			baseURL = BBS2chProxyURL("https", url);
			if (mitm_mode == 2) useMITM = true;
			else if (mitm_mode == 1 && baseURL.isFamilyOf5chNet()) useMITM = true;
		}
		if (useMITM) {
			socketToClient->writeString("HTTP/1.1 200 Connection established\r\n\r\n");
			if (port == 80) {
				baseURL.setScheme("http");
				goto beginHandleRequest;
			}
			else {
				try {
					BBS2chProxySecureSocket *secureSocket = new BBS2chProxySecureSocket(sock_c, url);
					delete socketToClient;
					socketToClient = secureSocket;
					if (port != 443) baseURL.setPort(port);
					isHttps = true;
					goto beginHandleRequest;
				} catch (const std::runtime_error& e) {
					log_printf(0, "%s\n", e.what());
					sendResponse(400, "Bad Request", socketToClient);
					statusCode = 400;
					goto end;
				}
			}
		}
		else
#endif
		{
			statusCode = tunnel(url, port);
			/* if a return value is non-zero, tunnel function failed to establish a tunnelling connection */
			if (statusCode == 0) {
				delete socketToClient;
				socketToClient = NULL;
			}
			goto end;
		}
	}
#if USE_MITM
	if (baseURL.isValid()) {
		requestURL = BBS2chProxyURL(baseURL, url);
		log_printf(1, "Running as MITM proxy for %s\n", requestURL.absoluteString().c_str());
	} else
#endif
	requestURL = BBS2chProxyURL(url);
	if (!requestURL.isHttp()) {
		sendResponse(400, "Bad Request", socketToClient);
		statusCode = 400;
		goto end;
	}
	if (force_5chnet) {
		if (requestURL.getHost() != "menu.2ch.net" && requestURL.replaceHost("2ch.net", "5ch.net")) {
			force5ch = true;
			log_printf(1, "Detected *.2ch.net URL, changed target URL to %s\n", requestURL.absoluteString().c_str());
		}
	}

	/* parse request headers */
	while (socketToClient->readLine(buf, 16384)) {
		if (!strcmp("\r\n",buf)) break;
		requestHeaders.add(buf);
	}
	if (requestHeaders.hasNameAndValue("Transfer-Encoding", "chunked")) {
		isClientChunked = true;
	}
	else if (requestHeaders.has("Content-Length")) {
		content_length = atoi(requestHeaders.get("Content-Length").c_str());
	}
	if (requestHeaders.has("Expect")) {
		if (!strcasecmp(requestHeaders.get("Expect").c_str(), "100-continue") && !isClientHttp1_0) {
			log_printf(1, "Received Expect: 100-continue header, sending 100 Continue response to the client\n");
			socketToClient->writeString("HTTP/1.1 100 Continue\r\n\r\n");
		}
	}
	
	if (regexec(&regex, requestURL.absoluteString().c_str(), 6, match, 0) != REG_NOMATCH) {
		if ((appKey && (api_mode & 1)) || !requestURL.hostStartsWith("headline.")) datProxyMode = 1;
	}
	else if (regexec(&regex_kako, requestURL.absoluteString().c_str(), 7, match, 0) != REG_NOMATCH) {
		datProxyMode = 3;
	}
	else if (regexec(&regex_offlaw, requestURL.absoluteString().c_str(), 5, match, 0) != REG_NOMATCH) {
		const char *tmp = requestURL.absoluteString().c_str();
		const char *thread = strstr(tmp, "key=");
		if (thread) {
			match[6].rm_so = thread+4-tmp;
			match[6].rm_eo = thread+4-tmp;
			const char *ptr = thread+4;
			while (*ptr != '&' && *ptr != 0) {
				ptr++;
				match[6].rm_eo++;
			}
			if (match[6].rm_so != match[6].rm_eo) datProxyMode = 3;
		}
	}
	else if (api_override && appKey) {
		if (regexec(&regex_api, requestURL.absoluteString().c_str(), 5, match, 0) != REG_NOMATCH) {
			datProxyMode = 2;
			match[1] = match[2];
			match[5] = match[4];
			match[4] = match[3];
		}
		else if (regexec(&regex_api_auth, requestURL.absoluteString().c_str(), 2, match, 0) != REG_NOMATCH) {
			/* return dummy response immediately */
			log_printf(1, "Returning dummy response because API overriding is enabled\n");
			statusCode = 200;
			sendBasicHeaders(200, "OK", socketToClient);
			if (0 >= socketToClient->writeString("Content-Type: text/plain\r\n")) goto end;
			if (0 >= socketToClient->writeString("Content-Length: 203\r\n\r\n")) goto end;
			if (0 >= socketToClient->writeString("SESSION-ID=Monazilla/1.00:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")) goto end;
			goto end;
		}
	}
	
	if (datProxyMode) {
		// match[1] : hostPrefix
		// match[2] : host
		// match[4] : bbs
		// match[5 or 6] : key
		const char *original = requestURL.absoluteString().c_str();
		const regmatch_t *threadMatch = datProxyMode != 3 ? match+5 : match+6;
		std::string board(original + match[4].rm_so, match[4].rm_eo - match[4].rm_so);
		std::string thread(original + threadMatch->rm_so, threadMatch->rm_eo - threadMatch->rm_so);
		if (!appKey || (datProxyMode == 1 && !(api_mode & 1)) || datProxyMode == 3) {
			log_printf(1, "Retrieving thread via read.cgi...\n");
			std::string host(original + match[1].rm_so, match[2].rm_eo - match[1].rm_so);
			threadKey = host;
			threadKey += '/';
			threadKey += board;
			threadKey += '/';
			threadKey += thread;
			std::string targetURL = (force_5chnet_https || isHttps) ? "https://" : "http://";
			targetURL += host;
			targetURL += "/test/read.cgi/";
			targetURL += board;
			targetURL += '/';
			targetURL += thread;
			targetURL += '/';
			if (force_5chnet_https) isHttps = true;
			statusCode = datProxy(targetURL.c_str(), method, requestHeaders);
		}
		else {
			log_printf(1, "Retrieving thread via API...\n");
			std::string hostPrefix(original + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
			std::string targetURL = "https://";
			targetURL += api_server;
			targetURL += "/v1/";
			targetURL += hostPrefix;
			targetURL += '/';
			targetURL += board;
			targetURL += '/';
			targetURL += thread;
			isHttps = true;
			statusCode = datProxyAPI(targetURL.c_str(), method, requestHeaders);
		}
	}
	else {
		if (force_5chnet_https && !isHttps && requestURL.isFamilyOf5chNet()) {
			requestURL.setScheme("https");
			isHttps = true;
			log_printf(1, "The host %s is 5ch.net family, connecting with HTTPS\n", requestURL.getHost().c_str());
		}
		
		if (bbsmenu_url && requestURL.equals(BBS2chProxyURL(bbsmenu_url), true)) {
			log_printf(1, "Running as a BBS menu proxy...\n");
			statusCode = bbsmenuProxy(requestURL.absoluteString().c_str(), method, requestHeaders);
		}
		else {
			bool isPostRequest = !strcasecmp(method, "POST");
			bool isPutRequest = !strcasecmp(method, "PUT");
			if (isPostRequest && requestURL.isFamilyOf5chNet() && requestURL.pathStartsWith("/test/bbs.cgi")) bbscgi = true;
			if (bbscgi) log_printf(1, "Looks like a request to bbs.cgi, will be modified before sending...\n");
			else log_printf(1, "Not a notable request, will be forwarded to server...\n");
			if (force_5chnet) {
				if (requestHeaders.has("Host")) {
					std::string host = requestHeaders.get("Host");
					size_t pos = host.find("2ch.net");
					if (pos != std::string::npos && pos+7 == host.length()) {
						if (pos == 0 || host[pos-1] == '.') {
							host.replace(pos, 1, "5");
							requestHeaders.set("Host", host);
						}
					}
				}
				if (bbscgi && requestHeaders.has("Referer")) {
					std::string referrer = requestHeaders.get("Referer");
					size_t pos = referrer.find("2ch.net");
					if (pos != std::string::npos) {
						if (pos == 0 || referrer[pos-1] == '.') {
							referrer.replace(pos, 1, "5");
							requestHeaders.set("Referer", referrer);
						}
					}
				}
			}
			requestHeaders.remove("Connection");
			if (user_agent) requestHeaders.remove("User-Agent");
			if (bbscgi && (content_length > 0 || isClientChunked)) {
				bool isNotFormURLEncoded = false;
				if (requestHeaders.has("Content-Type") && requestHeaders.get("Content-Type").find("application/x-www-form-urlencoded") == std::string::npos) {
					isNotFormURLEncoded = true;
				}
				if (!isNotFormURLEncoded) {
					requestHeaders.remove("Content-Length");
					if (!bbscgi_headers.empty()) {
						for (std::map<std::string, std::string>::iterator it = bbscgi_headers.begin(); it != bbscgi_headers.end(); it++) {
							if (requestHeaders.has(it->first)) {
								log_printf(1, "Ignoring header \"%s\" because custom header will be appended\n", it->first.c_str());
								requestHeaders.remove(it->first);
							}
						}
					}
					char *postdata = NULL;
					if (isClientChunked) {
						content_length = readChunkedBodyIntoBuffer(&postdata, socketToClient);
						requestHeaders.remove("Transfer-Encoding");
					}
					else {
						postdata = (char *)calloc(content_length+1, 1);
						content_length = socketToClient->read(postdata, content_length);
					}
					if (gikofix) {
						char *ptr = postdata+content_length-1;
						while (ptr >= postdata && (*ptr == '\r' || *ptr == '\n')) {
							*ptr-- = 0;
						}
					}
					curl_slist *headersForCurl = NULL;
					headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl);
					statusCode = bbsCgiProxy(requestURL.absoluteString().c_str(), requestHeaders, postdata);
					free(postdata);
					curl_slist_free_all(headersForCurl);
					goto end;
				}
			}
			curl_slist *headersForCurl = NULL;
			headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl);
			if (!requestHeaders.has("Expect")) {
				headersForCurl = curl_slist_append(headersForCurl, "Expext:");
			}
			
			if(curl) {
				CURLcode res;
				if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
				curl_easy_setopt(curl, CURLOPT_URL, requestURL.absoluteString().c_str());
				curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
				curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
				if(content_length) {
					/* set Content-Length explicitly via API to work properly with curl >= 7.66.0 */
					if(isPostRequest)
						curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, content_length);
					else if(isPutRequest)
						curl_easy_setopt(curl, CURLOPT_INFILESIZE, content_length);
				}
				curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_READDATA, this);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
				//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
				if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
				curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
				curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headersForCurl);
				if(user_agent) {
					curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
				}
				if(isPostRequest) {
					curl_easy_setopt(curl, CURLOPT_POST, 1L);
				}
				else if(isPutRequest) {
					curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
				}
				else if(!strcasecmp(method, "HEAD")) {
					curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
				}
				else if(!strcasecmp(method, "OPTIONS")) {
					curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "OPTIONS");
				}
				if(proxy_server) {
					curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
					curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
					curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
				}
				res = curl_easy_perform(curl);
				if(res != CURLE_OK) {
					log_printf(0, "curl error: %s (%s)\n", curl_easy_strerror(res), requestURL.absoluteString().c_str());
					if(!status) sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
				else {
					if(isResponseChunked) {
						socketToClient->writeString("0\r\n\r\n");
					}
					curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
				}
				curl_easy_reset(curl);
			}
			curl_slist_free_all(headersForCurl);
		}
	}
	
  end:
	if(statusCode) log_printf(1, "Returned status code %d to client\n",statusCode);
	if(buf) free(buf);
	if(socketToClient) socketToClient->close();
}

int BBS2chProxyConnection::datProxy(const char *url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	DataStorage *html = NULL;
	long statusCode = 0;
	long rangeStart = 0, rangeEnd = 0;
	time_t lastModified = 0;
	time_t ifModifiedSince = 0;
	char *buf = (char *)malloc(16384);
	if(!buf) goto last;
	if(requestHeaders.has("Range")) {
		std::string value = requestHeaders.get("Range");
		if(value.find("bytes=") == 0 && value.find(",") == std::string::npos) {
			char *ptr = (char *)value.c_str() + 6;
			if(*ptr == '-') {
				rangeStart = atoi(ptr);
			}
			else {
				rangeStart = strtol(ptr, &ptr, 10);
				if(*ptr == '-') ptr++;
				if(*ptr && *ptr != '\r') {
					rangeEnd = strtol(ptr, NULL, 10);
					if(rangeEnd && rangeStart > rangeEnd) {
						sendResponse(416, "Requested range not satisfiable", socketToClient);
						statusCode = 416;
						goto last;
					}
				}
			}
			//fprintf(stderr, "range=%ld-%ld\n",rangeStart,rangeEnd);
		}
		else {
			sendResponse(416, "Requested range not satisfiable", socketToClient);
			statusCode = 416;
			goto last;
		}
	}
	if(requestHeaders.has("If-Modified-Since")) {
		struct tm time_ = {};
		strptime(requestHeaders.get("If-Modified-Since").c_str(), httpTimestampFmt, &time_);
		ifModifiedSince = mktime(&time_);
	}
	if(rangeStart > 0) {
		PBBS2chProxyThreadInfo info;
		pthread_mutex_lock(mutex);
		BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
		if(it != threadCache->end()) {
			info = it->second;
		}
		pthread_mutex_unlock(mutex);
		log_printf(5,"range request from %ld bytes\n",rangeStart);
		if(info) {
			int from = info->lastResNum;
			int alreadyRead = info->cachedSize;
			int lastResLength = info->cachedData->length;
			log_printf(5,"hit %s: cached %d bytes, last res size %d\n",threadKey.c_str(),alreadyRead,lastResLength);
			if(rangeStart <= alreadyRead && rangeStart >= alreadyRead - lastResLength) {
				if(curl) {
					CURLcode res;
					DataStorage *dat = new DataStorage();
					log_printf(5,"partial access from res num %d\n",from);
					snprintf(buf,16384,"%s%d-n",url,from);
					if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
					curl_easy_setopt(curl, CURLOPT_URL, buf);
					curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
					curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
					curl_easy_setopt(curl, CURLOPT_ENCODING, "");
					curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
					curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
					curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
					curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
					curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
					if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
					if(proxy_server) {
						curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
						curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
						curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
					}
					if(user_agent) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
					}
					else if(requestHeaders.has("User-Agent")) {
						curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
					}
					res = curl_easy_perform(curl);
					if(res == CURLE_OK) {
						curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
						curl_easy_reset(curl);
						if(statusCode == 200 && dat->length) {
							DataStorage *updated = html2dat(dat, from, &lastModified, true);
							if(ifModifiedSince && lastModified && updated && updated->length == lastResLength) {
								struct tm time_ = {};
								gmtime_r(&lastModified,&time_);
								time_t tmp = mktime(&time_);
								if(ifModifiedSince >= tmp) {
									sendResponse(304, "Not Modified", socketToClient);
									log_printf(5,"not modified!\n");
									delete updated;
									delete dat;
									statusCode = 304;
									goto last;
								}
							}
							if(updated && updated->length && updated->length >= lastResLength) {
								html = new DataStorage(alreadyRead - lastResLength);
								html->appendBytes(updated->bytes, updated->length);
								if(!rangeEnd) rangeEnd = html->length - 1;
								if(rangeStart > rangeEnd) {
									sendResponse(416, "Requested range not satisfiable", socketToClient);
									delete updated;
									delete dat;
									statusCode = 416;
									goto last;
								}
								statusCode = 206;
								log_printf(5,"cache hit; reconstructed data length:%ld\n",(long)html->length);
							}
							else {
								log_printf(5,"cache misshit?\n");
								sendResponse(416, "Requested range not satisfiable", socketToClient);
								delete updated;
								delete dat;
								statusCode = 416;
								goto last;
							}
							delete updated;
						}
					}
					else {
						log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),buf);
						curl_easy_reset(curl);
					}
					delete dat;
					if(html) goto resp;
				}
			}
			else {
				log_printf(5,"invalid cache contents\n");
				pthread_mutex_lock(mutex);
				BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
				if(it != threadCache->end()) {
					threadCache->erase(it);
				}
				pthread_mutex_unlock(mutex);
			}
		}
	}
	
	{
		if(curl) {
			CURLcode res;
			DataStorage *dat = new DataStorage();
			if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
#if 1
			/* This is an ad-hoc fix against malformed read.cgi behaviors (e.g. krsw server) */
			snprintf(buf, 16384, "%s1-", url);
			curl_easy_setopt(curl, CURLOPT_URL, buf);
#else
			curl_easy_setopt(curl, CURLOPT_URL, url);
#endif
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			curl_easy_setopt(curl, CURLOPT_ENCODING, "");
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
			curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			if(proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			if(user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			else if(requestHeaders.has("User-Agent")) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
			}
			res = curl_easy_perform(curl);
			if(res != CURLE_OK) {
				log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),url);
				sendResponse(503, "Service Unavailable", socketToClient);
				curl_easy_reset(curl);
				delete dat;
				statusCode = 503;
				goto last;
			}
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			curl_easy_reset(curl);
			
			if(statusCode == 200) {
				html = html2dat(dat, 1, &lastModified, false);
			}
			delete dat;
		}
		
		if(!html || !html->length) {
			sendResponse(503, "Service Unavailable", socketToClient);
			statusCode = 503;
			goto last;
		}
		if((rangeStart || rangeEnd) && html && html->length) {
			if(!rangeEnd) rangeEnd = html->length - 1;
			if(rangeStart < 0) rangeStart = html->length + rangeStart;
			if(rangeStart < html->length && rangeEnd < html->length && rangeStart <= rangeEnd) {
				statusCode = 206;
			}
			else {
				if(ifModifiedSince && lastModified && rangeStart == html->length) {
					struct tm time_ = {};
					gmtime_r(&lastModified,&time_);
					time_t tmp = mktime(&time_);
					if(ifModifiedSince >= tmp) {
						sendResponse(304, "Not Modified", socketToClient);
						log_printf(5,"not modified!\n");
						statusCode = 304;
						goto last;
					}
				}
				sendResponse(416, "Requested range not satisfiable", socketToClient);
				statusCode = 416;
				goto last;
			}
		}
	}

resp:
	if(statusCode == 206) sendBasicHeaders(statusCode,"Partial Content",socketToClient);
	else sendBasicHeaders(statusCode,"OK",socketToClient);
	if(0 >= socketToClient->writeString("Content-Type: text/plain\r\n")) goto last;
	if(0 >= socketToClient->writeString("Accept-Ranges: bytes\r\n")) goto last;
	if(statusCode == 206) {
		std::ostringstream ss;
		ss << "Content-Range: bytes " << rangeStart << "-" << rangeEnd << "/" << html->length << "\r\n";
		if (0 >= socketToClient->writeString(ss.str())) goto last;
		//fprintf(stderr,"Content-Length: %ld\r\n",rangeEnd - rangeStart + 1);
		//fprintf(stderr,"Content-Range: bytes %ld-%ld/%ld\r\n",rangeStart,rangeEnd,(long)html->length);
		DataStorage *newHtml = new DataStorage();
		newHtml->appendBytes(html->bytes+rangeStart, rangeEnd - rangeStart + 1);
		delete html;
		html = newHtml;
	}
	{
		std::ostringstream ss;
		ss << "Content-Length: " << html->length << "\r\n";
		if(0 >= socketToClient->writeString(ss.str())) goto last;
	}
	if(lastModified) {
		struct tm time_ = {};
		char date[256];
		gmtime_r(&lastModified,&time_);
		strftime(date,256,httpTimestampFmt,&time_);
		std::string header = "Last-Modified: ";
		header += date;
		header += "\r\n";
		if(0 >= socketToClient->writeString(header)) goto last;
		//fprintf(stderr,"Last-Modified: %s\r\n",date);
	}
	if(0 > socketToClient->writeString("\r\n")) goto last;
	
	if(html && statusCode >= 200 && statusCode < 300 && strcasecmp(method, "HEAD")) {
		if(html->length > socketToClient->write(html->bytes, html->length)) goto last;
	}
	
last:
	if(buf) free(buf);
	if(html) delete html;
	return statusCode;
}

DataStorage *BBS2chProxyConnection::html2dat_old(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[32];
	char title[1024];
	int cachedSize = 0;
	bool bbspink = strstr(threadKey.c_str(),"bbspink.com") ? true : false;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<title>", 7);
	if(!ptr) {
		delete txt;
		return NULL;
	}
	ptr += 7;
	while(1) {
		if(*ptr == '<') {
			if(!strncasecmp(ptr,"</title>",8)) {
				ptr += 8;
				break;
			}
			else title[i++] = *ptr++;
		}
		else title[i++] = *ptr++;
	}
	title[i] = 0;
	
	snprintf(signature,32,"<dt>%d ",res);
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		ptr = strstr(ptr,signature);
		ptr += strlen(signature);
		while(*ptr != '<') ptr++;
		ptr++;
		const char *endStr;
		if(*ptr == 'a' || *ptr == 'A') {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			endStr = "</a>";
		}
		else if(*ptr == 'b') {
			endStr = NULL;
		}
		else {
			endStr = "</font>";
		}
		
		if(endStr) {
			ptr = strstr(ptr,"<b>");
			ptr += 3;
		}
		else {
			ptr = strchr(ptr,'>');
			ptr++;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"</b>",4) && (!endStr || !strncasecmp(ptr+4,endStr,strlen(endStr)))) {
					ptr += 4;
					if(endStr) ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr += 2;
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<dd>",4)) {
					ptr += 4;
					break;
				}
				else if(!strncmp(ptr,"<a href=\"javascript:be(",23)) {
					memcpy(date+i,"BE:",3);
					ptr += 23;
					i += 3;
					while(*ptr != ')') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"<br><br>\n",9)) {
					ptr += 9;
					break;
				}
				else if(!strncasecmp(ptr,"<dt>",4) || !strncasecmp(ptr,"</dl>",5)) {
					while(i>0 &&body[i-1] == '\n') i--;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"http",13)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(body+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10) || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!bbspink && !strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else if(!bbspink && *ptr == ' ') {
				if(*(ptr+1) == ' ') ptr++;
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		snprintf(signature,32,"<dt>%d ",res);
		if(!memmem_priv(ptr, end-ptr+1, signature, strlen(signature))) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				snprintf(formattedDate,256,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

DataStorage *BBS2chProxyConnection::html2dat(DataStorage *html, int startResNum, time_t *lastModified, bool useCache)
{
	char *ptr = html->bytes;
	char *end = html->bytes + html->length - 1;
	DataStorage *txt = new DataStorage();
	int res = startResNum, i=0;
	char signature[64];
	char title[1024];
	int cachedSize = 0;
	char signatureTag[32];
	char closeTag[32];
	int closeTagLen;
	bool isNewHTML = false;
	
	ptr = (char *)memmem_priv(ptr, end-ptr+1, "<div id=\"threadtitle\">", 22);
	if (ptr) {
		isNewHTML = true;
		char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, "<article id=\"", 13);
		if (!ptr2) {
			delete txt;
			return NULL;
		}
		ptr += 22;
		while (1) {
			if (*ptr == '<') {
				if (!strncasecmp(ptr, "</div>", 6)) {
					ptr += 6;
					break;
				}
				else title[i++] = *ptr++;
			}
			else if(*ptr == '\n') break;
			else title[i++] = *ptr++;
		}
		title[i] = 0;

		snprintf(signature, 32, "<article id=\"%d\"", res);
	}
	else {
		ptr = html->bytes;
		ptr = (char *)memmem_priv(ptr, end-ptr+1, "<h1 class=\"title\">", 18);
		if(!ptr) {
			delete txt;
			return html2dat_old(html, startResNum, lastModified, useCache);
		}
		else {
			char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, " class=\"post\"", 13);
			if(ptr2) {
				char *tmp = ptr2;
				*ptr2 = 0;
				while(*ptr2 != '<') ptr2--;
				strcpy(signatureTag, ptr2);
				*tmp = ' ';
			}
			else {
				delete txt;
				return NULL;
			}
			/*char *ptr2 = (char *)memmem_priv(ptr, end-ptr+1, "<dl class=\"post\"", 16);
			if(ptr2) {
				delete txt;
				return html2dat_pink(html, startResNum, lastModified, useCache);
			}*/
		}
		
		ptr += 18;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,"</h1>",5)) {
					ptr += 5;
					break;
				}
				else title[i++] = *ptr++;
			}
			else if(*ptr == '\n') break;
			else title[i++] = *ptr++;
		}
		title[i] = 0;
		
		snprintf(signature,32,"%s class=\"post\" id=\"%d\"",signatureTag,res);
	}
	ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
	if(!ptr) {
		delete txt;
		return NULL;
	}
	
	unsigned char *buffer = (unsigned char *)malloc(65536+1024+1024+1024+2048);
	if(!buffer) {
		delete txt;
		return NULL;
	}
	
	unsigned char *body = buffer;
	char *mail = (char *)body + 65536;
	char *name = mail + 1024;
	char *date = name + 1024;
	char *encrypted = date + 1024;
	
	while(ptr < end) {
		//fprintf(stderr,"%s\n",signature);
		DataStorage *resData = new DataStorage();
		i=0;
		mail[0] = 0;
		if (isNewHTML) ptr = strstr(ptr," class=\"postusername\"><b>");
		else ptr = strstr(ptr," class=\"name\"><b>");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			snprintf(closeTag,32,"</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			if (isNewHTML) ptr = tmp + 25;
			else ptr = tmp + 17;
		}
		else {
			delete resData;
			break;
		}
		
		char endStr[64];
		if(!strncmp(ptr,"<a href=\"mailto:",16)) {
replay:
			// has mail
			while(*ptr != '"') ptr++;
			ptr++;
			if(!strncmp(ptr,"/cdn-cgi/l/email-protection#",28)) {
				ptr += 28;
				while(*ptr != '"' && *ptr != 'X') encrypted[i++] = *ptr++;
				encrypted[i] = 0;
				i = decryptMail((unsigned char *)mail,encrypted);
				int reconstruct_len = *ptr == 'X' ? i + 15 : i + 16;
				ptr -= reconstruct_len;
				char *start = ptr;
				memcpy(ptr, "<a href=\"mailto:", 16);
				ptr += 16;
				memcpy(ptr, mail, i);
				ptr = start;
				i=0;
				goto replay;
			}
			else {
				if(!strncmp(ptr,"mailto:",7)) ptr += 7;
				while(1) {
					if(*ptr == '<' && !strncmp(ptr,"<a href=\"",9)) {
						ptr = strchr(ptr,'>');
						ptr++;
						char *link = ptr;
						ptr = strstr(link,"</a>");
						memcpy(mail+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
					else if(*ptr == '"') break;
					else mail[i++] = *ptr++;
				}
				//while(*ptr != '"') mail[i++] = *ptr++;
				mail[i] = 0;
			}
			snprintf(endStr,64,"</a></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}
		/* we do not have to handle this special case because read.cgi on bbspink doesn't
		   emit font tags anymore and it conflicts with text decorations using "melon point" */
		/*else if(!strncmp(ptr,"<font",5)) {
			snprintf(endStr,64,"</font></b>%s",closeTag);
			while(*ptr != '>') ptr++;
			ptr++;
		}*/
		else {
			snprintf(endStr,64,"</b>%s",closeTag);
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncmp(ptr,endStr,strlen(endStr))) {
					ptr += strlen(endStr);
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail((unsigned char *)name+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a href=\"",9)) {
					ptr = strchr(ptr,'>');
					ptr++;
					char *link = ptr;
					ptr = strstr(link,"</a>");
					memcpy(name+i,link,ptr-link);
					i += ptr-link;
					ptr += 4;
				}
				else name[i++] = *ptr++;
			}
			else name[i++] = *ptr++;
		}
		
		resData->appendBytes(name, i);
		resData->appendBytes("<>", 2);
		if(mail[0]) resData->appendBytes(mail ,strlen(mail));
		resData->appendBytes("<>", 2);
		
		ptr = strstr(ptr," class=\"date\">");
		if(ptr) {
			char *tmp = ptr;
			*ptr = 0;
			while(*ptr != '<') ptr--;
			snprintf(closeTag,32,"</%s>",ptr+1);
			closeTagLen = strlen(closeTag);
			ptr = tmp + 14;
		}
		else {
			delete resData;
			break;
		}
		
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else date[i++] = *ptr++;
			}
			else date[i++] = *ptr++;
		}
		
		if(!strncmp(ptr,"<div class=\"uid",15) || !strncmp(ptr,"<span class=\"uid",16)) {
			char *tmp = ptr+1;
			while(*ptr != ' ') ptr++;
			*ptr = 0;
			snprintf(closeTag,32,"</%s>",tmp);
			closeTagLen = strlen(closeTag);
			ptr += 11;
			while(*ptr != '>') ptr++;
			ptr++;
			date[i++] = ' ';
			while(1) {
				if(*ptr == '<') {
					if(!strncasecmp(ptr,closeTag,closeTagLen)) {
						ptr += closeTagLen;
						break;
					}
					else date[i++] = *ptr++;
				}
				else date[i++] = *ptr++;
			}
		}
		
		if(!strncmp(ptr,"<div class=\"be",14) || !strncmp(ptr,"<span class=\"be",15)) {
			ptr += 14;
			while(*ptr != '>') ptr++;
			ptr++;
			if(!strncmp(ptr,"<a href=\"",9)) {
				ptr += 9;
				while(*ptr != '/' && *ptr != '"') ptr++;
				if(*ptr == '/' && (!strncmp(ptr,"//be.2ch.net/user/",18) || !strncmp(ptr,"//be.5ch.net/user/",18))) {
					memcpy(date+i," BE:",4);
					i += 4;
					ptr += 18;
					while(*ptr != '"') date[i++] = *ptr++;
					date[i++] = '-';
					ptr = strchr(ptr,'?');
					ptr++;
					char *tmp = strstr(ptr,"</a>");
					memcpy(date+i,ptr,tmp-ptr);
					i += tmp-ptr;
					ptr = tmp + 4;
				}
			}
		}
		
		resData->appendBytes(date ,i);
		resData->appendBytes("<>", 2);
		
		if (isNewHTML) {
			ptr = strstr(ptr,"<section class=\"post-content\">");
			if (!ptr) {
				delete resData;
				break;
			}
			else {
				ptr += 30;
				if (!strncasecmp(ptr, "<span class=\"AA\">", 17)) {
					strcpy(closeTag, "</span></section>");
					closeTagLen = 17;
					ptr += 17;
				}
				else {
					strcpy(closeTag, "</section>");
					closeTagLen = 10;
				}
			}
		}
		else if(!strcmp(signatureTag,"<div")) {
			ptr = strstr(ptr,"<div class=\"message\">");
			if(!ptr) {
				delete resData;
				break;
			}
			else {
				ptr += 21;
				if(!strncasecmp(ptr,"<span class=\"escaped\">",22)) {
					if(!strncasecmp(ptr+22,"<span class=\"AA\">",17)) {
						strcpy(closeTag,"</span></span></div>");
						closeTagLen = 20;
						ptr += 22+17;
					}
					else {
						strcpy(closeTag,"</span></div>");
						closeTagLen = 13;
						ptr += 22;
					}
				}
				else {
					strcpy(closeTag,"</div>");
					closeTagLen = 6;
				}
			}
		}
		else {
			ptr = strstr(ptr,"<dd class=\"thread_in\">");
			if(!ptr) {
				delete resData;
				break;
			}
			strcpy(closeTag,"</dd>");
			closeTagLen = 5;
			ptr += 22;
		}
		i=0;
		while(1) {
			if(*ptr == '<') {
				if(!strncasecmp(ptr,closeTag,closeTagLen)) {
					ptr += closeTagLen;
					break;
				}
				else if(!strncmp(ptr,"<span class=\"__cf_email__\"",26) || !strncmp(ptr,"<a class=\"__cf_email__\"",23)) {
					int j=0;
					ptr = strstr(ptr,"data-cfemail=\"");
					ptr += 14;
					while(*ptr != '"') encrypted[j++] = *ptr++;
					encrypted[j] = 0;
					j = decryptMail(body+i,encrypted);
					i += j;
					ptr = strstr(ptr,"</script>");
					ptr += 9;
				}
				else if(!strncmp(ptr,"<a ",3)) {
					char *tmp = strchr(ptr,'>');
					char *href = (char *)memmem_priv(ptr,tmp-ptr,"href=\"",6);
					char *link = tmp+1;
					if(href && !strncmp(link,"&gt;&gt;",8) && memmem_priv(href,link-href,"test/read.cgi/",14)) {
						while(ptr < link) {
							if(!strncmp(ptr," class=\"",8)) {
								ptr += 8;
								while(*ptr != '"' && *ptr != '>') ptr++;
								if(*ptr == '"') ptr++;
							}
							else body[i++] = *ptr++;
						}
					}
					else {
						ptr = strstr(link,"</a>");
						memcpy(body+i,link,ptr-link);
						i += ptr-link;
						ptr += 4;
					}
				}
				else if(!strncmp(ptr,"<img src=\"",10)) {
					ptr += 10;
					char *img = ptr;
					ptr = strstr(img,"\">");
					memcpy(body+i,img,ptr-img);
					if(memmem_priv(img,ptr-img,"/img.2ch.net",12) || memmem_priv(img,ptr-img,"/img.5ch.net",12) || memmem_priv(img,ptr-img,"/o.8ch.net",10)  || memmem_priv(img,ptr-img,"/o.5ch.net",10)) {
						int length = ptr-img;
						while(*img != '/') {
							img++;
							length--;
						}
						memcpy(body+i,"sssp:",5);
						memcpy(body+i+5,img,length);
						i += length + 5;
					}
					else i += ptr-img;
					ptr += 2;
				}
				else if(!strncmp(ptr,"<br>",4)) {
					if(i>5 && !strncmp((char *)body+i-5,"<br> ",5)) {
						memcpy(body+i," <br>",5);
						i += 5;
					}
					else {
						memcpy(body+i,"<br>",4);
						i += 4;
					}
					ptr += 4;
				}
				else body[i++] = *ptr++;
			}
			else body[i++] = *ptr++;
		}
		
		resData->appendBytes(body ,i);
		resData->appendBytes("<>", 2);
		if(res == 1) resData->appendBytes(title ,strlen(title));
		resData->appendBytes("\n" ,1);
		
		if(useCache && res == startResNum) {
			PBBS2chProxyThreadInfo info;
			bool hit = false;
			pthread_mutex_lock(mutex);
			BBS2chProxyThreadCache::iterator it = threadCache->find(threadKey);
			if(it != threadCache->end()) {
				info = it->second;
				threadCache->erase(it);
			}
			pthread_mutex_unlock(mutex);
			if(info) {
				log_printf(5,"cache hit");
				if(info->cachedData->length == resData->length) {
					log_printf(5,"... size match");
					if(!memcmp(info->cachedData->bytes,resData->bytes,resData->length)) {
						log_printf(5,"... content match");
						hit = true;
						cachedSize = info->cachedSize - resData->length;
					}
				}
				log_printf(5,"\n");
			}
			if(!hit) {
				delete resData;
				free(buffer);
				return NULL;
			}
		}
		
		txt->appendBytes(resData->bytes, resData->length);
		res++;
		while(*ptr == '\n' || *ptr == '\r') ptr++;
		if (isNewHTML) strcpy(signature, "<article id=\"");
		else snprintf(signature,64,"%s class=\"post\" id=\"",signatureTag);
		ptr = (char *)memmem_priv(ptr, end-ptr+1, signature, strlen(signature));
		if(ptr) {
			int next = atoi(ptr+strlen(signature));
			if(next >= res) {
				while(next > res) {
					txt->appendBytes("broken<><>broken<> broken <>\n", 29);
					res++;
				}
			}
			else ptr = NULL;
		}
		if(!ptr) {
			PBBS2chProxyThreadInfo info(new BBS2chProxyThreadInfo());
			info->lastResNum = res-1;
			info->cachedSize = txt->length+cachedSize;
			info->cachedData = resData;
			pthread_mutex_lock(mutex);
			threadCache->insert(std::make_pair(threadKey,info));
			pthread_mutex_unlock(mutex);
			log_printf(5,"cached thread %s (%ld bytes)\n",threadKey.c_str(),(long)resData->length);
			
			if(lastModified) {
				*lastModified = 0;
				char formattedDate[256];
				char *ptr;
				ptr = date;
				int year = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int month = strtol(ptr,&ptr,10);
				if(*ptr != '/') break;
				ptr++;
				int day = strtol(ptr,&ptr,10);
				if(!*ptr) break;
				while(*ptr != ' ' && *ptr != 0) ptr++;
				if(!*ptr) break;
				ptr++;
				int hour = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int minutes = strtol(ptr,&ptr,10);
				if(*ptr != ':') break;
				ptr++;
				int seconds = strtol(ptr,&ptr,10);
				if(!(month>0 && month<13) || !(day>0 && day<32)) break;
				if(year < 100) year += 2000;
				snprintf(formattedDate,256,"%d/%d/%d %02d:%02d:%02d JST",year,month,day,hour,minutes,seconds);
				//fprintf(stderr,"%s\n",formattedDate);
				
				struct tm time = {};
				strptime(formattedDate,threadTimestampFmt,&time);
				*lastModified = mktime(&time);
				//gmtime_r(lastModified,&time);
				//strftime(formattedDate,256,httpTimestampFmt,&time);
				//fprintf(stderr,"%s\n",formattedDate);
			}
			//fprintf(stderr,"not found,%ld\n",end-ptr+1);
			break;
		}
		delete resData;
	}
	
	free(buffer);
	return txt;
}

int BBS2chProxyConnection::datProxyAPI(const char *url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	long statusCode = 0;
	const std::string &postBody = auth.requestBodyForURL(url, curl);
	bool directMode = false;
	if (postBody.empty()) {
		sendResponse(401, "Unauthorized", socketToClient);
		return 401;
	}
	/* just read and strip off post body */
	if (!strcasecmp(method, "POST")) {
		char *postdata = NULL;
		if (isClientChunked) {
			readChunkedBodyIntoBuffer(&postdata, socketToClient);
		}
		else if (content_length) {
			postdata = (char *)calloc(content_length+1, 1);
			socketToClient->read(postdata, content_length);
		}
		if (postdata && strstr(postdata, "sid=")) directMode = true;
		if (postdata) free(postdata);
	}
	if (curl) {
		CURLcode res;
		struct curl_slist *headersForCurl = NULL;
		DataStorage receivedHeader;
		DataStorage receivedBody;
		headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "Range");
		headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "If-Modified-Since");
		headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "Accept-Encoding");
		if (x_2ch_ua_dat) headersForCurl = curl_slist_append(headersForCurl, x_2ch_ua_dat);
		
		if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headersForCurl);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &receivedBody);
		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_download);
		curl_easy_setopt(curl, CURLOPT_HEADERDATA, &receivedHeader);
		curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if (proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if (api_ua_dat) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, api_ua_dat);
		}
		else {
			if (user_agent && !strncmp(user_agent, "Monazilla/", strlen("Monazilla/")))
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			else
				curl_easy_setopt(curl, CURLOPT_USERAGENT, "");
		}
		curl_easy_setopt(curl, CURLOPT_POST, 1L);
#if LIBCURL_VERSION_NUM >= 0x071101
		curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, postBody.c_str());
#else
		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postBody.c_str());
#endif
		//return;
		res = curl_easy_perform(curl);
		if (res == CURLE_OK) {
			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
			receivedHeader.appendBytes("", 1);
			const char *ptr = receivedHeader.bytes;
			/* this is necessary because the raw header may contain chunk trailers after real headers */
			const char *end = strstr(receivedHeader.bytes, "\r\n\r\n");
			int threadStatus = 0;
			if (end && !directMode) {
				BBS2chProxyHttpHeaders headers;
				while (ptr < end) {
					const char *lineEnd = strchr(ptr, '\n');
					if (!lineEnd) break;
					headers.add(ptr, lineEnd-ptr);
					ptr = lineEnd + 1;
				}
				if (headers.has("Thread-Status")) {
					threadStatus = atoi(headers.get("Thread-Status").c_str());
				}
			}
			if (threadStatus == 1 || (directMode && end)) {
				if (end+4-receivedHeader.bytes > socketToClient->write(receivedHeader.bytes, end+4-receivedHeader.bytes)) goto last;
				if (receivedBody.length > socketToClient->write(receivedBody.bytes, receivedBody.length)) goto last;
				goto last;
			}
			else if (threadStatus == 8) {
				sendBasicHeaders(302, "Found", socketToClient);
				if (0 >= socketToClient->writeString("Location: http://www2.2ch.net/live.html\r\n")) goto last;
				if (0 >= socketToClient->writeString("\r\n")) goto last;
				statusCode = 302;
				goto last;
			}
			else {
				if (statusCode < 400) {
					sendResponse(401, "Unauthorized", socketToClient);
					statusCode = 401;
				}
				else {
					sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
				receivedBody.appendBytes("",1);
				if (!strncasecmp(receivedBody.bytes,"ng (",4)) {
					log_printf(0, "API gateway returned error: %s\n", receivedBody.bytes);
				}
			}
			//fprintf(stderr,"%ld\n",statusCode);
		}
		else {
			log_printf(0, "curl error: %s\n", curl_easy_strerror(res));
			sendResponse(503, "Service Unavailable", socketToClient);
			statusCode = 503;
		}
last:
		curl_easy_reset(curl);
		curl_slist_free_all(headersForCurl);
	}
	return statusCode;
}

int BBS2chProxyConnection::bbsmenuProxy(const char *url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	long statusCode = 0;
	DataStorage *dat = new DataStorage();
	DataStorage *outHTML = new DataStorage();
	if(curl) {
		CURLcode res;
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_ENCODING, "");
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, dat);
		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
		curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if(user_agent) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
		}
		else if(requestHeaders.has("User-Agent")) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
		}
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			if(statusCode == 200 && dat->length) {
				dat->appendBytes("",1);
				dat->length--;
				char *ptr = dat->bytes;
				while(*ptr) {
					if(!strncasecmp(ptr,"<a href=",8)) {
						char *start = ptr+8;
						char *end = strchr(start,'>');
						if (end) {
							char *urlEnd = end;
							if (*start == '"') {
								start++;
								char *tmp = strchr(start, '"');
								if (tmp && tmp < end) urlEnd = tmp;
							}
							BBS2chProxyURL url(std::string(start, urlEnd-start).c_str());
							if (url.isKindOfHost("5ch.net")) {
								url.replaceHost("5ch.net", "2ch.net");
								url.setScheme("http");
								const std::string &replacedURL = url.absoluteString();
								outHTML->appendBytes("<A HREF=", 8);
								outHTML->appendBytes(replacedURL.c_str(), replacedURL.length());
								outHTML->appendBytes(">", 1);
								ptr = end+1;
								continue;
							} else if (url.isKindOfHost("bbspink.com")) {
								url.setScheme("http");
								const std::string &replacedURL = url.absoluteString();
								outHTML->appendBytes("<A HREF=", 8);
								outHTML->appendBytes(replacedURL.c_str(), replacedURL.length());
								outHTML->appendBytes(">", 1);
								ptr = end+1;
								continue;
							}
						}
					}
					outHTML->appendBytes(ptr++, 1);
				}
			}
		}
		else {
			log_printf(0,"curl error: %s (%s)\n", curl_easy_strerror(res), url);
			statusCode = 503;
		}
	}
	if(statusCode == 200) {
		std::ostringstream ss;
		ss << "Content-Length: " << outHTML->length << "\r\n";
		sendBasicHeaders(statusCode,"OK",socketToClient);
		if(0 >= socketToClient->writeString("Content-Type: text/html\r\n")) goto last;
		if(0 >= socketToClient->writeString(ss.str())) goto last;
		if(0 >= socketToClient->writeString("\r\n")) goto last;
		if(strcasecmp(method, "HEAD")) {
			if(outHTML->length > socketToClient->write(outHTML->bytes, outHTML->length)) goto last;
		}
	}
	else {
		sendResponse(503, "Service Unavailable", socketToClient);
		statusCode = 503;
	}
	
last:
	if(curl) curl_easy_reset(curl);
	if(dat) delete dat;
	if(outHTML) delete outHTML;
	return statusCode;
}

int BBS2chProxyConnection::bbsCgiProxy(const char *url, BBS2chProxyHttpHeaders &requestHeaders, const char *requestBody)
{
	long statusCode = 0;
	std::string hostStr = requestHeaders.get("Host");
	std::string boardStr;
	std::string threadStr;
	requestHeaders.remove("Host");
	if (user_agent) requestHeaders.set("User-Agent", user_agent);

	if (requestBody && (lua_script || !bbscgi_headers.empty() || !bbscgi_postorder.empty())) {
		std::map<std::string, std::string> fields;
		const char *ptr = requestBody;
		size_t bodyLength = 0;
		while (1) {
			const char *tmp = ptr;
			while (*tmp != '=' && *tmp != 0) tmp++;
			if (*tmp == 0) {
				bodyLength = tmp - requestBody;
				break;
			}
			std::string key(ptr, tmp-ptr);
			tmp++;
			ptr = tmp;
			while (*tmp != '&' && *tmp != 0) tmp++;
			std::string value(ptr, tmp-ptr);
			fields.insert(std::make_pair(key, value));
			if (*tmp == 0) {
				bodyLength = tmp - requestBody;
				break;
			}
			ptr = tmp + 1;
		}
		std::map<std::string, std::string>::iterator it;
		if (it = fields.find("bbs"), it != fields.end()) boardStr = it->second;
		if (it = fields.find("key"), it != fields.end()) threadStr = it->second;
		if (!bbscgi_postorder.empty()) {
			std::string newBody;
			for (std::vector<std::string>::iterator it2 = bbscgi_postorder.begin(); it2 != bbscgi_postorder.end(); it2++) {
				const std::string &name = *it2;
				if (it = fields.find(name), it != fields.end()) {
					if (!newBody.empty()) newBody.append("&");
					newBody.append(name);
					newBody.append("=");
					newBody.append(it->second);
					fields.erase(name);
				}
			}
			for (it = fields.begin(); it != fields.end(); it++) {
				if (!newBody.empty()) newBody.append("&");
				newBody.append(it->first);
				newBody.append("=");
				newBody.append(it->second);
			}
			if (bodyLength == newBody.length()) {
				strcpy((char *)requestBody, newBody.c_str());
				log_printf(1, "Reordered request body is: %s\n", requestBody);
			}
			else {
				log_printf(0, "Error occured while reordering the request body - skipping\n");
			}
		}
	}
	if (!bbscgi_headers.empty()) {
		for (std::map<std::string, std::string>::iterator it = bbscgi_headers.begin(); it!=bbscgi_headers.end(); it++) {
			/* we cannot use a reference here, because the original string shouldn't be replaced */
			std::string value = it->second;
			if (!hostStr.empty()) {
				std::string::size_type pos = value.find("%HOST%");
				while (pos != std::string::npos) {
					value.replace(pos, 6, hostStr);
					pos = value.find("%HOST%", pos+hostStr.length());
				}
			}
			if (!boardStr.empty()) {
				std::string::size_type pos = value.find("%BOARD%");
				while (pos != std::string::npos) {
					value.replace(pos, 7, boardStr);
					pos = value.find("%BOARD%", pos+boardStr.length());
				}
			}
			if (!threadStr.empty()) {
				std::string::size_type pos = value.find("%THREAD%");
				while (pos != std::string::npos) {
					value.replace(pos, 8, threadStr);
					pos = value.find("%THREAD%", pos+threadStr.length());
				}
			}
			requestHeaders.set(it->first, value);
			log_printf(1, "Appended custom header \"%s: %s\"\n", it->first.c_str(), value.c_str());
		}
	}

	for (int run=0; run<2; run++) {
		BBS2chProxyHttpHeaders *_headers = new BBS2chProxyHttpHeaders(requestHeaders);
		curl_slist *headersForCurl = NULL;
		char *_body = (char *)requestBody;
		std::string nic, forceProxy;
		long verbose = 0;
		status = 0;
		monaKeyForRequest = "";
#ifdef USE_LUA
		if (lua_script) {
			lua_State* l = luaL_newstate();
			luaL_openlibs(l);
			if (luaL_loadfile(l, lua_script) != LUA_OK) {
				log_printf(0, "Lua: Failed to open script %s:\n  %s\n", lua_script, lua_tostring(l, -1));
				goto lua_end;
			}
			lua_newtable(l);
			lua_pushcfunction(l, lua_hmacSHA256);
			lua_setfield(l, -2, "hmacSHA256");
			lua_pushcfunction(l, lua_decodeURIComponent);
			lua_setfield(l, -2, "decodeURIComponent");
			lua_pushcfunction(l, lua_encodeURIComponent);
			lua_setfield(l, -2, "encodeURIComponent");
			lua_pushcfunction(l, lua_convertShiftJISToUTF8);
			lua_setfield(l, -2, "convertShiftJISToUTF8");
			lua_pushcfunction(l, lua_isExpiredKey);
			lua_setfield(l, -2, "isExpiredKey");
			lua_pushcfunction(l, lua_isValidAsUTF8);
			lua_setfield(l, -2, "isValidAsUTF8");
			lua_pushcfunction(l, lua_getMonaKey);
			lua_setfield(l, -2, "getMonaKey");
			lua_pushstring(l, keyManager.getKey().c_str());
			lua_setfield(l, -2, "monaKey");
			lua_pushinteger(l, serverPort);
			lua_setfield(l, -2, "port");
			lua_setglobal(l, "proxy2ch");
			BBS2chProxyHttpHeaders::getClassDefinitionForLua(l);
			lua_setglobal(l, "HttpHeaders");
			if (lua_pcall(l, 0, 0, 0) != LUA_OK) {
				log_printf(0, "Lua: Failed to run script %s:\n  %s\n", lua_script, lua_tostring(l, -1));
				goto lua_end;
			}
			lua_getglobal(l, "willSendRequestToBbsCgi");
			if (!lua_isfunction(l, -1)) {
				log_printf(0, "Lua: willSendRequestToBbsCgi function does not exist in the script\n");
				goto lua_end;
			}
			lua_newtable(l);
			_headers->getUserdataForLua(l);
			lua_setfield(l, -2, "headers");
			lua_pushstring(l, _body);
			lua_setfield(l, -2, "body");
			lua_pushstring(l, hostStr.c_str());
			lua_pushstring(l, boardStr.c_str());
			lua_pushstring(l, threadStr.c_str());
			if (lua_pcall(l, 4, 1, 0) != LUA_OK) {
				log_printf(0, "Lua: Failed to call willSendRequestToBbsCgi function:\n  %s\n", lua_tostring(l, -1));
				goto lua_end;
			}
			if (!lua_istable(l, -1)) {
				log_printf(0, "Lua: A return type of willSendRequestToBbsCgi function should be a table\n");
				goto lua_end;
			}
			lua_pushstring(l, "body");
			lua_rawget(l, -2);
			if (lua_isstring(l, -1)) {
				const char *newBody = lua_tostring(l, -1);
				_body = strdup(newBody);
				log_printf(1, "Lua: Set request body \"%s\"\n", newBody);
			}
			lua_pop(l, 1);
			lua_pushstring(l, "headers");
			lua_rawget(l, -2);
			if (lua_istable(l, -1)) {
				delete _headers;
				_headers = new BBS2chProxyHttpHeaders();
				lua_pushnil(l);
				while (lua_next(l, -2)) {
					if (lua_isstring(l, -1) && lua_isstring(l, -2)) {
						const char *name = lua_tostring(l, -2);
						const char *value = lua_tostring(l, -1);
						_headers->add(name, value);
						log_printf(1, "Lua: Set request header \"%s: %s\"\n", name, value);
					}
					lua_pop(l, 1);
				}
			}
			else if (lua_isuserdata(l, -1)) {
				if (lua_getmetatable(l, -1)) {
					if (lua_getfield(l, -1, "_type") == LUA_TSTRING) {
						if (!strcmp(lua_tostring(l, -1), "HttpHeaders")) {
							BBS2chProxyHttpHeaders *newHeaders = *((BBS2chProxyHttpHeaders **)lua_touserdata(l, -3));
							if (newHeaders != _headers) {
								/* remove metatable to prevent the object from garbage collected by lua */
								lua_newtable(l);
								lua_setmetatable(l, -4);
								delete _headers;
								_headers = newHeaders;
							}
							for (std::map<std::string, PBBS2chProxyHttpHeaderEntry>::iterator it = _headers->getMap().begin(); it != _headers->getMap().end(); it++) {
								log_printf(1, "Lua: Set request header \"%s\"\n", it->second->getFull().c_str());
							}
						}
					}
					lua_pop(l, 2);
				}
			}
			lua_pop(l, 1);
			lua_pushstring(l, "options");
			lua_rawget(l, -2);
			if (lua_istable(l, -1)) {
				lua_pushstring(l, "interface");
				lua_rawget(l, -2);
				if (lua_isstring(l, -1)) {
					nic = std::string(lua_tostring(l, -1));
				}
				lua_pop(l, 1);
				lua_pushstring(l, "verbose");
				lua_rawget(l, -2);
				if (lua_isboolean(l, -1)) {
					verbose = lua_toboolean(l, -1);
				}
				lua_pop(l, 1);
				lua_pushstring(l, "proxy");
				lua_rawget(l, -2);
				if (lua_isstring(l, -1)) {
					forceProxy = std::string(lua_tostring(l, -1));
				}
			}
lua_end:
			lua_close(l);
		}
#endif
		do {
			bool isPink = hostStr.find("bbspink.com") != std::string::npos;
			bool shouldSign = appKey && (((api_mode & 2) && !isPink) || (api_mode & 4));
			bool shouldConvertBodyToUTF8 = (bbscgi_utf8 == 1 && shouldSign) || (bbscgi_utf8 == 2);
			userAgentForRequest = _headers->get("User-Agent");
			if (userAgentForRequest.empty() && user_agent) userAgentForRequest = user_agent;
			if (_headers->has("X-MonaKey")) {
				monaKeyForRequest = _headers->get("X-MonaKey");
			}
			if (shouldConvertBodyToUTF8 && !_headers->has("X-PostSig")) {
				std::string newBody = convertBodyToUTF8(_body);
				if (!newBody.empty()) {
					if (_body != requestBody) {
						free(_body);
					}
					_body = strdup(newBody.c_str());
					log_printf(1, "Converted request body to UTF-8: %s\n", _body);
				}
				else {
					log_printf(1, "Request body seems already to be UTF-8, will be sent without conversion\n");
				}
				std::string contentType = _headers->get("Content-Type");
				std::transform(contentType.begin(), contentType.end(), contentType.begin(), tolower);
				if (contentType.find("charset=utf-8") == std::string::npos) {
					_headers->set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
					log_printf(1, "Appended header \"Content-Type: application/x-www-form-urlencoded; charset=UTF-8\"\n");
				}
			}
			if (shouldSign && (!lua_script || !_headers->has("X-PostSig"))) {
				if (!userAgentForRequest.empty()) {
					monaKeyForRequest = keyManager.getKey(userAgentForRequest);
					appendPostSignature(_body, userAgentForRequest, monaKeyForRequest, _headers);
				} else {
					log_printf(0, "API: User-Agent muse be set explicitly to post with API.\n");
				}
			}
			if (!monaKeyForRequest.empty()) {
				double wait = keyManager.secondsToWaitBeforePosting(monaKeyForRequest);
				if (wait > 0) {
					log_printf(1, "Sleeping for %.1f seconds to avoid posting too fast...\n", wait);
#ifdef _WIN32
					Sleep(wait * 1e+3);
#else
					usleep(wait * 1e+6);
#endif
				}
			}
			headersForCurl = _headers->appendToCurlSlist(headersForCurl);
			if (!_headers->has("Expect")) headersForCurl = curl_slist_append(headersForCurl, "Expect:");
			if (!_headers->has("Accept")) headersForCurl = curl_slist_append(headersForCurl, "Accept:");
		} while (0);

		if (curl) {
			CURLcode res;
			if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
			curl_easy_setopt(curl, CURLOPT_URL, url);
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			if (run == 0)
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_bbscgi);
			else
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
			curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
			curl_easy_setopt(curl, CURLOPT_POST, 1L);
			curl_easy_setopt(curl, CURLOPT_POSTFIELDS, _body);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			curl_easy_setopt(curl, CURLOPT_VERBOSE, verbose);
			if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headersForCurl);
			if (!nic.empty()) curl_easy_setopt(curl, CURLOPT_INTERFACE, nic.c_str());
			if (user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			if (!forceProxy.empty()) {
				curl_easy_setopt(curl, CURLOPT_PROXY, forceProxy.c_str());
			}
			else if (proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			res = curl_easy_perform(curl);
			if (res != CURLE_OK) {
				if (res == CURLE_WRITE_ERROR && status == 2) {
					log_printf(1, "MonaKey should be reset. Sending the same request automatically...\n");
					curl_easy_reset(curl);
					curl_slist_free_all(headersForCurl);
					delete _headers;
					if (_body != requestBody) free(_body);
					continue;
				}
				else {
					log_printf(0,"curl error: %s (%s)\n",curl_easy_strerror(res),url);
					if (!status) sendResponse(503, "Service Unavailable", socketToClient);
					statusCode = 503;
				}
			}
			else {
				if (isResponseChunked) {
					socketToClient->writeString("0\r\n\r\n");
				}
				curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
			}
			curl_easy_reset(curl);
		}
		curl_slist_free_all(headersForCurl);
		delete _headers;
		if (_body != requestBody) free(_body);
		break;
	}
	return statusCode;
}

void BBS2chProxyConnection::compileRegex(void)
{
	static int compiled;
	if (compiled) return;
	regcomp(&regex, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com)(:[0-9]+)?/([^/]+)/dat/([0-9]+)\\.dat", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_kako, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com)(:[0-9]+)?/([^/]+)/kako/[0-9]+/([0-9]+/)?([0-9]+)\\.dat", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_offlaw, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com)(:[0-9]+)?/test/offlaw2.so\\?.*bbs=([^&]+)", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_api, "^https?://api\\.[25]ch\\.net(:[0-9]+)?/v1/([^/]+)/([^/]+)/([0-9]+)", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_api_auth, "^https?://api\\.[25]ch\\.net(:[0-9]+)?/v1/auth/?$", REG_EXTENDED|REG_ICASE);
	compiled = 1;
}