proxy2ch/BBS2chProxyConnection.cpp

#include <string>
#include <vector>
#include <map>
#include <set>
#include <sstream>
#include <stdexcept>
#include <algorithm>
#include <pthread.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#ifdef _WIN32
#include <fcntl.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <mswsock.h>
#define CLOSESOCKET(x) closesocket(x)
#define gmtime_r(a, b) gmtime_s(b, a)
#else
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <poll.h>
#define CLOSESOCKET(x) close(x)
#endif
#include "BBS2chProxyConnection.h"
#include "BBS2chProxyRawSocket.h"
#include "BBS2chProxyPoster.h"
#ifdef USE_MITM
#include "BBS2chProxySecureSocket.h"
#endif
//#define DEBUG 1

#define HTTP_TIMESTAMP_FMT_PARSE "%a, %d %b %Y %H:%M:%S %Z"
#define HTTP_TIMESTAMP_FMT_PRINT "%a, %d %b %Y %H:%M:%S GMT"

extern char *proxy_server;
extern long proxy_port;
extern long proxy_type;
extern long timeout;
extern char *user_agent;
extern BBS2chProxyHttpHeaders api_dat_headers;
extern char *appKey;
extern char *hmacKey;
extern int allow_chunked;
extern int curl_features;
extern unsigned int curl_version_number;
extern bool accept_https;
extern int force_5chnet;
extern int force_5chnet_https;
extern int force_ipv4;
extern char *bbsmenu_url;
extern char *api_server;
extern BBS2chProxyHttpHeaders bbscgi_headers;
extern int gikofix;
extern CURLSH *curl_share;
extern unsigned int api_mode;
extern int api_override;
extern int direct_dat;
extern int fool_janestyle;
extern int talk_to_5ch;
extern int subject_to_lastmodify;
extern std::set<std::string> subject_to_lastmodify_hosts;
#ifdef USE_MITM
extern unsigned int mitm_mode;
#endif

extern void log_printf(int level, const char *format ...);

#include "utils.h"

BBS2chProxyKeyManager BBS2chProxyConnection::keyManager;
BBS2chProxyAuth BBS2chProxyConnection::auth;
BBS2chProxyBoardManager BBS2chProxyConnection::boardManager;

static regex_t regex;
static regex_t regex_kako;
static regex_t regex_offlaw;
static regex_t regex_api;
static regex_t regex_api_auth;
static regex_t regex_talk;
static regex_t regex_talk_api;
static regex_t regex_talk_bbscgi;

static bool isPinkPrefix(const std::string &prefix)
{
	     if (prefix == "aoi") return true;
	else if (prefix == "babiru") return true;
	else if (prefix == "idol") return true;
	else if (prefix == "kilauea") return true;
	else if (prefix == "mercury") return true;
	else if (prefix == "nasu") return true;
	else if (prefix == "okazu") return true;
	else if (prefix == "peach") return true;
	else if (prefix == "pele") return true;
	else if (prefix == "phoebe") return true;
	else if (prefix == "pie") return true;
	else if (prefix == "pink") return true;
	else if (prefix == "qiufen") return true;
	else if (prefix == "sakura01") return true;
	else if (prefix == "sakura02") return true;
	else if (prefix == "set") return true;
	else if (prefix == "venus") return true;
	else if (prefix == "vip") return true;
	else if (prefix == "wow") return true;
	else if (prefix == "yomi") return true;
	return false;
}

void BBS2chProxyConnection::run(void * (*func)(void *))
{
	pthread_t thread;
	pthread_attr_t thread_attr;
	pthread_attr_init(&thread_attr);
	pthread_attr_setdetachstate(&thread_attr , PTHREAD_CREATE_DETACHED);
	if(0 != pthread_create(&thread , &thread_attr , func , this))
		perror("pthread_create");
	pthread_attr_destroy(&thread_attr);
}

struct TunnelSockets {
	int sock_c;
	int sock_s;
	std::string addr;
	int port;
};

static void *tunnelMain(void *param)
{
	TunnelSockets *sockets = (TunnelSockets *)param;
	char *buf = new char[16384];
#ifdef _WIN32
	fd_set fds;
	int nfds = sockets->sock_c > sockets->sock_s ? sockets->sock_c + 1 : sockets->sock_s + 1;
#else
	struct pollfd fds[2];
	memset(fds, 0, sizeof(fds));
	fds[0].fd = sockets->sock_c;
	fds[0].events = POLLIN;
	fds[1].fd = sockets->sock_s;
	fds[1].events = POLLIN;
#endif
	while (1) {
#ifdef _WIN32
		FD_ZERO(&fds);
		FD_SET(sockets->sock_c, &fds);
		FD_SET(sockets->sock_s, &fds);
		if (select(nfds, &fds, NULL, NULL, NULL) < 0) break;
		if (FD_ISSET(sockets->sock_c, &fds)) {
			int ret = recv(sockets->sock_c, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_s, buf, ret, 0);
			else if (ret <= 0) break;
		}
		if (FD_ISSET(sockets->sock_s, &fds)) {
			int ret = recv(sockets->sock_s, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_c, buf, ret, 0);
			else if (ret <= 0) break;
		}
#else
		if (poll(fds, 2, -1) < 0) break;
		if (fds[0].revents & POLLIN) {
			int ret = recv(sockets->sock_c, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_s, buf, ret, 0);
			else if (ret <= 0) break;
		}
		else if (fds[0].revents != 0) break;
		if (fds[1].revents & POLLIN) {
			int ret = recv(sockets->sock_s, buf, 16384, 0);
			if (ret > 0) send(sockets->sock_c, buf, ret, 0);
			else if (ret <= 0) break;
		}
		else if (fds[1].revents != 0) break;
#endif
	}
	CLOSESOCKET(sockets->sock_c);
	CLOSESOCKET(sockets->sock_s);
	log_printf(1, "Finished tunneling to %s:%d\n", sockets->addr.c_str(), sockets->port);
	delete sockets;
	delete[] buf;
	return NULL;
}

int BBS2chProxyConnection::tunnel(const char *addr, int port)
{
	struct sockaddr_in server;
	memset(&server, 0, sizeof(server));
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = inet_addr(addr);
	server.sin_port = htons(port);
	if(server.sin_addr.s_addr == 0xffffffff) {
		struct hostent *host;
		host = gethostbyname(addr);
		if (host == NULL) {
			log_printf(0, "Failed to lookup hostname %s\n", addr);
			socketToClient->sendResponse(400, "Bad Request");
			return 400;
		}
		server.sin_addr.s_addr = *(unsigned int *)host->h_addr_list[0];
	 }
	
	log_printf(1,"Tunneling connection to %s:%d\n",addr,port);
	
	int sock_s = socket(AF_INET, SOCK_STREAM, 0);
	if(-1 == ::connect(sock_s, (struct sockaddr *)&server, sizeof(server))) {
		perror("connect");
		socketToClient->sendResponse(400, "Bad Request");
		return 400;
	}
	send(sock_c, "HTTP/1.1 200 Connection established\r\n\r\n", 39, 0);

	TunnelSockets *sockets = new TunnelSockets();
	sockets->sock_c = sock_c;
	sockets->sock_s = sock_s;
	sockets->addr = addr;
	sockets->port = port;
	pthread_t thread;
	if(0 != pthread_create(&thread, NULL, tunnelMain, sockets))
		perror("pthread_create");
	pthread_detach(thread);
	
	return 0;
}

void BBS2chProxyConnection::connect(void)
{
	char method[32], url[1024], protocol[32];
	int i;
	char *buf, *ptr;
	unsigned int datProxyMode = 0; // 0: no dat, 1: read.cgi or API, 2: force API, 3: kakolog
	regmatch_t match[8];
	long statusCode = 0;
	BBS2chProxyURL baseURL;
	BBS2chProxyHttpHeaders requestHeaders;
	BBS2chThreadIdentifier threadIdentifier;
	bool talkTo5chBbsCgi = false;
	
	socketToClient = new BBS2chProxyRawSocket(sock_c);
	
	buf = (char *)malloc(16384);
	if(!buf) goto end;
beginHandleRequest:
	ptr = buf;
	if(!socketToClient->readLine(buf, 1024)) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 32) method[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	method[i] = 0;
	ptr++;
	i=0;
	while(*ptr != ' ' && *ptr != 0 && i < 1024) url[i++] = *ptr++;
	if(*ptr == 0 || i == 1024) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	url[i] = 0;
	ptr++;
	i=0;
	while(*ptr != '\r' && *ptr != '\n' && *ptr != 0 && i < 32) protocol[i++] = *ptr++;
	if(*ptr == 0 || i == 32) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	protocol[i] = 0;
	if(!strncasecmp(protocol,"HTTP/1.0",8)) {
		isClientHttp1_0 = true;
	}
	else isClientHttp1_0 = false;
	log_printf(1, "Received %s %s %s\n",method,url,protocol);
	
	if(strcasecmp(method,"GET") && strcasecmp(method,"POST") && strcasecmp(method,"HEAD") && strcasecmp(method,"CONNECT") && strcasecmp(method,"PUT") && strcasecmp(method, "OPTIONS")) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	if(!url[0]) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	if(strncasecmp(protocol,"HTTP",4)) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	
	if(!strcasecmp(method,"CONNECT")) {
		if(!accept_https || baseURL.isValid()) {
			socketToClient->sendResponse(400, "Bad Request");
			statusCode = 400;
			goto end;
		}
		while(socketToClient->readLine(buf, 16384)) {
			if(!strcmp("\r\n",buf)) break;
		}
		int port = 443;
		char *ptr = strchr(url, ':');
		if(ptr) {
			*ptr = 0;
			port = atoi(ptr+1);
		}
#ifdef USE_MITM
		bool useMITM = false;
		if (mitm_mode) {
			baseURL = BBS2chProxyURL("https", url);
			if (mitm_mode == 2) useMITM = true;
			else if (mitm_mode == 1 && (baseURL.isFamilyOf5chNet() || (talk_to_5ch && baseURL.isKindOfHost("talk-platform.com")))) useMITM = true;
		}
		if (useMITM) {
			socketToClient->writeString("HTTP/1.1 200 Connection established\r\n\r\n");
			if (port == 80) {
				baseURL.setScheme("http");
				goto beginHandleRequest;
			}
			else {
				try {
					BBS2chProxySecureSocket *secureSocket = new BBS2chProxySecureSocket(sock_c, url);
					delete socketToClient;
					socketToClient = secureSocket;
					if (port != 443) baseURL.setPort(port);
					isHttps = true;
					goto beginHandleRequest;
				} catch (const std::runtime_error& e) {
					log_printf(0, "%s\n", e.what());
					socketToClient->sendResponse(400, "Bad Request");
					statusCode = 400;
					goto end;
				}
			}
		}
		else
#endif
		{
			statusCode = tunnel(url, port);
			/* if a return value is non-zero, tunnel function failed to establish a tunnelling connection */
			if (statusCode == 0) {
				delete socketToClient;
				socketToClient = NULL;
			}
			goto end;
		}
	}
#if USE_MITM
	if (baseURL.isValid()) {
		requestURL = BBS2chProxyURL(baseURL, url);
		log_printf(1, "Running as MITM proxy for %s\n", requestURL.absoluteString().c_str());
	} else
#endif
	if (url[0] == '/') {
		requestURL = BBS2chProxyURL("http", "localhost", serverPort, url);
	}
	else {
		requestURL = BBS2chProxyURL(url);
		if (requestURL.getPort() == serverPort) requestURL.replaceHost("127.0.0.1", "localhost");
	}
	if (!requestURL.isHttp()) {
		socketToClient->sendResponse(400, "Bad Request");
		statusCode = 400;
		goto end;
	}
	if (force_5chnet) {
		if (requestURL.getHost() != "menu.2ch.net" && requestURL.replaceHost("2ch.net", "5ch.net")) {
			force5ch = true;
			log_printf(1, "Detected *.2ch.net URL, changed target URL to %s\n", requestURL.absoluteString().c_str());
		}
	}

	/* parse request headers */
	while (socketToClient->readLine(buf, 16384)) {
		if (!strcmp("\r\n",buf)) break;
		requestHeaders.add(buf);
		log_printf(5, "  %s", buf);
	}
	if (requestHeaders.hasNameAndValue("Transfer-Encoding", "chunked")) {
		isClientChunked = true;
	}
	else if (requestHeaders.has("Content-Length")) {
		content_length = atoi(requestHeaders.get("Content-Length").c_str());
	}
	if (requestHeaders.has("Expect")) {
		if (!strcasecmp(requestHeaders.get("Expect").c_str(), "100-continue") && !isClientHttp1_0) {
			log_printf(1, "Received Expect: 100-continue header, sending 100 Continue response to the client\n");
			socketToClient->writeString("HTTP/1.1 100 Continue\r\n\r\n");
		}
	}
	
	if (regexec(&regex, requestURL.absoluteString().c_str(), 6, match, 0) != REG_NOMATCH) {
		const std::string &url = requestURL.absoluteString();
		threadIdentifier.hostPrefix = url.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
		threadIdentifier.host = requestURL.getHost();
		threadIdentifier.board = url.substr(match[4].rm_so, match[4].rm_eo - match[4].rm_so);
		threadIdentifier.key = url.substr(match[5].rm_so, match[5].rm_eo - match[5].rm_so);
		if (requestURL.isKindOfHost("talk-platform.com")) {
			if (!appKey || !(api_mode & 8)) datProxyMode = 3;
			else datProxyMode = 2;
		}
		else if (direct_dat) {
			long datStatus;
			bool foundAsKakolog;
			BBS2chProxyURL newURL = getRawDatURLAndStatus(threadIdentifier, requestHeaders, false, &datStatus, &foundAsKakolog);
			if (datStatus == 200) {
				directDatDownloading = 1;
				if (foundAsKakolog) {
					requestURL = newURL;
				}
			} else {
				if (datStatus == 404) {
					if (!requestHeaders.has("Range") && !requestURL.hostStartsWith("headline.")) datProxyMode = 1;
					else {
						statusCode = 302;
						socketToClient->sendBasicHeaders(302, "Found");
						if (0 >= socketToClient->writeString("Location: http://www2.2ch.net/live.html\r\n")) goto end;
						if (0 >= socketToClient->writeString("\r\n")) goto end;
						goto end;
					}
				} else {
					statusCode = 503;
					socketToClient->sendResponse(503, "Service Unavailable");
					goto end;
				}
			}
		}
		else if ((appKey && (api_mode & 1)) || !requestURL.hostStartsWith("headline.")) datProxyMode = 1;
	}
	else if (regexec(&regex_kako, requestURL.absoluteString().c_str(), 8, match, 0) != REG_NOMATCH) {
		const std::string &url = requestURL.absoluteString();
		threadIdentifier.hostPrefix = url.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
		threadIdentifier.host = requestURL.getHost();
		threadIdentifier.board = url.substr(match[4].rm_so, match[4].rm_eo - match[4].rm_so);
		threadIdentifier.key = url.substr(match[7].rm_so, match[7].rm_eo - match[7].rm_so);
		if (requestURL.isKindOfHost("talk-platform.com")) {
			if (!appKey || !(api_mode & 8)) datProxyMode = 3;
			else datProxyMode = 2;
		}
		else if (direct_dat) {
			long datStatus;
			bool foundAsKakolog;
			BBS2chProxyURL newURL = getRawDatURLAndStatus(threadIdentifier, requestHeaders, true, &datStatus, &foundAsKakolog);
			if (datStatus == 200) {
				directDatDownloading = 1;
				requestURL = newURL;
			} else {
				if (!requestHeaders.has("Range")) datProxyMode = 3;
				else {
					statusCode = 503;
					socketToClient->sendResponse(503, "Service Unavailable");
					goto end;
				}
			}
		}
		else datProxyMode = 3;
	}
	else if (regexec(&regex_offlaw, requestURL.absoluteString().c_str(), 5, match, 0) != REG_NOMATCH) {
		const char *tmp = requestURL.absoluteString().c_str();
		const char *thread = strstr(tmp, "key=");
		if (thread) {
			match[6].rm_so = thread+4-tmp;
			match[6].rm_eo = thread+4-tmp;
			const char *ptr = thread+4;
			while (*ptr != '&' && *ptr != 0) {
				ptr++;
				match[6].rm_eo++;
			}
			if (match[6].rm_so != match[6].rm_eo) {
				const std::string &url = requestURL.absoluteString();
				threadIdentifier.hostPrefix = url.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
				threadIdentifier.host = requestURL.getHost();
				threadIdentifier.board = url.substr(match[4].rm_so, match[4].rm_eo - match[4].rm_so);
				threadIdentifier.key = url.substr(match[6].rm_so, match[6].rm_eo - match[6].rm_so);
				if (direct_dat) {
					long datStatus;
					bool foundAsKakolog;
					BBS2chProxyURL newURL = getRawDatURLAndStatus(threadIdentifier, requestHeaders, true, &datStatus, &foundAsKakolog);
					if (datStatus == 200) {
						directDatDownloading = 1;
						requestURL = newURL;
					} else {
						if (!requestHeaders.has("Range")) datProxyMode = 3;
						else {
							statusCode = 503;
							socketToClient->sendResponse(503, "Service Unavailable");
							goto end;
						}
					}
				}
				else datProxyMode = 3;
			}
		}
	}
	else if (api_override || talk_to_5ch) {
		const std::string &url = requestURL.absoluteString();
		if (api_override && regexec(&regex_api, url.c_str(), 5, match, 0) != REG_NOMATCH) {
			threadIdentifier.hostPrefix = url.substr(match[2].rm_so, match[2].rm_eo - match[2].rm_so);
			threadIdentifier.host = threadIdentifier.hostPrefix;
			threadIdentifier.host += isPinkPrefix(threadIdentifier.hostPrefix) ? ".bbspink.com" : ".5ch.net";
			threadIdentifier.board = url.substr(match[3].rm_so, match[3].rm_eo - match[3].rm_so);
			threadIdentifier.key = url.substr(match[4].rm_so, match[4].rm_eo - match[4].rm_so);
			if (direct_dat) {
				long datStatus;
				bool foundAsKakolog;
				BBS2chProxyURL newURL = getRawDatURLAndStatus(threadIdentifier, requestHeaders, false, &datStatus, &foundAsKakolog);
				if (datStatus == 200) {
					directDatDownloading = 2;
					strcpy(method, "GET");
					requestURL = newURL;
					requestHeaders.set("Host", newURL.getHost());
					requestHeaders.remove("Content-Length");
					requestHeaders.remove("Content-Type");
				} else {
					if (datStatus == 404) {
						if (!requestHeaders.has("Range") && !requestURL.hostStartsWith("headline.")) datProxyMode = 3;
						else {
							statusCode = 200;
							socketToClient->sendBasicHeaders(200, "OK");
							if (0 >= socketToClient->writeString("Thread-Status: 8\r\n")) goto end;
							if (0 >= socketToClient->writeString("User-Status: 3\r\n\r\n")) goto end;
							goto end;
						}
					} else {
						statusCode = 503;
						socketToClient->sendResponse(503, "Service Unavailable");
						goto end;
					}
				}
			} else if (appKey && !(api_mode & 1)) {
				datProxyMode = 2;
			} else {
				datProxyMode = 3;
			}
			if (datProxyMode == 3) {
				requestHeaders.set("X-Proxy2ch-API-Bypass", "1");
			}
		}
		else if (api_override && regexec(&regex_api_auth, url.c_str(), 2, match, 0) != REG_NOMATCH) {
			/* return dummy response immediately */
			log_printf(1, "Returning dummy response because API overriding is enabled\n");
			statusCode = 200;
			socketToClient->sendBasicHeaders(200, "OK");
			if (0 >= socketToClient->writeString("Content-Type: text/plain\r\n")) goto end;
			if (0 >= socketToClient->writeString("Content-Length: 203\r\n\r\n")) goto end;
			if (0 >= socketToClient->writeString("SESSION-ID=Monazilla/1.00:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")) goto end;
			goto end;
		}
		else if (talk_to_5ch && regexec(&regex_talk, url.c_str(), 3, match, 0) != REG_NOMATCH) {
			std::string board = url.substr(match[2].rm_so, match[2].rm_eo - match[2].rm_so);
			std::string path = url.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
			std::string host = boardManager.getServerForBoard(board);
			if (!host.empty()) {
				requestURL = BBS2chProxyURL(force_5chnet_https ? "https" : "http", host.c_str(), 0, path.c_str());
				requestHeaders.set("Host", requestURL.getHost());
				log_printf(1, "Enabled Talk to 5ch proxy, will connect to %s\n", requestURL.absoluteString().c_str());
			}
		}
		else if (talk_to_5ch && regexec(&regex_talk_api, url.c_str(), 3, match, 0) != REG_NOMATCH) {
			threadIdentifier.board = url.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
			threadIdentifier.key = url.substr(match[2].rm_so, match[2].rm_eo - match[2].rm_so);
			threadIdentifier.host = boardManager.getServerForBoard(threadIdentifier.board);
			threadIdentifier.hostPrefix = threadIdentifier.host.substr(0, threadIdentifier.host.find("."));
			if (!threadIdentifier.host.empty()) {
				log_printf(1, "Enabled Talk to 5ch proxy, will connect to %s\n", threadIdentifier.host.c_str());
				if (direct_dat) {
					long datStatus;
					bool foundAsKakolog;
					BBS2chProxyURL newURL = getRawDatURLAndStatus(threadIdentifier, requestHeaders, false, &datStatus, &foundAsKakolog);
					if (datStatus == 200) {
						directDatDownloading = 2;
						strcpy(method, "GET");
						requestURL = newURL;
						requestHeaders.set("Host", newURL.getHost());
						requestHeaders.remove("Content-Length");
						requestHeaders.remove("Content-Type");
					} else {
						if (datStatus == 404) {
							if (!requestHeaders.has("Range") && threadIdentifier.hostPrefix != "headline") {
								datProxyMode = 3;
								requestHeaders.set("X-Proxy2ch-API-Bypass", "1");
							}
							else {
								statusCode = 200;
								socketToClient->sendBasicHeaders(200, "OK");
								if (0 >= socketToClient->writeString("Thread-Status: 8\r\n")) goto end;
								if (0 >= socketToClient->writeString("User-Status: 3\r\n\r\n")) goto end;
								goto end;
							}
						} else {
							statusCode = 503;
							socketToClient->sendResponse(503, "Service Unavailable");
							goto end;
						}
					}
				}
				else if ((appKey && (api_mode & 1)) || threadIdentifier.hostPrefix != "headline") datProxyMode = 1;
			}
		}
		else if (talk_to_5ch && regexec(&regex_talk_bbscgi, url.c_str(), 1, match, 0) != REG_NOMATCH) {
			talkTo5chBbsCgi = true;
		}
	}
	
	if (datProxyMode) {
		if (!appKey || (datProxyMode == 1 && !(api_mode & 1)) || datProxyMode == 3) {
			if (threadIdentifier.host == "classic.talk-platform.com") {
				isHttps = true;
				log_printf(1, "Retrieving thread via json...\n");
				BBS2chProxyHTML2DatTalk html2dat(threadCache, threadIdentifier, curl);
				html2dat.setRequestHeaders(requestHeaders);
				statusCode = datProxy(html2dat, method, requestHeaders);
			}
#if 0
			// this will be used when legacy read.cgi stops working completely
			else if (threadIdentifier.host.substr(threadIdentifier.hostPrefix.size()) == ".5ch.net") {
				isHttps = true;
				log_printf(1, "Retrieving thread via itest json...\n");
				BBS2chProxyHTML2Dat5chItest html2dat(threadCache, threadIdentifier, curl);
				html2dat.setRequestHeaders(requestHeaders);
				statusCode = datProxy(html2dat, method, requestHeaders);
			}
#endif
			else {
				log_printf(1, "Retrieving thread via read.cgi...\n");
				BBS2chProxyHTML2Dat5ch html2dat(threadCache, threadIdentifier, force_5chnet_https || isHttps, curl);
				html2dat.setRequestHeaders(requestHeaders);
				statusCode = datProxy(html2dat, method, requestHeaders);
			}
		}
		else {
			log_printf(1, "Retrieving thread via API...\n");
			std::string targetURL = "https://";
			targetURL += api_server;
			targetURL += "/v1/";
			targetURL += threadIdentifier.hostPrefix;
			targetURL += '/';
			targetURL += threadIdentifier.board;
			targetURL += '/';
			targetURL += threadIdentifier.key;
			isHttps = true;
			statusCode = datProxyAPI(targetURL, method, requestHeaders);
		}
	}
	else {
		if (force_5chnet_https && !isHttps && requestURL.isFamilyOf5chNet()) {
			requestURL.setScheme("https");
			isHttps = true;
			log_printf(1, "The host %s is 5ch.net family, connecting with HTTPS\n", requestURL.getHost().c_str());
		}
		else if (!isHttps && requestURL.isKindOfHost("talk-platform.com")) {
			requestURL.setScheme("https");
			isHttps = true;
			log_printf(1, "The host talk-platform.com only accepts HTTPS, connecting with HTTPS\n");
		}
		
		if (bbsmenu_url && requestURL.equalsTo(BBS2chProxyURL(bbsmenu_url), true)) {
			log_printf(1, "Running as a BBS menu proxy...\n");
			statusCode = bbsmenuProxy(requestURL.absoluteString(), method, requestHeaders);
		}
		else if (fool_janestyle && requestURL.equalsTo(BBS2chProxyURL("http", "janesoft.net", 0, "/janestyle/setting.php"), true)) {
			log_printf(1, "Fooling JaneStyle as if API is still alive...\n");
			socketToClient->sendBasicHeaders(200, "OK");
			socketToClient->writeString("\r\n0");
			statusCode = 200;
		}
		else if (talk_to_5ch && requestURL.equalsTo(BBS2chProxyURL("https", "classic.talk-platform.com", 0, "/bbsmenu.json"), true)) {
			log_printf(1, "Returning bbsmenu.json including fake 5ch.net boards...\n");
			socketToClient->sendBasicHeaders(200, "OK");
			socketToClient->writeString("Content-Type: application/json\r\n\r\n");
			socketToClient->writeString(boardManager.getBoardJSONForTalkAndFake5ch());
			statusCode = 200;
		}
		else if (requestURL.getHost() == "localhost" && requestURL.getPort() == serverPort && requestURL.isHttp()) {
			if (requestURL.getPath() == "/5ch/bbsmenu.html") {
				std::map<std::string, std::string> query = requestURL.getQuery();
				bool useHttps = query["https"] == "true";
				bool use2ch = query["2ch"] == "true";
				bool hasTalk = query["talk"] == "true";
				log_printf(1, "Returning bbsmenu.html...\n");
				socketToClient->sendBasicHeaders(200, "OK");
				socketToClient->writeString("Content-Type: text/html; charset=Shift_JIS\r\n\r\n");
				socketToClient->writeString(boardManager.getBoardHTML(!useHttps, use2ch, hasTalk));
				statusCode = 200;
			}
			else if (requestURL.getPath() == "/5ch/bbsmenu.json") {
				std::map<std::string, std::string> query = requestURL.getQuery();
				bool useHttps = query["https"] == "true";
				bool use2ch = query["2ch"] == "true";
				bool hasTalk = query["talk"] == "true";
				log_printf(1, "Returning bbsmenu.json...\n");
				socketToClient->sendBasicHeaders(200, "OK");
				socketToClient->writeString("Content-Type: application/json\r\n\r\n");
				socketToClient->writeString(boardManager.getBoardJSON(!useHttps, use2ch, hasTalk));
				statusCode = 200;
			}
			else {
				socketToClient->sendResponse(404, "Not Found");
				statusCode = 404;
			}
		}
		else if (((subject_to_lastmodify == 1 && requestURL.isFamilyOf5chNet()) ||
			      (subject_to_lastmodify == 2 && subject_to_lastmodify_hosts.find(requestURL.getHost()) != subject_to_lastmodify_hosts.end())) &&
			     requestURL.pathEndsWith("/subject.txt") && requestURL.numberOfPathComponents() == 2) {
			log_printf(1, "Running as subject.txt to lastmodify.txt proxy...\n");
			statusCode = subjectTxtProxy(requestURL, method, requestHeaders);
		}
		else {
			bool isPostRequest = !strcasecmp(method, "POST");
			bool isPutRequest = !strcasecmp(method, "PUT");
			char *postdata = NULL;
			if (isPostRequest && (requestURL.isFamilyOf5chNet() || requestURL.isKindOfHost("classic.talk-platform.com")) && requestURL.pathStartsWith("/test/bbs.cgi")) bbscgi = true;
			else if (talkTo5chBbsCgi) bbscgi = true;
			if (bbscgi) log_printf(1, "Looks like a request to bbs.cgi, will be modified before sending...\n");
			else if (directDatDownloading) log_printf(1, "Downloading .dat directly from %s...\n", requestURL.absoluteString().c_str());
			else log_printf(1, "Not a notable request, will be forwarded to server...\n");
			if (force_5chnet) {
				if (requestHeaders.has("Host")) {
					std::string host = requestHeaders.get("Host");
					size_t pos = host.find("2ch.net");
					if (pos != std::string::npos && pos+7 == host.length()) {
						if (pos == 0 || host[pos-1] == '.') {
							host.replace(pos, 1, "5");
							requestHeaders.set("Host", host);
						}
					}
				}
				if (bbscgi && requestHeaders.has("Referer")) {
					std::string referrer = requestHeaders.get("Referer");
					size_t pos = referrer.find("2ch.net");
					if (pos != std::string::npos) {
						if (pos == 0 || referrer[pos-1] == '.') {
							referrer.replace(pos, 1, "5");
							requestHeaders.set("Referer", referrer);
						}
					}
				}
			}
			requestHeaders.remove("Connection");
			if (user_agent) requestHeaders.remove("User-Agent");
			if (bbscgi && (content_length > 0 || isClientChunked)) {
				bool isNotFormURLEncoded = false;
				if (requestHeaders.has("Content-Type") && requestHeaders.get("Content-Type").find("application/x-www-form-urlencoded") == std::string::npos) {
					isNotFormURLEncoded = true;
				}
				if (!isNotFormURLEncoded) {
					requestHeaders.remove("Content-Length");
					if (!bbscgi_headers.empty()) {
						for (BBS2chProxyHttpHeaders::iterator it = bbscgi_headers.begin(); it != bbscgi_headers.end(); ++it) {
							if (requestHeaders.has(it->first->getName())) {
								log_printf(1, "Ignoring header \"%s\" because custom header will be appended\n", it->first->getName().c_str());
								requestHeaders.remove(it->first->getName());
							}
						}
					}
					if (isClientChunked) {
						content_length = readChunkedBodyIntoBuffer(&postdata, socketToClient);
						requestHeaders.remove("Transfer-Encoding");
					}
					else {
						postdata = (char *)calloc(content_length+1, 1);
						content_length = socketToClient->read(postdata, content_length);
					}
					if (gikofix) {
						char *ptr = postdata+content_length-1;
						while (ptr >= postdata && (*ptr == '\r' || *ptr == '\n')) {
							*ptr-- = 0;
						}
					}
					BBS2chProxyFormData requestBody(postdata, strlen(postdata));
					statusCode = bbsCgiProxy(requestURL, requestHeaders, requestBody, talkTo5chBbsCgi);
					if (statusCode >= 0) {
						free(postdata);
						goto end;
					}
				}
			}
			curl_slist *headersForCurl = NULL;
			headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl);
			if (!requestHeaders.has("Expect")) {
				headersForCurl = curl_slist_append(headersForCurl, "Expect:");
			}
			
			if(curl) {
				CURLcode res;
				if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
				curl_easy_setopt(curl, CURLOPT_URL, requestURL.absoluteString().c_str());
				curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
				curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
				curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_proxy);
				curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
				if (postdata) curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postdata);
				else {
					if(content_length) {
						/* set Content-Length explicitly via API to work properly with curl >= 7.66.0 */
						if(isPostRequest)
							curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, content_length);
						else if(isPutRequest)
							curl_easy_setopt(curl, CURLOPT_INFILESIZE, content_length);
					}
					curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback_proxy);
					curl_easy_setopt(curl, CURLOPT_READDATA, this);
				}
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
				curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
				//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
				if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
				curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
				curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headersForCurl);
				if(user_agent) {
					curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
				}
				if(isPostRequest) {
					curl_easy_setopt(curl, CURLOPT_POST, 1L);
				}
				else if(isPutRequest) {
					curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
				}
				else if(!strcasecmp(method, "HEAD")) {
					curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
				}
				else if(!strcasecmp(method, "OPTIONS")) {
					curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "OPTIONS");
				}
				if(proxy_server) {
					curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
					curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
					curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
				}
				res = curl_easy_perform(curl);
				if(res != CURLE_OK) {
					log_printf(0, "curl error: %s (%s)\n", curl_easy_strerror(res), requestURL.absoluteString().c_str());
					if(!status) socketToClient->sendResponse(503, "Service Unavailable");
					statusCode = 503;
				}
				else {
					if(isResponseChunked) {
						socketToClient->writeString("0\r\n\r\n");
					}
					curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
				}
				curl_easy_reset(curl);
			}
			curl_slist_free_all(headersForCurl);
			if (postdata) free(postdata);
		}
	}
	
  end:
	if(statusCode) log_printf(1, "Returned status code %d to client\n",statusCode);
	if(buf) free(buf);
	if(socketToClient) socketToClient->close();
}

int BBS2chProxyConnection::datProxy(IBBS2chProxyHTML2Dat &html2dat, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	std::string dat;
	long statusCode = 0;
	long rangeStart = 0, rangeEnd = 0;
	time_t lastModified = 0;
	time_t ifModifiedSince = 0;
	if(requestHeaders.has("Range")) {
		std::string value = requestHeaders.get("Range");
		if(value.find("bytes=") == 0 && value.find(",") == std::string::npos) {
			char *ptr = (char *)value.c_str() + 6;
			if(*ptr == '-') {
				rangeStart = atoi(ptr);
			}
			else {
				rangeStart = strtol(ptr, &ptr, 10);
				if(*ptr == '-') ptr++;
				if(*ptr && *ptr != '\r') {
					rangeEnd = strtol(ptr, NULL, 10);
					if(rangeEnd && rangeStart > rangeEnd) {
						socketToClient->sendResponse(416, "Requested range not satisfiable");
						statusCode = 416;
						goto last;
					}
				}
			}
			//fprintf(stderr, "range=%ld-%ld\n",rangeStart,rangeEnd);
		}
		else {
			socketToClient->sendResponse(416, "Requested range not satisfiable");
			statusCode = 416;
			goto last;
		}
	}
	if(requestHeaders.has("If-Modified-Since")) {
#if LIBCURL_VERSION_NUM >= 0x070c02 /* curl 7.12.2 or later */
		ifModifiedSince = curl_getdate(requestHeaders.get("If-Modified-Since").c_str(), NULL);
#else
		struct tm time_ = {0};
		strptime(requestHeaders.get("If-Modified-Since").c_str(), HTTP_TIMESTAMP_FMT_PARSE, &time_);
		ifModifiedSince = mktime(&time_);
#endif
	}
	if(rangeStart > 0) {
		PBBS2chProxyThreadInfo info = threadCache->get(html2dat.getKey());
		log_printf(5,"range request from %ld bytes\n",rangeStart);
		if(info) {
			int from = info->lastResNum;
			int alreadyRead = info->cachedSize;
			int lastResLength = info->cachedData.size();
			log_printf(5,"hit %s: cached %d bytes, last res size %d\n",html2dat.getKey().c_str(),alreadyRead,lastResLength);
			if(rangeStart <= alreadyRead && rangeStart >= alreadyRead - lastResLength) {
				std::string updated = html2dat.generateDatFrom(from, &lastModified, true, &statusCode);
				if (!updated.empty()) {
					if (ifModifiedSince && lastModified && updated.size() == lastResLength) {
						if (ifModifiedSince >= lastModified) {
							socketToClient->sendResponse(304, "Not Modified");
							log_printf(5, "not modified!\n");
							statusCode = 304;
							goto last;
						}
					}
					if (updated.size() >= lastResLength) {
						dat = std::string(alreadyRead - lastResLength, ' ');
						dat += updated;
						if (!rangeEnd) rangeEnd = dat.size() - 1;
						if (rangeStart > rangeEnd) {
							socketToClient->sendResponse(416, "Requested range not satisfiable");
							statusCode = 416;
							goto last;
						}
						statusCode = 206;
						log_printf(5, "cache hit; reconstructed data length:%ld\n", (long)dat.size());
					}
					else {
						log_printf(5, "cache misshit?\n");
						socketToClient->sendResponse(416, "Requested range not satisfiable");
						statusCode = 416;
						goto last;
					}
				}
				if (!dat.empty()) goto resp;
			}
			else {
				log_printf(5,"invalid cache contents\n");
				threadCache->remove(html2dat.getKey());
			}
		}
	}
	
	{
		dat = html2dat.generateDatFrom(1, &lastModified, false, &statusCode);
		if(dat.empty()) {
			socketToClient->sendResponse(503, "Service Unavailable");
			statusCode = 503;
			goto last;
		}
		if((rangeStart || rangeEnd) && !dat.empty()) {
			if(!rangeEnd) rangeEnd = dat.size() - 1;
			if(rangeStart < 0) rangeStart = dat.size() + rangeStart;
			if (ifModifiedSince && lastModified) {
				if (rangeStart == dat.size() || (rangeStart == dat.size() - 1 && rangeEnd == dat.size() - 1)) {
					if (ifModifiedSince >= lastModified) {
						socketToClient->sendResponse(304, "Not Modified");
						log_printf(5,"not modified!\n");
						statusCode = 304;
						goto last;
					}
				}
			}
			if(rangeStart < dat.size() && rangeEnd < dat.size() && rangeStart <= rangeEnd) {
				statusCode = 206;
			}
			else {
				socketToClient->sendResponse(416, "Requested range not satisfiable");
				statusCode = 416;
				goto last;
			}
		}
	}

resp:
	if(statusCode == 206) socketToClient->sendBasicHeaders(statusCode, "Partial Content");
	else socketToClient->sendBasicHeaders(statusCode, "OK");
	if(0 >= socketToClient->writeString("Content-Type: text/plain\r\n")) goto last;
	if(0 >= socketToClient->writeString("Accept-Ranges: bytes\r\n")) goto last;
	if (requestHeaders.has("X-Proxy2ch-API-Bypass")) {
		if (0 >= socketToClient->writeString("Thread-Status: 1\r\n")) goto last;
		if (0 >= socketToClient->writeString("User-Status: 3\r\n")) goto last;
	}
	if(statusCode == 206) {
		std::ostringstream ss;
		ss << "Content-Range: bytes " << rangeStart << "-" << rangeEnd << "/" << dat.size() << "\r\n";
		if (0 >= socketToClient->writeString(ss.str())) goto last;
		//fprintf(stderr,"Content-Length: %ld\r\n",rangeEnd - rangeStart + 1);
		//fprintf(stderr,"Content-Range: bytes %ld-%ld/%ld\r\n",rangeStart,rangeEnd,(long)dat.size());
		dat = dat.substr(rangeStart, rangeEnd - rangeStart + 1);
	}
	{
		std::ostringstream ss;
		ss << "Content-Length: " << dat.size() << "\r\n";
		if(0 >= socketToClient->writeString(ss.str())) goto last;
	}
	if(lastModified) {
		struct tm time_ = {0};
		char date[256];
		gmtime_r(&lastModified,&time_);
		strftime(date, 256, HTTP_TIMESTAMP_FMT_PRINT, &time_);
		std::string header = "Last-Modified: ";
		header += date;
		header += "\r\n";
		if(0 >= socketToClient->writeString(header)) goto last;
		//fprintf(stderr,"Last-Modified: %s\r\n",date);
	}
	if(0 >= socketToClient->writeString("\r\n")) goto last;
	
	if(!dat.empty() && statusCode >= 200 && statusCode < 300 && strcasecmp(method, "HEAD")) {
		if(0 >= socketToClient->writeString(dat)) goto last;
	}
	
last:
	return statusCode;
}

int BBS2chProxyConnection::datProxyAPI(const std::string &url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	long statusCode = 0;
	const std::string &postBody = auth.requestBodyForURL(url.c_str(), curl);
	bool directMode = false;
	if (postBody.empty()) {
		socketToClient->sendResponse(401, "Unauthorized");
		return 401;
	}
	/* just read and strip off post body */
	if (!strcasecmp(method, "POST")) {
		char *postdata = NULL;
		if (isClientChunked) {
			readChunkedBodyIntoBuffer(&postdata, socketToClient);
		}
		else if (content_length) {
			postdata = (char *)calloc(content_length+1, 1);
			socketToClient->read(postdata, content_length);
		}
		if (postdata && strstr(postdata, "sid=")) directMode = true;
		if (postdata) free(postdata);
	}
	if (curl) {
		CURLcode res;
		struct curl_slist *headersForCurl = NULL;
		BBS2chProxyHttpHeaders receivedHeaders;
		std::vector<char> receivedBody;
		bool explicitAcceptEncoding = false;
		headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "Range");
		headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "If-Modified-Since");
		if (!api_dat_headers.has("Accept-Encoding")) {
			headersForCurl = requestHeaders.appendToCurlSlist(headersForCurl, "Accept-Encoding");
		}
		else explicitAcceptEncoding = true;
		if (!api_dat_headers.empty()) {
			std::set<std::string> excludes;
			if (explicitAcceptEncoding) {
				excludes.insert("accept-encoding");
			}
			headersForCurl = api_dat_headers.appendToCurlSlist(headersForCurl, excludes);
		}
		
		if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headersForCurl);
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &receivedBody);
		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_download);
		curl_easy_setopt(curl, CURLOPT_HEADERDATA, &receivedHeaders);
		curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if (proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if (!api_dat_headers.has("User-Agent")) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, "");
		}
		if (explicitAcceptEncoding) {
			curl_easy_setopt(curl, CURLOPT_ENCODING, api_dat_headers.get("Accept-Encoding").c_str());
		}
		curl_easy_setopt(curl, CURLOPT_POST, 1L);
#if LIBCURL_VERSION_NUM >= 0x071101
		curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, postBody.c_str());
#else
		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postBody.c_str());
#endif
		//return;
		res = curl_easy_perform(curl);
		if (res == CURLE_OK) {
			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
			int threadStatus = 0;
			if (!directMode && receivedHeaders.has("Thread-Status")) {
				threadStatus = atoi(receivedHeaders.get("Thread-Status").c_str());
			}
			if (threadStatus == 1 || threadStatus == 2 || directMode) {
				if (explicitAcceptEncoding && receivedHeaders.has("Content-Encoding")) {
					std::ostringstream ss;
					ss << receivedBody.size();
					receivedHeaders.set("Content-Length", ss.str());
					receivedHeaders.remove("Content-Encoding");
				}
				if (requestHeaders.has("Range") && statusCode == 200 && !strcmp(api_server, "api.talk-platform.com")) {
					/* api.talk-platform.com does not return 206 for ranged requests. so we try emulating 206 or 304 response. sucks! */
					if (receivedBody.size() == 0) {
						if (0 >= socketToClient->writeString("HTTP/1.1 304 Not Modified\r\n")) goto last;
						receivedHeaders.remove("Content-Length");
						statusCode = 304;
					} else if (requestHeaders.has("If-Modified-Since") && receivedBody.size() <= 1) {
						/* we will never reach here since api.talk-platform.com does not return Last-Modified header. sucks! */
						if (0 >= socketToClient->writeString("HTTP/1.1 304 Not Modified\r\n")) goto last;
						receivedHeaders.remove("Content-Length");
						statusCode = 304;
					} else {
						if (0 >= socketToClient->writeString("HTTP/1.1 206 Partial Content\r\n")) goto last;
						if (!receivedHeaders.has("Content-Length")) {
							std::ostringstream ss;
							ss << receivedBody.size();
							receivedHeaders.set("Content-Length", ss.str());
						}
						size_t pos = requestHeaders.get("Range").find("bytes=");
						if (pos != std::string::npos) {
							pos += 6;
							std::string range = requestHeaders.get("Range").substr(pos);
							const char *rangeData = range.c_str();
							char *ptr;
							unsigned long rangeStart = strtoul(rangeData, &ptr, 10);
							if (*ptr == '-' && *(ptr+1) == 0) {
								unsigned long rangeEnd = rangeStart + receivedBody.size() - 1;
								std::ostringstream ss;
								ss << "bytes " << rangeStart << "-" << rangeEnd << "/" << rangeEnd + 1;
								receivedHeaders.set("Content-Range", ss.str());
							}
						}
						statusCode = 206;
					}
				}
				else {
					if (0 >= socketToClient->writeString(receivedHeaders.getStatusLine() + "\r\n")) goto last;
				}
				for (BBS2chProxyHttpHeaders::iterator it = receivedHeaders.begin(); it != receivedHeaders.end(); ++it) {
					if (0 >= socketToClient->writeString(it->second + "\r\n")) goto last;
				}
				if (0 >= socketToClient->writeString("\r\n")) goto last;
				if (statusCode != 304) {
					if (receivedBody.size() > socketToClient->write(&receivedBody.front(), receivedBody.size())) goto last;
				}
				goto last;
			}
			else if (threadStatus == 8) {
				socketToClient->sendBasicHeaders(302, "Found");
				if (0 >= socketToClient->writeString("Location: http://www2.2ch.net/live.html\r\n")) goto last;
				if (0 >= socketToClient->writeString("\r\n")) goto last;
				statusCode = 302;
				goto last;
			}
			else {
				if (statusCode < 400) {
					socketToClient->sendResponse(401, "Unauthorized");
					statusCode = 401;
				}
				else {
					socketToClient->sendResponse(503, "Service Unavailable");
					statusCode = 503;
				}
				receivedBody.push_back('\0');
				if (!strncasecmp(&receivedBody.front(), "ng (", 4)) {
					log_printf(0, "API gateway returned error: %s\n", &receivedBody.front());
				}
			}
			//fprintf(stderr,"%ld\n",statusCode);
		}
		else {
			log_printf(0, "curl error: %s\n", curl_easy_strerror(res));
			socketToClient->sendResponse(503, "Service Unavailable");
			statusCode = 503;
		}
last:
		curl_easy_reset(curl);
		curl_slist_free_all(headersForCurl);
	}
	return statusCode;
}

int BBS2chProxyConnection::bbsmenuProxy(const std::string &url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	long statusCode = 0;
	std::vector<char> dat;
	std::string outHTML;
	if(curl) {
		CURLcode res;
		if(curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
		curl_easy_setopt(curl, CURLOPT_ENCODING, "");
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dat);
		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
		curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
		if(force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
		if(proxy_server) {
			curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
			curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
			curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
		}
		if(user_agent) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
		}
		else if(requestHeaders.has("User-Agent")) {
			curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
		}
		res = curl_easy_perform(curl);
		if(res == CURLE_OK) {
			curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &statusCode);
			if(statusCode == 200 && dat.size()) {
				dat.push_back('\0');
				char *ptr = &dat.front();
				while(*ptr) {
					if(!strncasecmp(ptr,"<a href=",8)) {
						char *start = ptr+8;
						char *end = strchr(start,'>');
						if (end) {
							char *urlEnd = end;
							if (*start == '"') {
								start++;
								char *tmp = strchr(start, '"');
								if (tmp && tmp < end) urlEnd = tmp;
							}
							BBS2chProxyURL url(std::string(start, urlEnd-start).c_str());
							if (url.isKindOfHost("5ch.net")) {
								url.replaceHost("5ch.net", "2ch.net");
								url.setScheme("http");
								outHTML.append("<A HREF=");
								outHTML.append(url.absoluteString());
								outHTML.append(">");
								ptr = end+1;
								continue;
							} else if (url.isKindOfHost("bbspink.com")) {
								url.setScheme("http");
								outHTML.append("<A HREF=");
								outHTML.append(url.absoluteString());
								outHTML.append(">");
								ptr = end+1;
								continue;
							}
						}
					}
					outHTML.append(ptr++, 1);
				}
			}
		}
		else {
			log_printf(0,"curl error: %s (%s)\n", curl_easy_strerror(res), url.c_str());
			statusCode = 503;
		}
	}
	if(statusCode == 200) {
		std::ostringstream ss;
		ss << "Content-Length: " << outHTML.size() << "\r\n";
		socketToClient->sendBasicHeaders(statusCode, "OK");
		if(0 >= socketToClient->writeString("Content-Type: text/html\r\n")) goto last;
		if(0 >= socketToClient->writeString(ss.str())) goto last;
		if(0 >= socketToClient->writeString("\r\n")) goto last;
		if(strcasecmp(method, "HEAD")) {
			if(0 >= socketToClient->writeString(outHTML)) goto last;
		}
	}
	else {
		socketToClient->sendResponse(503, "Service Unavailable");
		statusCode = 503;
	}
	
last:
	if(curl) curl_easy_reset(curl);
	return statusCode;
}

int BBS2chProxyConnection::bbsCgiProxy(BBS2chProxyURL &url, BBS2chProxyHttpHeaders &requestHeaders, BBS2chProxyFormData &requestBody, bool talkTo5ch)
{
	if (talkTo5ch) {
		std::string board = requestBody.get("bbs");
		if (board.size() >= 10 && !board.compare(0, 9, "5channel_")) {
			board = board.substr(9);
			std::string host = boardManager.getServerForBoard(board);
			if (!host.empty()) {
				std::string referrer = force_5chnet_https ? "https://" : "http://";
				referrer += host;
				referrer += "/";
				referrer += board;
				referrer += "/";
				requestHeaders.set("Host", host);
				requestHeaders.set("Referer", referrer);
				requestHeaders.remove("X-Write-Key");
				requestHeaders.remove("X-Write-Token");
				requestBody.set("bbs", board);
				requestBody.remove("sid");
				requestBody.remove("appkey");
				requestBody.remove("anonymous");
				url = BBS2chProxyURL(force_5chnet_https ? "https" : "http", host.c_str(), 0, "/test/bbs.cgi");
				log_printf(1, "Enabled Talk to 5ch proxy, will connect to %s\n", url.absoluteString().c_str());
			}
			else return -1;
		}
		else return -1;
		return BBS2chProxyTalkTo5chPoster(url, requestHeaders, requestBody, this).post();
	}
	if (url.isKindOfHost("talk-platform.com")) {
		return BBS2chProxyTalkPoster(url, requestHeaders, requestBody, this).post();
	}
	return BBS2chProxy5chPoster(url, requestHeaders, requestBody, this).post();
}

BBS2chProxyURL BBS2chProxyConnection::getRawDatURLAndStatus(const BBS2chThreadIdentifier &threadIdentifier, BBS2chProxyHttpHeaders &requestHeaders, bool findKakologOnly, long *status, bool *foundAsKakolog)
{
	long statusCode = 0;
	std::string datURL;
	for (int i=findKakologOnly ? 1 : 0; i<2; i++) {
		datURL = force_5chnet_https ? "https://" : "http://";
		datURL += threadIdentifier.host;
		datURL += '/';
		datURL += threadIdentifier.board;
		if (i) {
			datURL += "/oyster/";
			datURL += threadIdentifier.key.substr(0, std::max(threadIdentifier.key.size() - 6, (size_t)0));
			datURL += "/";
			datURL += threadIdentifier.key;
			datURL += ".dat";
		} else {
			datURL += "/dat/";
			datURL += threadIdentifier.key;
			datURL += ".dat";
		}
		statusCode = 0;
		if (curl) {
			CURLcode res;
			if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
			curl_easy_setopt(curl, CURLOPT_URL, datURL.c_str());
			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
			curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
			//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
			if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
			curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
			curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
			if (user_agent) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
			}
			else if (requestHeaders.has("User-Agent")) {
				curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
			}
			if (proxy_server) {
				curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
				curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
				curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
			}
			res = curl_easy_perform(curl);
			if (res == CURLE_OK) {
				curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
			}
			curl_easy_reset(curl);
			if (statusCode == 200) {
				if (foundAsKakolog) *foundAsKakolog = !!(i);
				break;
			}
		}
	}
	if (status) *status = statusCode;
	return BBS2chProxyURL(datURL.c_str());
}

struct ThreadInfo {
    std::string dat;
    std::string title;
    unsigned long long updated;
    bool operator<(const ThreadInfo& another) const {
        return updated > another.updated;
    };
};

int BBS2chProxyConnection::subjectTxtProxy(BBS2chProxyURL &url, const char *method, BBS2chProxyHttpHeaders &requestHeaders)
{
	long statusCode = 0;
	std::string path = url.getPath();
	size_t pos = path.find("/subject.txt");
	path = path.substr(0, pos);
	path += "/lastmodify.txt";
	BBS2chProxyURL newURL = BBS2chProxyURL(url, path.c_str());
	std::vector<char> data;
	BBS2chProxyHttpHeaders receivedHeaders;
	if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
	curl_easy_setopt(curl, CURLOPT_URL, newURL.absoluteString().c_str());
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
	curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
	curl_easy_setopt(curl, CURLOPT_ENCODING, "");
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
	curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback_download);
	curl_easy_setopt(curl, CURLOPT_HEADERDATA, &receivedHeaders);
	curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
	if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
	if (proxy_server) {
		curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
		curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
		curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
	}
	if (user_agent) {
		curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
	}
	else if (requestHeaders.has("User-Agent")) {
		curl_easy_setopt(curl, CURLOPT_USERAGENT, requestHeaders.get("User-Agent").c_str());
	}
	CURLcode res = curl_easy_perform(curl);
	if (res == CURLE_OK) {
		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
		if (statusCode == 200) {
			std::string outText;
			std::vector<ThreadInfo> threads;
			data.push_back('\0');
			char *ptr = &data.front();
			while (1) {
				char *start = ptr;
				char *lineEnd = strchr(ptr, '\n');
				ThreadInfo info;
				int i = 0;
				if (!lineEnd) break;
				while (ptr = strstr(start, "<>"), i < 6 && ptr != NULL && ptr < lineEnd) {
					if (i == 0) {
						info.dat = std::string(start, ptr-start);
					} else if (i == 1) {
						info.title = std::string(start, ptr-start);
					} else if (i == 2) {
						std::string suppl = " (" + std::string(start, ptr-start);
						suppl += ")";
						if (*(start-3) == ')') {
							size_t len1 = info.title.length();
							size_t len2 = suppl.length();
							if (len1 < len2 || info.title.compare(len1-len2, len2, suppl)) {
								info.title += suppl;
							}
						}
						else info.title += suppl;
					} else if (i == 5) {
						info.updated = strtoull(start, NULL, 10);
					}
					start = ptr + 2;
					i++;
				}
				if (i == 6) {
					threads.push_back(info);
				}
				ptr = lineEnd+1;
			}
			std::sort(threads.begin(), threads.end());
			for (std::vector<ThreadInfo>::iterator it = threads.begin(); it != threads.end(); ++it) {
				outText += it->dat;
				outText += "<>";
				outText += it->title;
				outText += "\n";
			}
			std::ostringstream ss;
			socketToClient->sendBasicHeaders(200, "OK");
			if (receivedHeaders.has("Last-Modified")) {
				socketToClient->writeString(receivedHeaders.getFull("Last-Modified", true));
			}
			socketToClient->writeString("Content-Type: text/plain\r\n");
			ss << "Content-Length: " << outText.size() << "\r\n\r\n";
			socketToClient->writeString(ss.str());
			if (strcasecmp(method, "HEAD")) {
				socketToClient->writeString(outText);
			}
		}
	}
	else {
		log_printf(0,"curl error: %s (%s)\n", curl_easy_strerror(res), newURL.absoluteString().c_str());
		statusCode = 503;
	}
	if (statusCode != 200) {
		socketToClient->sendResponse(statusCode, "Error");
	}
	curl_easy_reset(curl);
	return statusCode;
}

void BBS2chProxyConnection::compileRegex(void)
{
	static int compiled;
	if (compiled) return;
	regcomp(&regex, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com|talk-platform.com)(:[0-9]+)?/([^/]+)/dat/([0-9]+)\\.dat", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_kako, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com|talk-platform.com)(:[0-9]+)?/([^/]+)/(kako|oyster)/[0-9]+/([0-9]+/)?([0-9]+)\\.dat", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_offlaw, "^https?://([^:/.]+)\\.(2ch\\.net|5ch\\.net|bbspink\\.com)(:[0-9]+)?/test/offlaw2.so\\?.*bbs=([^&]+)", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_api, "^https?://api\\.[25]ch\\.net(:[0-9]+)?/v1/([^/]+)/([^/]+)/([0-9]+)", REG_EXTENDED|REG_ICASE);
	regcomp(&regex_api_auth, "^https?://api\\.[25]ch\\.net(:[0-9]+)?/v1/auth/?$", REG_EXTENDED|REG_ICASE);
	if (talk_to_5ch) {
		regcomp(&regex_talk, "^https?://classic.talk-platform.com/5channel_(([^/]+)/.*)$", REG_EXTENDED|REG_ICASE);
		regcomp(&regex_talk_api, "^https?://api.talk-platform.com/v1/classic/5channel_([^/]+)/([0-9]+)", REG_EXTENDED|REG_ICASE);
		regcomp(&regex_talk_bbscgi, "^https?://api.talk-platform.com/v1/bbs.cgi$", REG_EXTENDED|REG_ICASE);
	}
	compiled = 1;
}