proxy2ch/BBS2chProxyBoardManager.cpp

#include "BBS2chProxyBoardManager.h"
#include <vector>
#include <time.h>
#include <string.h>
#include <stdlib.h>
#include <curl/curl.h>
#include "parson/parson.h"
#include "BBS2chProxyURL.h"
#include "stringEncodingConverter.h"
#ifdef _WIN32
#define localtime_r(a, b) localtime_s(b, a)
#endif

extern char *proxy_server;
extern long proxy_port;
extern long proxy_type;
extern long timeout;
extern char *user_agent;
extern int force_ipv4;
extern CURLSH *curl_share;

extern void log_printf(int level, const char *format ...);

static size_t write_callback_download(char *buffer, size_t size, size_t nitems, void *userdata)
{
    std::vector<char> *data = static_cast<std::vector<char> *>(userdata);
    size_t downloaded = size*nitems;
    data->insert(data->end(), buffer, buffer+downloaded);
    return downloaded;
}

static JSON_Value *getJSONFromURL(const std::string &url)
{
	std::vector<char> data;
	JSON_Value *json = NULL;
	CURL *curl = curl_easy_init();
	if (curl_share) curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
	curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
	curl_easy_setopt(curl, CURLOPT_ENCODING, "");
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
	curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback_download);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
	if (user_agent) {
		curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
	}
	if (proxy_server) {
		curl_easy_setopt(curl, CURLOPT_PROXY, proxy_server);
		curl_easy_setopt(curl, CURLOPT_PROXYPORT, proxy_port);
		curl_easy_setopt(curl, CURLOPT_PROXYTYPE, proxy_type);
	}
	if (force_ipv4) curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
	CURLcode res = curl_easy_perform(curl);
	if (res == CURLE_OK) {
		long statusCode;
		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
		if (statusCode == 200) {
			data.push_back('\0');
			json = json_parse_string(&data.front());
		}
		else log_printf(0, "Cannot download json from %s: server returned %ld\n", url.c_str(), statusCode);
	}
	else log_printf(0, "curl error while downloading %s: %s\n", url.c_str(), curl_easy_strerror(res));
	curl_easy_cleanup(curl);
	return json;
}

static JSON_Value *mixBoardJSON(JSON_Value *base, JSON_Value *additional, std::string suffix, bool talkTo5ch)
{
	JSON_Value *mixed = json_value_init_object();
	JSON_Object *rootBase, *rootAdditional, *rootMixed;
	JSON_Array *categoriesBase, *categoriesAdditional, *categoriesMixed;
	std::map<std::string, size_t> categoriesBaseToIndex;
	time_t modifiedBase = 0, modifiedAdditional = 0;
	if (!base || !additional) goto end;
	if (json_type(base) != JSONObject || json_type(additional) != JSONObject ) {
		goto end;
	}
	rootBase = json_object(base);
	rootAdditional = json_object(additional);
	rootMixed = json_object(mixed);
	json_object_set_value(rootMixed, "menu_list", json_value_init_array());
	categoriesBase = json_object_get_array(rootBase, "menu_list");
	categoriesMixed = json_object_get_array(rootMixed, "menu_list");
	categoriesAdditional = json_object_get_array(rootAdditional, "menu_list");
	if (!categoriesBase || !categoriesAdditional) {
		goto end;
	}
	modifiedBase = json_object_get_number(rootBase, "last_modify");
	modifiedAdditional = json_object_get_number(rootAdditional, "last_modify");
	if (modifiedBase < modifiedAdditional) modifiedBase = modifiedAdditional;
	for (size_t i=0, length=json_array_get_count(categoriesBase); i<length; i++) {
		JSON_Value *value = json_array_get_value(categoriesBase, i);
		if (!value || json_type(value) != JSONObject) continue;
		JSON_Object *category = json_object(value);
		std::string categoryName = json_object_get_string(category, "category_name");
		categoriesBaseToIndex[categoryName] = json_array_get_count(categoriesMixed);
		json_array_append_value(categoriesMixed, json_value_deep_copy(value));
	}
	for (size_t i=0, length=json_array_get_count(categoriesAdditional); i<length; i++) {
		JSON_Value *value = json_array_get_value(categoriesAdditional, i);
		if (!value || json_type(value) != JSONObject) continue;
		JSON_Object *category = json_object(json_value_deep_copy(value));;
		JSON_Array *baseBoards = NULL;
		std::string categoryName = json_object_get_string(category, "category_name");
		std::map<std::string, size_t>::iterator it = categoriesBaseToIndex.find(categoryName);
		size_t categoryNum;
		bool isPink = false;
		if (categoryName == "BBSPINK") isPink = true;
		if (it != categoriesBaseToIndex.end()) {
			JSON_Object *baseCategory = json_array_get_object(categoriesMixed, it->second);
			categoryNum = it->second + 1;
			baseBoards = json_object_get_array(baseCategory, "category_content");
		} else {
			categoryNum = json_array_get_count(categoriesMixed)+1;
			json_object_set_number(category, "category_number", categoryNum);
			json_array_append_value(categoriesMixed, json_object_get_wrapping_value(category));
			if (!isPink) {
				categoryName += suffix;
				json_object_set_string(category, "category_name", categoryName.c_str());
			}
		}
		JSON_Array *boards = json_object_get_array(category, "category_content");
		if (boards) {
			for (size_t j=0, length=json_array_get_count(boards); j<length; j++) {
				JSON_Object *board = json_array_get_object(boards, j);
				if (!board) continue;
				json_object_set_number(board, "category", categoryNum);
				std::string directory = json_object_get_string(board, "directory_name");
				json_object_set_string(board, "category_name", categoryName.c_str());
				if (directory != "NONE") {
					std::string name = json_object_get_string(board, "board_name");
					if (!isPink) {
						name += suffix;
						json_object_set_string(board, "board_name", name.c_str());
					}
					if (talkTo5ch) {
						std::string modDirectory = "5channel_";
						modDirectory += directory;
						std::string url = "https://classic.talk-platform.com/";
						url += modDirectory;
						json_object_set_string(board, "directory_name", modDirectory.c_str());
						json_object_set_string(board, "url", url.c_str());
					}
				}
				if (baseBoards) {
					json_object_set_number(board, "category_order", json_array_get_count(baseBoards)+1);
					json_array_append_value(baseBoards, json_value_deep_copy(json_object_get_wrapping_value(board)));
				}
			}
		}
		if (baseBoards) {
			JSON_Object *baseCategory = json_array_get_object(categoriesMixed, it->second);
			json_object_set_number(baseCategory, "category_total", json_array_get_count(baseBoards));
			json_value_free(json_object_get_wrapping_value(category));
		}
	}
end:
	struct tm lastModified_tm = {};
	char dateStr[256] = "";
	localtime_r(&modifiedBase, &lastModified_tm);
	strftime(dateStr, 256, "%Y/%m/%d(%a) %H:%M:%S", &lastModified_tm);
	json_object_set_string(rootMixed, "last_modify_string", dateStr);
	json_object_set_number(rootMixed, "last_modify", modifiedBase);
	json_object_set_string(rootMixed, "description", "");
	return mixed;
}

static JSON_Value *getBoardJSONFor5ch(bool shouldUseHttp, bool shouldUse2ch, bool shouldIncludeTalk)
{
	JSON_Value *json5ch = getJSONFromURL("http://menu.5ch.net/bbsmenu.json");
	JSON_Value *jsonTalk = NULL;
	JSON_Value *jsonMixed = NULL;
	JSON_Object *root;
	JSON_Array *categories;
	if (!json5ch || json_type(json5ch) != JSONObject) goto end;
	if (shouldIncludeTalk) {
		jsonTalk = getJSONFromURL("http://classic.talk-platform.com/bbsmenu.json");
		jsonMixed = mixBoardJSON(json5ch, jsonTalk, " (Talk)", false);
		if (!jsonMixed || json_type(jsonMixed) != JSONObject) goto end;
	}
	root = shouldIncludeTalk ? json_object(jsonMixed) : json_object(json5ch);
	categories = json_object_get_array(root, "menu_list");
	if (!categories) goto end;
	for (size_t i=0, length=json_array_get_count(categories); i<length; i++) {
		JSON_Object *category = json_array_get_object(categories, i);
		if (!category) continue;
		const char *categoryName = json_object_get_string(category, "category_name");
		if (!categoryName) continue;
		JSON_Array *boards = json_object_get_array(category, "category_content");
		if (boards) {
			for (size_t j=0, length=json_array_get_count(boards); j<length; j++) {
				JSON_Object *board = json_array_get_object(boards, j);
				if (!board) continue;
				const char *url = json_object_get_string(board, "url");
				if (!url) continue;
				if (shouldUseHttp || shouldUse2ch) {
					BBS2chProxyURL modUrl(url);
					if (shouldUseHttp) modUrl.setScheme("http");
					if (shouldUse2ch) modUrl.replaceHost("5ch.net", "2ch.net");
					json_object_set_string(board, "url", modUrl.absoluteString().c_str());
				}
			}
		}
	}

end:
	if (jsonMixed && json5ch) json_value_free(json5ch);
	if (jsonTalk) json_value_free(jsonTalk);
	return jsonMixed ? jsonMixed : json5ch;
}

void BBS2chProxyBoardManager::updateMap() {
	JSON_Value *json = getJSONFromURL("http://menu.5ch.net/bbsmenu.json");
	JSON_Object *root;
	JSON_Array *categories;
	if (!json || json_type(json) != JSONObject) goto end;
	root = json_object(json);
	categories = json_object_get_array(root, "menu_list");
	if (!categories) {
		goto end;
	}
	for (size_t i=0, length=json_array_get_count(categories); i<length; i++) {
		JSON_Object *category = json_array_get_object(categories, i);
		if (!category) continue;
		JSON_Array *boards = json_object_get_array(category, "category_content");
		if (!boards) continue;
		for (size_t j=0, length=json_array_get_count(boards); j<length; j++) {
			JSON_Object *board = json_array_get_object(boards, j);
			if (!board) continue;
			const char *name = json_object_get_string(board, "directory_name");
			const char *url = json_object_get_string(board, "url");
			if (name && url && strcmp(name, "NONE")) {
				if (_boardToServer.find(name) != _boardToServer.end()) {
					continue;
				}
				_boardToServer[name] = BBS2chProxyURL(url).getHost();
			}
		}
	}
	_lastUpdated = time(NULL);
end:
	if (json) json_value_free(json);
}

std::string BBS2chProxyBoardManager::getBoardJSONForTalkAndFake5ch()
{
	JSON_Value *jsonTalk = getJSONFromURL("http://classic.talk-platform.com/bbsmenu.json");
	JSON_Value *json5ch = getJSONFromURL("http://menu.5ch.net/bbsmenu.json");
	JSON_Value *jsonMixed = NULL;
	JSON_Object *rootMixed;
	time_t now = time(NULL);
	struct tm now_tm = {};
	char nowStr[256];
	localtime_r(&now, &now_tm);
	strftime(nowStr, 256, "%Y/%m/%d(%a) %H:%M:%S", &now_tm);
	if (!jsonTalk || !json5ch) goto end;
	if (json_type(jsonTalk) != JSONObject || json_type(json5ch) != JSONObject ) {
		goto end;
	}
	jsonMixed = mixBoardJSON(jsonTalk, json5ch, " (5ch)", true);
	rootMixed = json_object(jsonMixed);
	json_object_set_string(rootMixed, "last_modify_string", nowStr);
	json_object_set_number(rootMixed, "last_modify", now);
	json_object_set_string(rootMixed, "description", "");
end:
	std::string out;
	if (jsonMixed) {
		char *jsonStr = json_serialize_to_string(jsonMixed);
		out = jsonStr;
		json_free_serialized_string(jsonStr);
		json_value_free(jsonMixed);
	}
	if (jsonTalk) json_value_free(jsonTalk);
	if (json5ch) json_value_free(json5ch);
	return out;
}

std::string BBS2chProxyBoardManager::getBoardHTML(bool shouldUseHttp, bool shouldUse2ch, bool shouldIncludeTalk)
{
	std::string outHtml = "<HTML>\n<HEAD>\n<META http-equiv=\"Content-Type\" content=\"text/html; charset=Shift_JIS\">\n<TITLE>BBSMENU for 5ch.net</TITLE>\n<BASE TARGET=\"_right\">\n</HEAD>\n<BODY TEXT=\"#CC3300\" BGCOLOR=\"#FFFFFF\" link=\"#0000FF\" alink=\"#ff0000\" vlink=\"#660099\">\n<font size=2>\n\n";
	JSON_Value *json = getBoardJSONFor5ch(shouldUseHttp, shouldUse2ch, shouldIncludeTalk);
	JSON_Object *root;
	JSON_Array *categories;
	time_t lastModified;
	struct tm lastModified_tm = {};
	char dateStr[256] = "";
	if (!json || json_type(json) != JSONObject) goto end;
	root = json_object(json);
	categories = json_object_get_array(root, "menu_list");
	if (!categories) goto end;
	lastModified = json_object_get_number(root, "last_modify");
	if (lastModified) {
		localtime_r(&lastModified, &lastModified_tm);
		strftime(dateStr, 256, "%Y/%m/%d", &lastModified_tm);
	}
	for (size_t i=0, length=json_array_get_count(categories); i<length; i++) {
		JSON_Object *category = json_array_get_object(categories, i);
		if (!category) continue;
		const char *categoryName = json_object_get_string(category, "category_name");
		if (!categoryName) continue;
		char *categoryNameSJIS = convertUTF8ToShiftJIS(categoryName, strlen(categoryName));
		outHtml += "<BR><BR><B>";
		outHtml += categoryNameSJIS;
		outHtml += "</B><BR>\n";
		free(categoryNameSJIS);
		JSON_Array *boards = json_object_get_array(category, "category_content");
		if (boards) {
			bool added = false;
			for (size_t j=0, length=json_array_get_count(boards); j<length; j++) {
				JSON_Object *board = json_array_get_object(boards, j);
				if (!board) continue;
				const char *boardName = json_object_get_string(board, "board_name");
				const char *url = json_object_get_string(board, "url");
				if (!boardName || !url) continue;
				char *boardNameSJIS = convertUTF8ToShiftJIS(boardName, strlen(boardName));
				if (!boardNameSJIS) continue;
				if (added) outHtml += "<br>\n";
				outHtml += "<A HREF=";
				outHtml += url;
				if (outHtml[outHtml.size()-1] != '/') outHtml += '/';
				outHtml += ">";
				outHtml += boardNameSJIS;
				outHtml += "</A>";
				added = true;
				free(boardNameSJIS);
			}
			if (added) outHtml += "\n\n";
		}
	}

end:
	if (json) json_value_free(json);
	outHtml += "<BR><BR>\n\x8d\x58\x90\x56\x93\xfa ";
	outHtml += dateStr;
	outHtml += "\n\n</font></BODY></HTML>";
	return outHtml;
}

std::string BBS2chProxyBoardManager::getBoardJSON(bool shouldUseHttp, bool shouldUse2ch, bool shouldIncludeTalk)
{
	std::string out;
	JSON_Value *json = getBoardJSONFor5ch(shouldUseHttp, shouldUse2ch, shouldIncludeTalk);
	if (json) {
		char *jsonStr = json_serialize_to_string(json);
		out = jsonStr;
		json_free_serialized_string(jsonStr);
		json_value_free(json);
	}
	return out;
}

std::string BBS2chProxyBoardManager::getServerForBoard(const std::string &name)
{
	std::string url;
	pthread_mutex_lock(&_mutex);
	if (_lastUpdated < time(NULL) - 86400) updateMap();
	std::map<std::string, std::string>::iterator it = _boardToServer.find(name);
	if (it != _boardToServer.end()) url = it->second;
	pthread_mutex_unlock(&_mutex);
	return url;
}