DataHelper.h 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. #ifndef DATAHELPER_H
  2. #define DATAHELPER_H
  3. #include "QAuthenticator"
  4. #include "QWebEngineView"
  5. #include "QWebEngineProfile"
  6. #include "QWebEngineSettings"
  7. #include "QWebEngineCookieStore"
  8. #include "QNetworkProxy"
  9. #include "QEventLoop"
  10. #include "QDir"
  11. #include "QByteArray"
  12. #include "QMessageBox"
  13. static QDir cacheDir ("cache");
  14. static QDir cookieDir("cookie");
  15. const static QString hCAPTCHA_source = "<form action='' method='POST'>\
  16. <script src='https://hcaptcha.com/1/api.js'></script>\
  17. <div class='h-captcha' data-sitekey='%1'></div>\
  18. <input type='submit' value='Submit' /></form>";
  19. const static QString reCAPTCHA2_source = "<form action='https://google.com' method='GET'>\
  20. <script src='https://www.google.com/recaptcha/api.js'></script>\
  21. <div class='g-recaptcha' data-sitekey='%1'></div>\
  22. <input id='1' type='submit' name='button' value='Enter' ></form>";
  23. const static QString reCAPTCHA3_source1 = "var script=document.createElement('script');script.type='text/javascript';script.src='https://www.google.com/recaptcha/api.js?render=%1';document.head.appendChild(script);";
  24. const static QString reCAPTCHA3_source2 = "var ResponseV3=0;grecaptcha.ready(function(){grecaptcha.execute('%1', {action: '%2'}).then(function(token){ResponseV3=token;});});";
  25. static int newTabId = 0;
  26. enum CAPTCHA{ none, reCap2, reCap3, hCap };
  27. enum PROXYSTATE{ NOT_SET, ON, OFF };
  28. class JS_Capture : public QWebEnginePage{
  29. protected:
  30. virtual void javaScriptConsoleMessage(QWebEnginePage::JavaScriptConsoleMessageLevel level, const QString &message, int lineNumber, const QString &sourceID) override {
  31. if(onJS_error_alert && level == ErrorMessageLevel){
  32. QMessageBox::information(nullptr, "Ошибка в JS", message + " : line " + QString::number(lineNumber));
  33. }
  34. if(captureJS_errors){
  35. if(logAll || level == logLevel){
  36. JS_errors += message + " : line " + QString::number(lineNumber);
  37. }
  38. }
  39. }
  40. public:
  41. bool captureJS_errors = true;
  42. bool logAll = false;
  43. bool onJS_error_alert = false;
  44. QWebEnginePage::JavaScriptConsoleMessageLevel logLevel = ErrorMessageLevel;
  45. QString JS_errors = "";
  46. };
  47. class Proxy{
  48. public:
  49. inline static QNetworkProxy::ProxyType Type = QNetworkProxy::DefaultProxy;
  50. inline static QString Adress;
  51. inline static uint16_t Port;
  52. inline static QString User;
  53. inline static QString Pass;
  54. inline static bool isProxyAuth = false;
  55. static bool set(QStringList *TaskList){
  56. if(TaskList->at(1) == "none"){//убрать прокси
  57. Adress = "";
  58. QNetworkProxy::setApplicationProxy(QNetworkProxy::DefaultProxy);
  59. }
  60. else{
  61. QList<QString> listData = TaskList->at(1).split(":");
  62. if(listData.size() < 3) return false;
  63. if(listData.at(0) == "http" || listData.at(0) == "https")
  64. Type = QNetworkProxy::HttpProxy;
  65. else if(listData.at(0) == "socks5")
  66. Type = QNetworkProxy::Socks5Proxy;
  67. else return false;
  68. Adress = listData.at(1);
  69. Port = listData.at(2).toUInt();
  70. if(listData.size() == 3){//http:1.2.3.4:80
  71. isProxyAuth = false;
  72. QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port));
  73. return true;
  74. }
  75. else if(listData.size() == 5){//http:1.2.3.4:80:user:pass
  76. User = listData.at(3);
  77. Pass = listData.at(4);
  78. isProxyAuth = true;
  79. QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port, User, Pass));
  80. return true;
  81. }
  82. return false;
  83. }
  84. return true;
  85. }
  86. static bool on_off(PROXYSTATE state){
  87. if(state == PROXYSTATE::ON){
  88. if(Adress.length() > 0){
  89. if(isProxyAuth)
  90. QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port, User, Pass));
  91. else
  92. QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port));
  93. }
  94. else return false;
  95. }
  96. else QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy);
  97. return true;
  98. }
  99. };
  100. class Tab : public QWebEngineView{
  101. static inline QString _scroll_y_max = "Math.max(document.body.scrollHeight, document.documentElement.scrollHeight, document.body.offsetHeight, document.documentElement.offsetHeight, document.body.clientHeight, document.documentElement.clientHeight);";
  102. static inline QString _scroll_y = "window.scrollBy(0, %1);";
  103. static inline QString _window_scroll_to = "window.scrollTo(%1, %2);";
  104. static inline QString _scroll_x_max = "Math.max(document.body.scrollWidth, document.documentElement.scrollWidth, document.body.offsetWidth, document.documentElement.offsetWidth, document.body.clientWidth, document.documentElement.clientWidth);";
  105. static inline QString _scroll_x = "window.scrollBy(%1, 0);";
  106. public:
  107. Tab(){
  108. this->id = newTabId++;
  109. setPage(new JS_Capture);
  110. settings()->setAttribute(QWebEngineSettings::LocalContentCanAccessRemoteUrls, true);
  111. settings()->setAttribute(QWebEngineSettings::AllowRunningInsecureContent, true);
  112. settings()->setUnknownUrlSchemePolicy(QWebEngineSettings::AllowAllUnknownUrlSchemes);
  113. settings()->setAttribute(QWebEngineSettings::ScreenCaptureEnabled, true);
  114. page()->profile()->setHttpUserAgent( "Mozilla/5.0 (X11; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0");
  115. page()->profile()->setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3");
  116. page()->profile()->setCachePath(cacheDir .path());
  117. //вызывает постоянную нагрузку
  118. //page()->profile()->setPersistentStoragePath(cookieDir.path());
  119. connect(this, &QWebEngineView::loadStarted, this, [&](){
  120. isLoaded = false;
  121. });
  122. connect(this, &QWebEngineView::loadFinished, this, [&](){
  123. isLoaded = true;
  124. });
  125. cookiesV = new QVector<QNetworkCookie>;
  126. connect(page()->profile()->cookieStore(), &QWebEngineCookieStore::cookieAdded, this, [&](const QNetworkCookie &cookie){
  127. addCookie(cookie);
  128. });
  129. connect(page(), &QWebEnginePage::proxyAuthenticationRequired, this, &Tab::proxyAuthenticationRequired);
  130. }
  131. int id;
  132. bool isLoaded;
  133. QVector<QNetworkCookie>* cookies(){
  134. return cookiesV;
  135. }
  136. void deleteCookies(){
  137. page()->profile()->cookieStore()->deleteAllCookies();
  138. cookiesV->clear();
  139. }
  140. QByteArray runJavaScript(QString script, bool getResponse = false){
  141. QByteArray JSresult;
  142. QEventLoop loop;
  143. page()->runJavaScript(script, [&JSresult, &loop] (const QVariant &result){
  144. JSresult = result.toByteArray();
  145. loop.exit(0);
  146. });
  147. loop.exec();
  148. if(getResponse) return JSresult;
  149. return JSresult.isEmpty() ? "empty result" : JSresult;
  150. }
  151. QString JS_Errors(){
  152. return static_cast<JS_Capture*>(page())->JS_errors;
  153. }
  154. void set_JS_capture(bool state){
  155. static_cast<JS_Capture*>(page())->captureJS_errors = state;
  156. }
  157. void set_JS_capture_level(int lvl){
  158. static_cast<JS_Capture*>(page())->logAll = false;
  159. switch (lvl){
  160. case 0: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::InfoMessageLevel; break;
  161. case 1: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::WarningMessageLevel; break;
  162. case 2: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::ErrorMessageLevel; break;
  163. case 3: static_cast<JS_Capture*>(page())->logAll = true; break;
  164. }
  165. }
  166. void set_JS_on_error_alert(bool state){
  167. static_cast<JS_Capture*>(page())->onJS_error_alert = state;
  168. }
  169. void clearJS_errors(){
  170. static_cast<JS_Capture*>(page())->JS_errors.clear();
  171. }
  172. void saveCookies(QString domain, QString fileName, bool isAppend, bool isDefaultPath){
  173. QByteArray data = "";//Куки в формате miniPoster'а
  174. auto addCookie = [&data](const QNetworkCookie &cook){
  175. data += (cook.domain().at(0) == '.'
  176. ? cook.domain().remove(0, 1).toUtf8() : cook.domain().toUtf8())
  177. + "|" + cook.name() + "|" + cook.value() + "\n";
  178. };
  179. if(domain.length() == 0)//Домен не указан => выгружаем все куки
  180. for(auto c : *cookies()) addCookie(c);
  181. else for(auto c : *cookies())
  182. if(c.domain().contains(domain))//Сохраняем куки только для указанного домена
  183. addCookie(c);
  184. if(data.length() > 0) data.remove(data.length() - 1, 1);//удаляем '\n' в конце
  185. QFile qf((isDefaultPath ? ("../save/cookies/" + fileName) : fileName) + ".cook");
  186. qf.open(isAppend ? QIODevice::Append : QIODevice::WriteOnly);
  187. qf.write(data); qf.close();
  188. }
  189. int set_cookies(QStringList &TaskList, bool fromFile = true){
  190. int added = 0;
  191. auto addCookie = [&](const QByteArray *host, const QByteArray *key, QByteArray value = ""){
  192. if(host->length() == 0 || key->length() == 0)
  193. return true;
  194. QNetworkCookie cookie(*key, value);
  195. ++added;
  196. page()->profile()->cookieStore()->setCookie(cookie, QUrl("https://" + *host));
  197. return false;
  198. };
  199. if(fromFile){
  200. QFile qf(TaskList.at(1));
  201. if(!qf.exists()) return -1;
  202. QByteArrayList split = qf.readAll().split('\n');//Куки из файла
  203. if(split.count() == 0) return -1;
  204. for(auto str : split){
  205. QByteArrayList c = str.split('|');
  206. bool bad = true;
  207. if(c.count() == 2){//Добавляем пустое значение
  208. bad = addCookie(&c.at(0), &c.at(1));
  209. }
  210. else if(c.count() == 3){//Нормальная строка
  211. bad = addCookie(&c.at(0), &c.at(1), c.at(2));
  212. }
  213. if(bad) return -1;//Странная строка
  214. }
  215. }
  216. else{
  217. QByteArrayList split = TaskList.at(1).toLocal8Bit().split('|');//Куки из строки
  218. if(split.count() == 0) return -1;
  219. for(int i = 0; i < split.size(); i += 3){
  220. bool bad = true;
  221. if(i + 2 < split.size()){//Нормальная строка
  222. bad = addCookie(&split.at(i), &split.at(i + 1), split.at(i + 2));
  223. }
  224. else if(i + 1 < split.size()){//Добавляем пустое значение
  225. bad = addCookie(&split.at(i), &split.at(i + 1));
  226. }
  227. if(bad) return -1;//Странная строка
  228. }
  229. }
  230. return added;
  231. }
  232. void get_html(QByteArray &data_to_server){
  233. QString res = "";
  234. QEventLoop loop;
  235. page()->toHtml([&res,&loop](const QString &result){
  236. res = result;
  237. loop.exit(0);
  238. });
  239. loop.exec();
  240. data_to_server = res.toLocal8Bit();
  241. }
  242. void get_cookies(QByteArray &data_to_server){
  243. data_to_server = "";
  244. for(auto c : *cookies())//Преобразуем куки в формат miniPoster'а
  245. data_to_server += (c.domain() + "|" + c.name() + "|" + c.value() + "|").toLocal8Bit();
  246. if(data_to_server.length() > 0) data_to_server.remove(data_to_server.length() - 1, 1);//удаляем '|' в конце
  247. }
  248. void get_cookie_value_by_name(QByteArray &data_to_server, QStringList &TaskList){
  249. data_to_server = "";
  250. for(auto c : *cookies())
  251. if(c.name() == TaskList.at(1)){
  252. data_to_server = c.value();
  253. break;
  254. }
  255. }
  256. void get_cookie_by_domain(QByteArray &data_to_server, QStringList &TaskList){
  257. data_to_server = "";
  258. for(auto c : *cookies())
  259. if(c.domain().contains(TaskList[1]))
  260. data_to_server += c.name() + "|" + c.value() + "|";
  261. if(data_to_server.length() > 0) data_to_server.remove(data_to_server.length() - 1, 1);//удаляем '|' в конце
  262. }
  263. void get_domain_cookie_value_by_name(QByteArray &data_to_server, QStringList &TaskList){
  264. data_to_server = "";
  265. for(auto c : *cookies())
  266. if(c.domain() == TaskList.at(1) && c.name() == TaskList.at(2)){
  267. data_to_server = c.value();
  268. break;
  269. }
  270. }
  271. void screen_shot(QByteArray &data_to_server, QStringList &TaskList){
  272. QList<QString> listData = TaskList.at(2).split(":");
  273. listData.append(TaskList.at(3).split(":"));
  274. if(listData.size() < 4) return;
  275. QDir *dir = new QDir();
  276. static const QString path = "../captcha/";
  277. if(!dir->exists(path)) dir->mkdir(path);
  278. delete dir;
  279. int x = listData.at(0).toInt(), y = listData.at(1).toInt();
  280. QPixmap img = grab(QRect(QPoint(x,y), QSize(listData.at(2).toInt() - x, listData.at(3).toInt() - y)));
  281. img.save(path + TaskList.at(1) + ".png");
  282. data_to_server = "ok";
  283. }
  284. void pixel_exists(QByteArray &data_to_server, QStringList &TaskList){
  285. QList<QString> listData = TaskList.at(1).split(":");
  286. listData.append(TaskList.at(2).split(":"));
  287. data_to_server = "no";
  288. if(listData.size() < 4) return;
  289. int x = listData.at(0).toInt(), y = listData.at(1).toInt();
  290. int x1 = listData.at(2).toInt(), y1 = listData.at(3).toInt();
  291. QImage img = grab(QRect(QPoint(x,y), QSize(x1 - x, y1 - y))).toImage();
  292. QColor pixel(TaskList.at(3));
  293. bool find = false;
  294. for(int y = 0; y < img.height() && !find; ++y)
  295. for(int x = 0; x < img.width() && !find; ++x)
  296. if(img.pixel(x, y) == pixel.rgb())
  297. { data_to_server = "yes"; find = true; }
  298. if(!find)
  299. data_to_server = "no";
  300. }
  301. //Scroll
  302. QByteArray scroll_y_position(){
  303. return QString::number(page()->scrollPosition().ry()).toLocal8Bit();
  304. }
  305. QByteArray scroll_y_max(){
  306. return runJavaScript(_scroll_y_max);
  307. }
  308. QByteArray scroll_y(QString arg){
  309. return runJavaScript(_scroll_y.arg(arg));
  310. }
  311. QByteArray scroll_to_y(QString arg){
  312. return runJavaScript(_window_scroll_to.arg(page()->scrollPosition().rx()).arg(arg));
  313. }
  314. QByteArray scroll_x_position(){
  315. return QString::number(page()->scrollPosition().rx()).toLocal8Bit();
  316. }
  317. QByteArray scroll_x_max(){
  318. return runJavaScript(_scroll_x_max);
  319. }
  320. QByteArray scroll_x(QString arg){
  321. return runJavaScript(_scroll_x.arg(arg));
  322. }
  323. QByteArray scroll_to_x(QString arg){
  324. return runJavaScript(_window_scroll_to.arg(arg).arg(page()->scrollPosition().ry()));
  325. }
  326. //Element
  327. void element_set_value_by_id(QString id, QString value){
  328. runJavaScript(QString("document.querySelector('#%1').value='%2')").arg(id).arg(value));
  329. }
  330. void element_set_value_by_class(QString class_name, QString value){
  331. runJavaScript(QString("document.querySelector('.%1').value='%2')").arg(class_name).arg(value));
  332. }
  333. void element_set_inner_html_by_id(QString id, QString inner_html){
  334. runJavaScript(QString("document.querySelector('#%1').innerHTML='%2')").arg(id).arg(inner_html));
  335. }
  336. void element_set_inner_html_by_class(QString class_name, QString inner_html){
  337. runJavaScript(QString("document.querySelector('.%1').innerHTML='%2')").arg(class_name).arg(inner_html));
  338. }
  339. void element_click_by_id(QString id){
  340. runJavaScript(QString("document.querySelector('#%1').click()").arg(id));
  341. }
  342. void element_clock_by_class(QString class_name){
  343. runJavaScript(QString("document.querySelector('.%1').click()").arg(class_name));
  344. }
  345. //CAPTCHA
  346. bool getCaptchaResponse(QByteArray &data_to_server){
  347. switch (currCAPTCHA) {
  348. case reCap2: { data_to_server = runJavaScript("grecaptcha.getResponse()", true); break; }
  349. case reCap3: { data_to_server = runJavaScript("ResponseV3", true); break; }
  350. case hCap: { data_to_server = runJavaScript("hcaptcha.getResponse()", true); break; }
  351. default: return false;
  352. }
  353. currCAPTCHA = CAPTCHA::none;
  354. return true;
  355. }
  356. void load_v2reCAPTHCA(QString *baseURL, QString *siteKey){
  357. isLoaded = true;
  358. currCAPTCHA = CAPTCHA::reCap2;
  359. //загрузка reCAPTCHA2
  360. setHtml(reCAPTCHA2_source.arg(*siteKey).toLocal8Bit(), QUrl(*baseURL));
  361. }
  362. void load_v3reCAPTHCA(QStringList &TaskList){
  363. QString baseURL = TaskList[1];
  364. QString siteKey = TaskList[2];
  365. QString action = TaskList.size() == 4 ? TaskList[3] : "";
  366. load_v2reCAPTHCA(&baseURL, &siteKey);
  367. currCAPTCHA = CAPTCHA::reCap3;
  368. //загрузка reCAPTCHA3
  369. runJavaScript(reCAPTCHA3_source1.arg(siteKey));
  370. runJavaScript(reCAPTCHA3_source2.arg(siteKey).arg(action.length() == 0 ? "homepage" : action));
  371. }
  372. void load_hcaptcha(QStringList &TaskList){
  373. QString baseURL = TaskList[1];
  374. QString siteKey = TaskList[2];
  375. currCAPTCHA = CAPTCHA::hCap;
  376. //загрузка hCaptcha
  377. setHtml(hCAPTCHA_source.arg(siteKey), QUrl(baseURL));
  378. isLoaded = true;
  379. }
  380. CAPTCHA currCAPTCHA = CAPTCHA::none;
  381. private:
  382. QVector<QNetworkCookie> *cookiesV;
  383. void addCookie(const QNetworkCookie cookie){
  384. for(auto item : *cookiesV)
  385. if(item.hasSameIdentifier(cookie))
  386. return;//добавляем только новые куки
  387. cookiesV->append(cookie);
  388. }
  389. void proxyAuthenticationRequired(const QUrl &, QAuthenticator *auth, const QString &){
  390. auth->setUser (Proxy::User);
  391. auth->setPassword(Proxy::Pass);
  392. }
  393. };
  394. #endif // DATAHELPER_H