123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473 |
- #ifndef DATAHELPER_H
- #define DATAHELPER_H
- #include "QAuthenticator"
- #include "QWebEngineView"
- #include "QWebEngineProfile"
- #include "QWebEngineSettings"
- #include "QWebEngineCookieStore"
- #include "QNetworkProxy"
- #include "QEventLoop"
- #include "QDir"
- #include "QByteArray"
- #include "QMessageBox"
- static QDir cacheDir ("cache");
- static QDir cookieDir("cookie");
- const static QString hCAPTCHA_source = "<form action='' method='POST'>\
- <script src='https://hcaptcha.com/1/api.js'></script>\
- <div class='h-captcha' data-sitekey='%1'></div>\
- <input type='submit' value='Submit' /></form>";
- const static QString reCAPTCHA2_source = "<form action='https://google.com' method='GET'>\
- <script src='https://www.google.com/recaptcha/api.js'></script>\
- <div class='g-recaptcha' data-sitekey='%1'></div>\
- <input id='1' type='submit' name='button' value='Enter' ></form>";
- const static QString reCAPTCHA3_source1 = "var script=document.createElement('script');script.type='text/javascript';script.src='https://www.google.com/recaptcha/api.js?render=%1';document.head.appendChild(script);";
- const static QString reCAPTCHA3_source2 = "var ResponseV3=0;grecaptcha.ready(function(){grecaptcha.execute('%1', {action: '%2'}).then(function(token){ResponseV3=token;});});";
- static int newTabId = 0;
- enum CAPTCHA{ none, reCap2, reCap3, hCap };
- enum PROXYSTATE{ NOT_SET, ON, OFF };
- class JS_Capture : public QWebEnginePage{
- protected:
- virtual void javaScriptConsoleMessage(QWebEnginePage::JavaScriptConsoleMessageLevel level, const QString &message, int lineNumber, const QString &sourceID) override {
- if(onJS_error_alert && level == ErrorMessageLevel){
- QMessageBox::information(nullptr, "Ошибка в JS", message + " : line " + QString::number(lineNumber));
- }
- if(captureJS_errors){
- if(logAll || level == logLevel){
- JS_errors += message + " : line " + QString::number(lineNumber);
- }
- }
- }
- public:
- bool captureJS_errors = true;
- bool logAll = false;
- bool onJS_error_alert = false;
- QWebEnginePage::JavaScriptConsoleMessageLevel logLevel = ErrorMessageLevel;
- QString JS_errors = "";
- };
- class Proxy{
- public:
- inline static QNetworkProxy::ProxyType Type = QNetworkProxy::DefaultProxy;
- inline static QString Adress;
- inline static uint16_t Port;
- inline static QString User;
- inline static QString Pass;
- inline static bool isProxyAuth = false;
- static bool set(QStringList *TaskList){
- if(TaskList->at(1) == "none"){//убрать прокси
- Adress = "";
- QNetworkProxy::setApplicationProxy(QNetworkProxy::DefaultProxy);
- }
- else{
- QList<QString> listData = TaskList->at(1).split(":");
- if(listData.size() < 3) return false;
- if(listData.at(0) == "http" || listData.at(0) == "https")
- Type = QNetworkProxy::HttpProxy;
- else if(listData.at(0) == "socks5")
- Type = QNetworkProxy::Socks5Proxy;
- else return false;
- Adress = listData.at(1);
- Port = listData.at(2).toUInt();
- if(listData.size() == 3){//http:1.2.3.4:80
- isProxyAuth = false;
- QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port));
- return true;
- }
- else if(listData.size() == 5){//http:1.2.3.4:80:user:pass
- User = listData.at(3);
- Pass = listData.at(4);
- isProxyAuth = true;
- QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port, User, Pass));
- return true;
- }
- return false;
- }
- return true;
- }
- static bool on_off(PROXYSTATE state){
- if(state == PROXYSTATE::ON){
- if(Adress.length() > 0){
- if(isProxyAuth)
- QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port, User, Pass));
- else
- QNetworkProxy::setApplicationProxy(QNetworkProxy(Type, Adress, Port));
- }
- else return false;
- }
- else QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy);
- return true;
- }
- };
- class Tab : public QWebEngineView{
- static inline QString _scroll_y_max = "Math.max(document.body.scrollHeight, document.documentElement.scrollHeight, document.body.offsetHeight, document.documentElement.offsetHeight, document.body.clientHeight, document.documentElement.clientHeight);";
- static inline QString _scroll_y = "window.scrollBy(0, %1);";
- static inline QString _window_scroll_to = "window.scrollTo(%1, %2);";
- static inline QString _scroll_x_max = "Math.max(document.body.scrollWidth, document.documentElement.scrollWidth, document.body.offsetWidth, document.documentElement.offsetWidth, document.body.clientWidth, document.documentElement.clientWidth);";
- static inline QString _scroll_x = "window.scrollBy(%1, 0);";
- public:
- Tab(){
- this->id = newTabId++;
- setPage(new JS_Capture);
- settings()->setAttribute(QWebEngineSettings::LocalContentCanAccessRemoteUrls, true);
- settings()->setAttribute(QWebEngineSettings::AllowRunningInsecureContent, true);
- settings()->setUnknownUrlSchemePolicy(QWebEngineSettings::AllowAllUnknownUrlSchemes);
- settings()->setAttribute(QWebEngineSettings::ScreenCaptureEnabled, true);
- page()->profile()->setHttpUserAgent( "Mozilla/5.0 (X11; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0");
- page()->profile()->setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3");
- page()->profile()->setCachePath(cacheDir .path());
- //вызывает постоянную нагрузку
- //page()->profile()->setPersistentStoragePath(cookieDir.path());
- connect(this, &QWebEngineView::loadStarted, this, [&](){
- isLoaded = false;
- });
- connect(this, &QWebEngineView::loadFinished, this, [&](){
- isLoaded = true;
- });
- cookiesV = new QVector<QNetworkCookie>;
- connect(page()->profile()->cookieStore(), &QWebEngineCookieStore::cookieAdded, this, [&](const QNetworkCookie &cookie){
- addCookie(cookie);
- });
- connect(page(), &QWebEnginePage::proxyAuthenticationRequired, this, &Tab::proxyAuthenticationRequired);
- }
- int id;
- bool isLoaded;
- QVector<QNetworkCookie>* cookies(){
- return cookiesV;
- }
- void deleteCookies(){
- page()->profile()->cookieStore()->deleteAllCookies();
- cookiesV->clear();
- }
- QByteArray runJavaScript(QString script, bool getResponse = false){
- QByteArray JSresult;
- QEventLoop loop;
- page()->runJavaScript(script, [&JSresult, &loop] (const QVariant &result){
- JSresult = result.toByteArray();
- loop.exit(0);
- });
- loop.exec();
- if(getResponse) return JSresult;
- return JSresult.isEmpty() ? "empty result" : JSresult;
- }
- QString JS_Errors(){
- return static_cast<JS_Capture*>(page())->JS_errors;
- }
- void set_JS_capture(bool state){
- static_cast<JS_Capture*>(page())->captureJS_errors = state;
- }
- void set_JS_capture_level(int lvl){
- static_cast<JS_Capture*>(page())->logAll = false;
- switch (lvl){
- case 0: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::InfoMessageLevel; break;
- case 1: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::WarningMessageLevel; break;
- case 2: static_cast<JS_Capture*>(page())->logLevel = QWebEnginePage::ErrorMessageLevel; break;
- case 3: static_cast<JS_Capture*>(page())->logAll = true; break;
- }
- }
- void set_JS_on_error_alert(bool state){
- static_cast<JS_Capture*>(page())->onJS_error_alert = state;
- }
- void clearJS_errors(){
- static_cast<JS_Capture*>(page())->JS_errors.clear();
- }
- void saveCookies(QString domain, QString fileName, bool isAppend, bool isDefaultPath){
- QByteArray data = "";//Куки в формате miniPoster'а
- auto addCookie = [&data](const QNetworkCookie &cook){
- data += (cook.domain().at(0) == '.'
- ? cook.domain().remove(0, 1).toUtf8() : cook.domain().toUtf8())
- + "|" + cook.name() + "|" + cook.value() + "\n";
- };
- if(domain.length() == 0)//Домен не указан => выгружаем все куки
- for(auto c : *cookies()) addCookie(c);
- else for(auto c : *cookies())
- if(c.domain().contains(domain))//Сохраняем куки только для указанного домена
- addCookie(c);
- if(data.length() > 0) data.remove(data.length() - 1, 1);//удаляем '\n' в конце
- QFile qf((isDefaultPath ? ("../save/cookies/" + fileName) : fileName) + ".cook");
- qf.open(isAppend ? QIODevice::Append : QIODevice::WriteOnly);
- qf.write(data); qf.close();
- }
- int set_cookies(QStringList &TaskList, bool fromFile = true){
- int added = 0;
- auto addCookie = [&](const QByteArray *host, const QByteArray *key, QByteArray value = ""){
- if(host->length() == 0 || key->length() == 0)
- return true;
- QNetworkCookie cookie(*key, value);
- ++added;
- page()->profile()->cookieStore()->setCookie(cookie, QUrl("https://" + *host));
- return false;
- };
- if(fromFile){
- QFile qf(TaskList.at(1));
- if(!qf.exists()) return -1;
- QByteArrayList split = qf.readAll().split('\n');//Куки из файла
- if(split.count() == 0) return -1;
- for(auto str : split){
- QByteArrayList c = str.split('|');
- bool bad = true;
- if(c.count() == 2){//Добавляем пустое значение
- bad = addCookie(&c.at(0), &c.at(1));
- }
- else if(c.count() == 3){//Нормальная строка
- bad = addCookie(&c.at(0), &c.at(1), c.at(2));
- }
- if(bad) return -1;//Странная строка
- }
- }
- else{
- QByteArrayList split = TaskList.at(1).toLocal8Bit().split('|');//Куки из строки
- if(split.count() == 0) return -1;
- for(int i = 0; i < split.size(); i += 3){
- bool bad = true;
- if(i + 2 < split.size()){//Нормальная строка
- bad = addCookie(&split.at(i), &split.at(i + 1), split.at(i + 2));
- }
- else if(i + 1 < split.size()){//Добавляем пустое значение
- bad = addCookie(&split.at(i), &split.at(i + 1));
- }
- if(bad) return -1;//Странная строка
- }
- }
- return added;
- }
- void get_html(QByteArray &data_to_server){
- QString res = "";
- QEventLoop loop;
- page()->toHtml([&res,&loop](const QString &result){
- res = result;
- loop.exit(0);
- });
- loop.exec();
- data_to_server = res.toLocal8Bit();
- }
- void get_cookies(QByteArray &data_to_server){
- data_to_server = "";
- for(auto c : *cookies())//Преобразуем куки в формат miniPoster'а
- data_to_server += (c.domain() + "|" + c.name() + "|" + c.value() + "|").toLocal8Bit();
- if(data_to_server.length() > 0) data_to_server.remove(data_to_server.length() - 1, 1);//удаляем '|' в конце
- }
- void get_cookie_value_by_name(QByteArray &data_to_server, QStringList &TaskList){
- data_to_server = "";
- for(auto c : *cookies())
- if(c.name() == TaskList.at(1)){
- data_to_server = c.value();
- break;
- }
- }
- void get_cookie_by_domain(QByteArray &data_to_server, QStringList &TaskList){
- data_to_server = "";
- for(auto c : *cookies())
- if(c.domain().contains(TaskList[1]))
- data_to_server += c.name() + "|" + c.value() + "|";
- if(data_to_server.length() > 0) data_to_server.remove(data_to_server.length() - 1, 1);//удаляем '|' в конце
- }
- void get_domain_cookie_value_by_name(QByteArray &data_to_server, QStringList &TaskList){
- data_to_server = "";
- for(auto c : *cookies())
- if(c.domain() == TaskList.at(1) && c.name() == TaskList.at(2)){
- data_to_server = c.value();
- break;
- }
- }
- void screen_shot(QByteArray &data_to_server, QStringList &TaskList){
- QList<QString> listData = TaskList.at(2).split(":");
- listData.append(TaskList.at(3).split(":"));
- if(listData.size() < 4) return;
- QDir *dir = new QDir();
- static const QString path = "../captcha/";
- if(!dir->exists(path)) dir->mkdir(path);
- delete dir;
- int x = listData.at(0).toInt(), y = listData.at(1).toInt();
- QPixmap img = grab(QRect(QPoint(x,y), QSize(listData.at(2).toInt() - x, listData.at(3).toInt() - y)));
- img.save(path + TaskList.at(1) + ".png");
- data_to_server = "ok";
- }
- void pixel_exists(QByteArray &data_to_server, QStringList &TaskList){
- QList<QString> listData = TaskList.at(1).split(":");
- listData.append(TaskList.at(2).split(":"));
- data_to_server = "no";
- if(listData.size() < 4) return;
- int x = listData.at(0).toInt(), y = listData.at(1).toInt();
- int x1 = listData.at(2).toInt(), y1 = listData.at(3).toInt();
- QImage img = grab(QRect(QPoint(x,y), QSize(x1 - x, y1 - y))).toImage();
- QColor pixel(TaskList.at(3));
- bool find = false;
- for(int y = 0; y < img.height() && !find; ++y)
- for(int x = 0; x < img.width() && !find; ++x)
- if(img.pixel(x, y) == pixel.rgb())
- { data_to_server = "yes"; find = true; }
- if(!find)
- data_to_server = "no";
- }
- //Scroll
- QByteArray scroll_y_position(){
- return QString::number(page()->scrollPosition().ry()).toLocal8Bit();
- }
- QByteArray scroll_y_max(){
- return runJavaScript(_scroll_y_max);
- }
- QByteArray scroll_y(QString arg){
- return runJavaScript(_scroll_y.arg(arg));
- }
- QByteArray scroll_to_y(QString arg){
- return runJavaScript(_window_scroll_to.arg(page()->scrollPosition().rx()).arg(arg));
- }
- QByteArray scroll_x_position(){
- return QString::number(page()->scrollPosition().rx()).toLocal8Bit();
- }
- QByteArray scroll_x_max(){
- return runJavaScript(_scroll_x_max);
- }
- QByteArray scroll_x(QString arg){
- return runJavaScript(_scroll_x.arg(arg));
- }
- QByteArray scroll_to_x(QString arg){
- return runJavaScript(_window_scroll_to.arg(arg).arg(page()->scrollPosition().ry()));
- }
- //Element
- void element_set_value_by_id(QString id, QString value){
- runJavaScript(QString("document.querySelector('#%1').value='%2')").arg(id).arg(value));
- }
- void element_set_value_by_class(QString class_name, QString value){
- runJavaScript(QString("document.querySelector('.%1').value='%2')").arg(class_name).arg(value));
- }
- void element_set_inner_html_by_id(QString id, QString inner_html){
- runJavaScript(QString("document.querySelector('#%1').innerHTML='%2')").arg(id).arg(inner_html));
- }
- void element_set_inner_html_by_class(QString class_name, QString inner_html){
- runJavaScript(QString("document.querySelector('.%1').innerHTML='%2')").arg(class_name).arg(inner_html));
- }
- void element_click_by_id(QString id){
- runJavaScript(QString("document.querySelector('#%1').click()").arg(id));
- }
- void element_clock_by_class(QString class_name){
- runJavaScript(QString("document.querySelector('.%1').click()").arg(class_name));
- }
- //CAPTCHA
- bool getCaptchaResponse(QByteArray &data_to_server){
- switch (currCAPTCHA) {
- case reCap2: { data_to_server = runJavaScript("grecaptcha.getResponse()", true); break; }
- case reCap3: { data_to_server = runJavaScript("ResponseV3", true); break; }
- case hCap: { data_to_server = runJavaScript("hcaptcha.getResponse()", true); break; }
- default: return false;
- }
- currCAPTCHA = CAPTCHA::none;
- return true;
- }
- void load_v2reCAPTHCA(QString *baseURL, QString *siteKey){
- isLoaded = true;
- currCAPTCHA = CAPTCHA::reCap2;
- //загрузка reCAPTCHA2
- setHtml(reCAPTCHA2_source.arg(*siteKey).toLocal8Bit(), QUrl(*baseURL));
- }
- void load_v3reCAPTHCA(QStringList &TaskList){
- QString baseURL = TaskList[1];
- QString siteKey = TaskList[2];
- QString action = TaskList.size() == 4 ? TaskList[3] : "";
- load_v2reCAPTHCA(&baseURL, &siteKey);
- currCAPTCHA = CAPTCHA::reCap3;
- //загрузка reCAPTCHA3
- runJavaScript(reCAPTCHA3_source1.arg(siteKey));
- runJavaScript(reCAPTCHA3_source2.arg(siteKey).arg(action.length() == 0 ? "homepage" : action));
- }
- void load_hcaptcha(QStringList &TaskList){
- QString baseURL = TaskList[1];
- QString siteKey = TaskList[2];
- currCAPTCHA = CAPTCHA::hCap;
- //загрузка hCaptcha
- setHtml(hCAPTCHA_source.arg(siteKey), QUrl(baseURL));
- isLoaded = true;
- }
- CAPTCHA currCAPTCHA = CAPTCHA::none;
- private:
- QVector<QNetworkCookie> *cookiesV;
- void addCookie(const QNetworkCookie cookie){
- for(auto item : *cookiesV)
- if(item.hasSameIdentifier(cookie))
- return;//добавляем только новые куки
- cookiesV->append(cookie);
- }
- void proxyAuthenticationRequired(const QUrl &, QAuthenticator *auth, const QString &){
- auth->setUser (Proxy::User);
- auth->setPassword(Proxy::Pass);
- }
- };
- #endif // DATAHELPER_H
|