DSBDirect/app/src/main/java/godau/fynn/dsbdirect/manager/DownloadManager.java

/*
 * DSBDirect
 * Copyright (C) 2019 Fynn Godau
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * This software is not affiliated with heinekingmedia GmbH, the
 * developer of the DSB platform.
 */

package godau.fynn.dsbdirect.manager;

import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import android.util.Base64;
import android.util.Log;

import godau.fynn.dsbdirect.table.*;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import godau.fynn.dsbdirect.Login;
import godau.fynn.dsbdirect.QueryMetadata;
import godau.fynn.dsbdirect.R;
import godau.fynn.dsbdirect.Utility;
import godau.fynn.dsbdirect.table.reader.Reader;

public class DownloadManager {

    private Context mContext;

    public DownloadManager(Context context) {
        mContext = context;
    }

    /**
     * Make a GET request (synchronously)
     *
     * @param url           URL to be requested
     * @param body          Request body (JSON String)
     * @param requestMethod Usually either GET or POST
     * @return Response
     * @throws IOException If networking error or other IO exception
     */
    private @NonNull
    InputStream request(String url, @Nullable String body, String requestMethod) throws IOException {

        if (!isNetworkAvailable()) throw new IOException();

        // Encode the url correctly, as the file name part can contain Umlaute or other weird things
        String[] urlParts =
                url.split("/(?!.*/)"); // Matches only the last '/' character (using a negative lookahead)

        url = urlParts[0] + "/" + URLEncoder.encode(
                urlParts[1] // This is the part that has to be encoded correctly
                        .replaceAll(
                                "%20", " " /* Spaces are already encoded as "%20". Let's decode them quickly so we won't have
                                 * %20 encoded as something like "%2520"
                                 */
                        ), "ISO-8859-1" // UTF-8 won't work here
        )
                .replaceAll(
                        "\\+", "%20" /* Spaces are encoded again, but they are now '+' chars. That's unfortunately not
                         * correct.  Let's replace them with "%20"s.
                         */
                );

        URL connectwat = new URL(url);
        HttpURLConnection urlConnection = (HttpURLConnection) connectwat.openConnection();

        urlConnection.setRequestMethod(requestMethod);

        // Add DNT header, as if it does anything
        urlConnection.addRequestProperty("DNT", "1");

        if (body != null) {

            // Get headers from sharedPreferences so they can be set through news
            HashSet queryHeaders = (HashSet) new Utility(mContext).getSharedPreferences()
                    .getStringSet("queryHeaders", new HashSet<>(Arrays.asList(
                            "Referer: https://www.dsbmobile.de/",
                            "Content-Type: application/json;charset=utf-8"
                    )));

            // Add each header to query
            for (Object header : queryHeaders) {
                String queryHeader = (String) header;

                // Split header into two parts
                String[] queryHeaderParts = queryHeader.split(": ");

                // Check whether header really is two parts
                if (queryHeaderParts.length != 2) {
                    Log.e("DOWNLOADHEADER", "invalid header: " + queryHeader);
                    continue;
                }

                // Add header to request
                urlConnection.setRequestProperty(queryHeaderParts[0], queryHeaderParts[1]);

                Log.d("DOWNLOADHEADER", queryHeader);
            }

            OutputStream outputStream = urlConnection.getOutputStream();
            OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream);
            outputStreamWriter.write(body);
            outputStreamWriter.flush();
            outputStreamWriter.close();
            outputStream.close();
        }

        urlConnection.connect();

        return new BufferedInputStream(urlConnection.getInputStream());
    }

    /**
     * Downloads a list of all contents currently available from DSB (synchronously)
     *
     * @param login Login to log in with
     * @return A ContentInformation object with all contents in DSB
     * @throws LoginFailureException       If the credentials or the request in general are incorrect
     * @throws UnexpectedResponseException If response is invalid JSON
     * @throws IOException                 If request fails in general (networking error?)
     */
    public ContentInformation downloadContentInformation(Login login) throws IOException {

        JSONArray contentArray = downloadContentJSONArray(login);

        Table[] tables = null;
        List<Notice> notices = null;
        List<NewsItem> news = null;

        try {
            for (int i = 0; i < contentArray.length(); i++) {
                JSONObject contentObject = contentArray.getJSONObject(i);

                String contentName = contentObject.getString("Title");

                JSONArray childs = contentObject.getJSONObject("Root").getJSONArray("Childs");

                // It has been observed that News are before Pläne if they exist
                switch (contentName) {
                    case "Pläne":
                        tables = Reader.readTableList(childs);
                        break;
                    case "News":
                        news = Reader.readNewsList(childs);
                        break;
                    case "Aushänge":
                        notices = Reader.readNoticeList(childs);
                }
            }
        } catch (JSONException e) {
            // Response is invalid, throw further
            throw new UnexpectedResponseException(e.getCause());
        }

        return new ContentInformation(tables, notices, news);
    }

    /**
     * Downloads and returns the Childs JSON array of Inhalte from DSB (synchronously)
     *
     * @param login Login to log in with
     * @return JSON array containing the Childs of Inhalte of server's response
     * @throws LoginFailureException       If the credentials or the request in general are incorrect
     * @throws UnexpectedResponseException If response is invalid JSON
     * @throws IOException                 If request fails in general (networking error?)
     */
    private JSONArray downloadContentJSONArray(Login login) throws IOException {
        Log.d("DOWNLOAD", "downloading data");

        if (!login.isNonZeroLength()) {
            // Empty credentials are not valid
            throw new LoginFailureException();
        }

        // Make request body
        JSONObject bodyObject;
        SharedPreferences sharedPreferences = new Utility(mContext).getSharedPreferences();
        try {
            // Query body base json might be overwritten by news, otherwise use hardcoded value
            String queryBodyBaseJson = sharedPreferences
                    .getString("queryBodyBaseJson", mContext.getString(R.string.query_body_base_json));
            Log.d("DOWNLOADBASEJSON", queryBodyBaseJson);


            bodyObject = new JSONObject(queryBodyBaseJson);
            login.put(bodyObject);

            // Add things configurable through news
            boolean sendAppId = sharedPreferences.getBoolean("querySendAppId", true);
            boolean sendAndroidVersion = sharedPreferences.getBoolean("querySendAndroidVersion", true);
            boolean sendDeviceModel = sharedPreferences.getBoolean("querySendDeviceModel", true);
            boolean sendLanguage = sharedPreferences.getBoolean("querySendLanguage", true);
            boolean sendDate = sharedPreferences.getBoolean("querySendDate", false);
            boolean sendLastDate = sharedPreferences.getBoolean("querySendLastDate", false);

            // Generate AppId
            if (sendAppId) {
                bodyObject.put("AppId", QueryMetadata.getAppId());
            }

            // Attach random android version
            if (sendAndroidVersion) {
                bodyObject.put("OsVersion", QueryMetadata.getAndroidVersion());
            }

            // Attach random device name
            if (sendDeviceModel) {
                bodyObject.put("Device", QueryMetadata.getDeviceModel());
            }

            // Attach some language
            if (sendLanguage) {
                bodyObject.put("Language", QueryMetadata.getLanguage());
            }

            // Send date
            if (sendDate) {
                // Date should look like this: 2019-10-04T14:21:3728600
                SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss:SSSS000");
                String date = dateFormat.format(new Date());
                Log.d("DOWNLOADDATE", date);
                bodyObject.put("Date", date);

                // Send last date
                if (sendLastDate) {
                    bodyObject.put("LastUpdate", sharedPreferences.getString("queryLastDate", ""));
                    sharedPreferences.edit()
                            .putString("queryLastDate", date)
                            .apply();
                }
            }

        } catch (JSONException e) {
            e.printStackTrace();
            // Shouldn't happen! Throw further as IOException since we don't really know what happened anyway
            throw new IOException(e.getCause());
        }

        // Request url
        String url = getEndpoint(sharedPreferences.getInt("queryEndpoint", 0));
        Log.d("DOWNLOADENDPOINT", url);

        // Request
        String response = string(
                request(url, obfuscateQuery(bodyObject), "POST"),
                "UTF-8"
        );

        // If request is very invalid, there won't be any json in the response, only some plain text…
        if (response.equals("Unzulässige Anforderung")) {
            Log.d("DOWNLOAD", "request failed: " + bodyObject.toString() + " obfuscated to " + obfuscateQuery(bodyObject));
            throw new UnexpectedResponseException();
        }

        try {
            JSONObject responseBody = deobfuscateResponse(response);

            // Check result code
            int resultcode = responseBody.getInt("Resultcode");
            switch (resultcode) {
                case 0:
                    // All is good, continue
                    break;
                case 1:
                    // Invalid credentials ("ResultStatusInfo": "Login fehlgeschlagen")
                    throw new LoginFailureException();
                default:
                    Log.d("DOWNLOAD", "unexpected Resultcode " + resultcode + ": " + response);
                    throw new UnexpectedResponseException();
            }

            JSONArray resultMenuItems = responseBody.getJSONArray("ResultMenuItems");
            resultMenu:
            for (int i = 0; i < resultMenuItems.length(); i++) {
                JSONObject resultMenuItem = resultMenuItems.getJSONObject(i);
                String resultMenuItemTitle = resultMenuItem.getString("Title");

                // Just to be sure that we select Inhalte, in practice it has only been observed to be the first one
                if (resultMenuItemTitle.equals("Inhalte")) {

                    return resultMenuItem.getJSONArray("Childs");
                }
            }

            // No Inhalte…?
            throw new UnexpectedResponseException("No Inhalte found");

        } catch (JSONException | EOFException e) {
            e.printStackTrace();
            // Response is invalid, throw further
            throw new UnexpectedResponseException(e.getCause());
        }
    }

    /**
     * Return the corresponding url for the endpoint id.
     *
     * @param id 0 (mobile) / 1 (web) / 2 (ihkmobile) / 3 (appihkbb)
     */
    private String getEndpoint(int id) throws IOException {
        switch (id) {
            case 0:
            default:
                return "https://app.dsbcontrol.de/JsonHandler.ashx/GetData";
            case 1:
                String webConfiguration = string(
                        request("https://www.dsbmobile.de/scripts/configuration.js", null, "GET"),
                        "UTF-8"
                );
                return "https://www.dsbmobile.de/" + webConfiguration.split("'")[3]; //bad solution, web endpoint obfuscated -> method outdated

            case 2:
                return "https://ihkmobile.dsbcontrol.de/new/JsonHandlerWeb.ashx/GetData";
            case 3:
                return "https://appihkbb.dsbcontrol.de/new/JsonHandlerWeb.ashx/GetData";
        }
    }

    /**
     * Downloads an html table file (synchronously)
     *
     * @param table       The table which is to be downloaded
     * @param fileManager A file manager to be used to save the file
     * @return The downloaded html String
     * @throws IOException In case of networking error
     */
    public String downloadHtmlTable(final Table table, final FileManager fileManager) throws IOException { // TODO
        Log.d("DOWNLOAD", "downloading html file at " + table.getUri());

        // Request
        String html = string(request(table.getUri(), null, "GET"), "ISO-8859-1");

        // If these characters appear, the wrong encoding has been used
        if (html.contains("")) {
            try {
                html = new String(html.getBytes("ISO-8859-1"), "UTF-8");
            } catch (UnsupportedEncodingException e) {
                // So be it!
                e.printStackTrace();
            }

        }

        // Save file
        fileManager.saveFile(table, html);

        return html;
    }

    /**
     * Downloads a bitmap table file (synchronously)
     *
     * @param table       The table which is to be downloaded
     * @param fileManager A file manager to be used to save the file
     * @return The bitmap
     * @throws IOException In case of networking error
     */
    public Bitmap downloadImageTable(final Table table, final FileManager fileManager) throws IOException { // TODO
        // We're doing bitmaps.
        Log.d("DOWNLOAD", "downloading image file at " + table.getUri());

        // Request bitmap
        InputStream inputStream = request(table.getUri(), null, "GET");

        // Create bitmap
        Bitmap bitmap = BitmapFactory.decodeStream(inputStream);

        // Save bitmap
        fileManager.saveFile(table, bitmap);

        // Return bitmap
        return bitmap;
    }

    /**
     * Obfuscate query. The DSB server requires this.
     * <br/><br/>
     * Queries are "compressed" using gzip (saving less than half a kilobyte) and then encoded using base64.
     * The result of that is then again hidden inside some more JSON.
     *
     * @param query The JSON query you want to make
     * @return The body you will have to send to the server to execute the query
     */
    private String obfuscateQuery(JSONObject query) throws IOException {
        String queryString = query.toString();

        // Thanks, https://stackoverflow.com/a/6718707
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream(queryString.length());
        GZIPOutputStream gzip = new GZIPOutputStream(outputStream);
        gzip.write(queryString.getBytes());
        gzip.close();
        byte[] gzipped = outputStream.toByteArray();
        outputStream.close();

        String encoded = Base64.encodeToString(gzipped, Base64.NO_WRAP); // Line wraps are useless here!

        return mContext.getString(R.string.query_body_outer_json, encoded);
    }

    /**
     * Deobfuscate response. The DSB server gives obfuscated responses.
     * <br/><br/>
     * Just the reverse of {@link #obfuscateQuery(JSONObject)}, except that "some more JSON" is different for responses
     * compared to queries.
     *
     * @param response The response the server gave you
     * @return The JSON hidden inside the response
     */
    private JSONObject deobfuscateResponse(String response) throws JSONException, IOException {

        JSONObject responseObject = new JSONObject(response);
        String encoded = responseObject.getString("d");


        byte[] gzipped = Base64.decode(encoded, Base64.DEFAULT);

        // Who knows how this works… thanks again, https://stackoverflow.com/a/6718707
        final int BUFFER_SIZE = 32;
        ByteArrayInputStream is = new ByteArrayInputStream(gzipped);
        GZIPInputStream gis = new GZIPInputStream(is, BUFFER_SIZE);
        StringBuilder stringBuilder = new StringBuilder();
        byte[] data = new byte[BUFFER_SIZE];
        int bytesRead;
        while ((bytesRead = gis.read(data)) != -1) {
            stringBuilder.append(new String(data, 0, bytesRead));
        }
        gis.close();
        is.close();

        return new JSONObject(stringBuilder.toString());
    }

    public JSONObject downloadUpdateCheck() throws IOException {
        Log.d("DOWNLOAD", "downloading update check");

        try {
            return new JSONObject(string(request(mContext.getString(R.string.uri_versioncheck), null, "GET"), "UTF-8"));
        } catch (JSONException e) {
            throw new UnexpectedResponseException(e.getCause());
        }
    }

    public JSONObject downloadNews() throws IOException {
        Log.d("DOWNLOAD", "downloading news");

        try {
            return new JSONObject(string(request(mContext.getString(R.string.uri_news), null, "GET"), "UTF-8"));
        } catch (JSONException e) {
            throw new UnexpectedResponseException(e.getCause());
        }
    }

    // Thanks, https://stackoverflow.com/a/35446009
    private String string(InputStream in, String charsetName) throws IOException {
        ByteArrayOutputStream result = new ByteArrayOutputStream();
        byte[] buffer = new byte[1024];
        int length;
        while ((length = in.read(buffer)) != -1) {
            result.write(buffer, 0, length);
        }

        return result.toString(charsetName);

    }

    /**
     * Uploads a url to the server at https://dsb.bixilon.de to ask the developer to develop a parser for it.
     * <br/><br/>The server code is available at <a href="https://notabug.org/fynngodau/dsbdirect-filedump/src/master/requestParser.php">fynngodau/dsbdirect-filedump</a>.
     *
     * @param url The url to upload. Must be at <a href="https://app.dsbcontrol.de">https://app.dsbcontrol.de</a>
     * @return whether the server returned 200 Success
     * @throws IOException in case of a network error
     */
    public boolean uploadParserRequest(String url) throws IOException {
        // request(…) returns an InputStream, not the response code as it is pretty much always 200, so we can't use it here

        if (!isNetworkAvailable()) {
            throw new IOException();
        }

        URL connectwat = new URL(mContext.getString(R.string.uri_requestparser));
        HttpURLConnection urlConnection = (HttpURLConnection) connectwat.openConnection();

        urlConnection.setRequestMethod("POST");

        OutputStream outputStream = urlConnection.getOutputStream();
        OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream);
        outputStreamWriter.write("url=" + url);
        outputStreamWriter.flush();
        outputStreamWriter.close();
        outputStream.close();

        urlConnection.connect();

        if (urlConnection.getResponseCode() == 200) return true;
        else return false;
    }

    private boolean isNetworkAvailable() {
        ConnectivityManager connectivityManager
                = (ConnectivityManager) mContext.getSystemService(Context.CONNECTIVITY_SERVICE);
        NetworkInfo activeNetworkInfo = connectivityManager.getActiveNetworkInfo();
        return activeNetworkInfo != null && activeNetworkInfo.isConnected();
    }

    /**
     * Download all shortcodes from eltern-portal.org (synchronously)
     *
     * @param url      URL to be requested
     * @param email    Login Username (email)
     * @param password Password for login
     * @return Shortcode    null on error, array on success
     * @throws IOException              If networking error or other IO exception
     * @throws IllegalArgumentException If the provided url is incorrect
     * @throws LoginFailureException    If the credentials were not correct
     */
    public Shortcode[] downloadShortcodesFromElternportal(String url, String email, String password)
            throws IOException, IllegalArgumentException, LoginFailureException {
        //is network available?
        if (!isNetworkAvailable()) throw new IOException();
        //attach an / if needed
        if (!url.endsWith("/"))
            url = url + "/";
        //check url
        if (!url.endsWith(".eltern-portal.org/")) {
            //url invalid
            throw new IllegalArgumentException();
        }
        if (url.startsWith("http://")) {
            url = url.replace("http://", "https://"); //force https
        }
        if (!url.toLowerCase().matches("^\\w+://.*")) {
            url = "https://" + url;
        }
        //check if url is parsable
        try {
            URL u = new URL(url);
            u.toURI();
        } catch (URISyntaxException e) {
            //nope. not valid
            e.printStackTrace();
            throw new IllegalArgumentException(e);
        }

        //obtain session key
        Connection.Response res = null;
        res = Jsoup.connect(url)
                .execute();

        //login with obtained session key
        res = Jsoup.connect(url + "includes/project/auth/login.php")
                .data("username", email, "password", password)
                .method(Connection.Method.POST)
                .cookies(res.cookies())
                .followRedirects(false)
                .execute();

        if (res.body().contains("Fehler bei der Anmeldung") || res.header("Location").contains(".eltern-portal.org/login?errno=1&username=")) {
            //fail: wrong session key or username/password wrong
            throw new LoginFailureException();
        }

        //get html of school information
        Document doc = Jsoup.connect(url + "service/schulinformationen")
                .cookies(res.cookies())
                .get();
        Elements e = doc.getElementById("asam_content").select("div[class=row m_bot]");
        boolean getthem = false; //there are not only shotcodes also school infos. Wait until shortcodes appear
        List<Shortcode> shortcodes = new ArrayList<Shortcode>();
        for (Element s : e) {
            String raw = s.select("div[class=col-md-4],div[class=col-md-6]").text();
            //empty line. skipping
            if (raw.equals(""))
                continue;
            if (raw.contains("Homepage")) { //Homepage is the last school info. Follows by shortcodes
                getthem = true;
                continue;
            }
            if (!getthem)
                continue;

            String split[] = raw.split(" ", 3); //short short n a m e
            shortcodes.add(new Shortcode(split[0], split[2]));
        }
        Shortcode array[] = new Shortcode[shortcodes.size()];
        array = shortcodes.toArray(array);
        return array;
    }

    /**
     * Holds information about all the content that is currently offered at the DSB. Notices
     * and news are treated as the same.
     */
    public static class ContentInformation implements Serializable {
        private Table[] tables;
        private ArrayList<NoticeBoardItem> notices;

        public ContentInformation(Table[] tables, List<Notice> notices, List<NewsItem> news) {
            this.tables = tables == null ? new Table[0] : tables;
            this.notices = new ArrayList<>();
            if (notices != null) this.notices.addAll(notices);
            if (news != null) this.notices.addAll(news);
        }

        public Table[] getTables() {
            return tables;
        }

        public ArrayList<NoticeBoardItem> getNotices() {
            return notices;
        }

        /**
         * @return Whether at least one notice board item is available.
         */
        public boolean hasNotices() {
            return notices.size() > 0;
        }
    }

    static public class UnexpectedResponseException extends IOException {
        public UnexpectedResponseException() {
            super();
        }

        public UnexpectedResponseException(Throwable cause) {
            super(cause);
        }

        public UnexpectedResponseException(String message) {
            super(message);
        }
    }

    /**
     * Possible causes: credentials incorrect, server rejected login due to an invalid request
     */
    static public class LoginFailureException extends UnexpectedResponseException {
    }

    static public class NoContentException extends UnexpectedResponseException {
    }
}