ematrix/js/assets.js

/*******************************************************************************

    ηMatrix - a browser extension to black/white list requests.
    Copyright (C) 2013-2019 Raymond Hill
    Copyright (C) 2019 Alessio Vanni

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://libregit.org/heckyel/ematrix
    uMatrix Home: https://github.com/gorhill/uMatrix
*/

'use strict';

/******************************************************************************/

ηMatrix.assets = (function() {

    let api = {};
    let observers = [];
    let externalPathRegex = /^(?:[a-z-]+):\/\//;
    let connectionError = vAPI.i18n('errorCantConnectTo');

    let notifyObservers = function (topic, details) {
        let result;

        for (let i=0; i<observers.length; ++i) {
            result = observers[i](topic, details);
        }

        return result;
    }

    function isEmptyString(s) {
        return (typeof s === 'string' && s === '');
    }

    function noOp() {
        return;
    }

    // Cache Registry
    var cacheRegistry = new Object();
    let cacheRegistryReady = false;
    let cacheRegistryCallbacks = undefined;
    let cacheRegistryStart = Date.now();

    let saveCacheRegistry = (function () {
        let timer;

        function save() {
            timer = undefined;
            vAPI.cacheStorage.set({
                assetCacheRegistry: cacheRegistry,
            });
        }

        return function (lazy) {
            if (timer !== undefined) {
                clearTimeout(timer);
            }

            if (lazy === true) {
                timer = vAPI.setTimeout(save, 500);
            } else {
                save();
            }
        };
    })();

    let getCacheRegistry = function (callback) {
        if (cacheRegistryReady == true) {
            callback(cacheRegistry);
            return;
        }

        if (cacheRegistryCallbacks !== undefined) {
            // If it's not undefined it's always an array
            //
            // eMatrix: this block in particular is probably never
            // called: originally, it was used because uMatrix called
            // some code in a callback that could wait
            //
            // While waiting, more elements could've been pushed in
            // the array
            //
            // Since the waiting callback is not used here, any
            // further callback are likely to be handled by the
            // condition above
            // This block is left here just in case
            cacheRegistryCallbacks.push(callback);
            return;
        }

        cacheRegistryCallbacks = [callback];
        cacheRegistryReady = true;

        let onRead = function (bin) {
            if (!bin || !bin['assetCacheRegistry']) {
                cacheRegistry = {};
            } else {
                cacheRegistry = bin['assetCacheRegistry'];
            }
        };

        vAPI.cacheStorage.get('assetCacheRegistry', onRead);

        let f;
        while ((f = cacheRegistryCallbacks.shift())) {
            f(cacheRegistry);
        }
    };

    let readCache = function (key, callback) {
        let report = function (content, error) {
            let details = {
                assetKey: key,
                content: content,
            };

            if (error) {
                details.error = error;
            }

            callback(details);
        };

        let onRead = function (bin) {
            if (!bin || !bin[key]) {
                report('', 'E_NOTFOUND');
                return;
            }

            let entry = cacheRegistry[key];
            if (entry === undefined) {
                let onRead2 = function (bin) {
                    if (!bin || !bin['assetCacheRegistry']) {
                        cacheRegistry = {};
                    } else {
                        cacheRegistry = bin['assetCacheRegistry'];
                    }
                };

                vAPI.cacheStorage.get('assetCacheRegistry', onRead2);

                entry = cacheRegistry[key];
                if (entry === undefined) {
                    report('', 'E_NOTFOUND');
                    return;
                }
            }

            entry.readTime = Date.now();
            saveCacheRegistry(true);

            report(bin[key]);
        };

        let onReady = function () {
            vAPI.cacheStorage.get(key, onRead);
        };

        getCacheRegistry(onReady);
    };

    let writeCache = function (key, details, callback) {
        let content = '';

        if (typeof details === 'string') {
            content = details;
        } else if (details instanceof Object) {
            content = details.content || '';
        }

        if (content === '') {
            removeCache(key, callback);
            return;
        }

        let report = function (content) {
            let details = {
                assetKey: key,
                content: content,
            };

            if (typeof callback === 'function') {
                callback(details);
            }

            notifyObservers('after-asset-updated', details);
        };

        let onReady = function () {
            let entry = cacheRegistry[key];
            if (entry === undefined) {
                entry = cacheRegistry[key] = {};
            }

            entry.writeTime = entry.readTime = Date.now();
            if (details instanceof Object && typeof details.url === 'string') {
                entry.remoteURL = details.url;
            }

            let bin = {
                assetCacheRegistry: cacheRegistry,
            };

            bin[key] = content;
            vAPI.cacheStorage.set(bin);
            report(content);
        };

        getCacheRegistry(onReady);
    };

    let cacheRemove = function (pattern, callback) {
        let onReady = function (cache) {
            let entries = [];

            for (let key in cache) {
                if (pattern instanceof RegExp && !pattern.test(key)) {
                    continue;
                }

                if (typeof pattern === 'string' && key !== pattern) {
                    continue;
                }

                entries.push(key);

                delete cache[key];
            }

            if (entries.length !== 0) {
                vAPI.cacheStorage.remove(content);

                let bin = {
                    assetCacheRegistry: cache,
                };
                vAPI.cacheStorage.set(bin);
            }

            if (typeof callback === 'function') {
                callback();
            }

            for (let i=0; i<entries.length; ++i) {
                notifyObservers('after-asset-updated', {
                    assetKey: entries[i],
                });
            }
        };

        getCacheRegistry(onReady);
    };

    let markDirtyCache = function (pattern, exclude, callback) {
        let onReady = function (registry) {
            let entry;
            let mustSave = false;

            for (let key in registry) {
                if (pattern instanceof RegExp && pattern.test(key) === false) {
                    continue;
                } else if (typeof pattern === 'string' && key !== pattern) {
                    continue;
                } else if (Array.isArray(pattern)
                           && pattern.indexOf(key) === -1) {
                    continue;
                }

                if (exclude instanceof RegExp && exclude.test(key)) {
                    continue;
                } else if (typeof exclude === 'string' && key === exclude) {
                    continue;
                } else if (Array.isArray(exclude)
                           && exclude.indexOf(key) !== -1) {
                    continue;
                }

                entry = registry[key];
                if (!entry.writeTime) {
                    continue;
                }

                registry[key].writeTime = 0;
                mustSave = true;
            }

            if (mustSave) {
                let bin = {
                    assetCacheRegistry: registry,
                };
                vAPI.cacheStorage.set(bin);
            }

            if (typeof callback === 'function') {
                callback();
            }
        };

        if (typeof exclude === 'function') {
            callback = exclude;
            exclude = undefined;
        }

        getCacheRegistry(onReady);
    };

    let removeCache = function (key, callback) {
        let onReady = function (cacheRegistry) {
            let removedEntries = [];
            let removedContent = [];

            for (let k in cacheRegistry) {
                if (key instanceof RegExp && !key.test(k)) {
                    continue;
                }
                if (typeof key === 'string' && k !== key) {
                    continue;
                }

                removedEntries.push(k);
                removedContent.push(k);

                delete cacheRegistry[k];
            }

            if (removedContent.length > 0) {
                vAPI.cacheStorage.remove(removedContent);

                let bin = {
                    assetCacheRegistry: cacheRegistry,
                };

                vAPI.cacheStorage.set(bin);
            }

            if (typeof callback === 'function') {
                callback();
            }

            for (let i=0; i<removedEntries.length; ++i) {
                notifyObservers('after-asset-updated', {
                    assetKey: removedEntries[i],
                });
            }
        };

        getCacheRegistry(onReady);
    };

    // Source Registry
    var sourceRegistry = Object.create(null);
    let sourceRegistryReady = false;
    let sourceRegistryCallbacks = undefined;

    let saveSourceRegistry = (function () {
        let timer;

        function save() {
            timer = undefined;
            vAPI.cacheStorage.set({
                assetSourceRegistry: sourceRegistry,
            });
        }

        return function (lazy) {
            if (timer !== undefined) {
                clearTimeout(timer);
            }

            if (lazy === true) {
                timer = vAPI.setTimeout(save, 500);
            } else {
                save();
            }
        };
    })();

    let registerSource = function (key, details) {
        let entry = sourceRegistry[key] || {};

        for (let p in details) {
            if (details.hasOwnProperty(p) === false) {
                continue;
            }

            if (details[p] !== undefined) {
                entry[p] = details[p];
            } else {
                delete entry[p];
            }
        }

        let contentUrl = details.contentURL;
        if (contentUrl) {
            if (typeof contentUrl === 'string') {
                contentUrl = entry.contentURL = [contentUrl];
            } else if (Array.isArray(contentUrl) === false) {
                contentUrl = entry.contentURL = [];
            }

            let remoteCount = 0;

            for (let i=0; i<contentUrl.length; ++i) {
                if (externalPathRegex.test(contentUrl[i])) {
                    ++remoteCount;
                }
            }

            entry.hasLocalURL = (remoteCount !== contentUrl.length);
            entry.hasRemoteURL = (remoteCount !== 0);
        } else {
            entry.contentURL = [];
        }

        if (typeof entry.updateAfter !== 'number') {
            entry.updateAfter = 13;
        }

        if (entry.submitter) {
            entry.submitTime = Date.now(); // Detects stale entries
        }

        sourceRegistry[key] = entry;
    };

    let unregisterSource = function (key) {
        removeCache(key);
        delete sourceRegistry[key];
        saveSourceRegistry();
    };

    let updateSourceRegistry = function (string, silent) {
        let json;

        try {
            json = JSON.parse(string);
        } catch (e) {
            return;
        }

        for (let key in sourceRegistry) {
            if (json[key] === undefined
                && sourceRegistry[key].submitter === undefined) {
                unregisterSource(key);
            }
        }

        for (let key in json) {
            if (sourceRegistry[key] === undefined && !silent) {
                notifyObservers('builtin-asset-source-added', {
                    assetKey: key,
                    entry: json[key],
                });
            }

            registerSource(key, json[key]);
        }

        saveSourceRegistry();
    };

    let getSourceRegistry = function (callback) {
        if (sourceRegistryReady === true) {
            callback(sourceRegistry);
            return;
        }

        if (sourceRegistryCallbacks !== undefined) {
            // If it's not undefined it's always an array
            sourceRegistryCallbacks.push(callback);
            return;
        }

        sourceRegistryCallbacks = [callback];

        let onReady = function () {
            sourceRegistryReady = true;

            let f;
            while ((f = sourceRegistryCallbacks.shift())) {
                f(sourceRegistry);
            }
        };

        let createRegistry = function () {
            api.fetchText
            (ηMatrix.assetsBootstrapLocation || 'assets/assets.json',
             function (details) {
                 updateSourceRegistry(details.content, true);
                 onReady();
             });
        };

        let onRead = function (bin) {
            if (!bin || !bin.assetSourceRegistry
                || Object.keys(bin.assetSourceRegistry).length == 0) {
                createRegistry();
                return;
            }

            sourceRegistry = bin.assetSourceRegistry;
            onReady();
        };

        vAPI.cacheStorage.get('assetSourceRegistry', onRead);
    };

    // Remote
    let getRemote = function (key, callback) {
        let assetDetails = {};
        let contentUrl;

        let report = function (content, error) {
            let details = {
                assetKey: key,
                content: content,
            };
            if (error) {
                details.error = assetDetails.lastError = error;
            }
            callback(details);
        };

        let tryLoad = function (tries) {
            let urls = [];

            let tr = (tries === undefined) ? 10 : tries;

            if (tr <= 0) {
                console.warn('ηMatrix could not load the asset '
                             +assetDetails.title);
                return;
            }

            if (typeof assetDetails.contentURL === 'string') {
                urls = [assetDetails.contentURL];
            } else if (Array.isArray(assetDetails.contentURL)) {
                urls = assetDetails.contentURL.slice(0);
            }

            while ((contentUrl = urls.shift())) {
                if (externalPathRegex.test(contentUrl)) {
                    break;
                }
            }

            if (!contentUrl) {
                report('', 'E_NOTFOUND');
                return;
            }

            api.fetchText(contentUrl, onRemoteContentLoad, onRemoteContentError,
                          tr-1);
        };

        let onRemoteContentLoad = function (details, tries) {
            if (isEmptyString(details.content) === true) {
                registerSource(key, {
                    error: {
                        time: Date.now(),
                        error: 'No content'
                    }
                });
                tryLoad(tries);
            }

            writeCache(key, {
                content: details.content,
                url: contentUrl,
            });

            registerSource(key, {error: undefined});
            report(details.content);
        };

        let onRemoteContentError = function (details, tries) {
            let text = detail.statusText;
            if (details.statusCode === 0) {
                text = 'network error';
            }
            registerSource(key, {
                error: {
                    time: Date.now(),
                    error: text,
                }
            });
            tryLoad(tries);
        };

        let onReady = function (registry) {
            assetDetails = registry[key] || {};
            tryLoad();
        };

        getSourceRegistry(onReady);
    };

    // Updater
    let updateStatus = 'stop';
    let updateDefaultDelay = 120000;
    let updateDelay = updateDefaultDelay;
    let updateTimer;
    let updated = [];
    let updateFetch = new Set();

    let updateStart = function () {
        updateStatus = 'running';
        updateFetch.clear();
        updated = [];
        notifyObservers('before-assets-updated');
        updateNext();
    };

    let updateNext = function () {
        let gcOne = function (key) {
            let entry = cacheRegistry[key];
            if (entry && entry.readTime < cacheRegistryStart) {
                cacheRemove(key);
            }
        };

        let findOne = function () {
            let now = Date.now();
            let sourceEntry;
            let cacheEntry;

            for (let key in sourceRegistry) {
                sourceEntry = sourceRegistry[key];
                if (sourceEntry.hasRemoteURL !== true) {
                    continue;
                }
                if (updateFetch.has(key)) {
                    continue;
                }

                cacheEntry = cacheRegistry[key];
                if (cacheEntry
                    && (cacheEntry.writeTime
                        + sourceEntry.updateAfter*86400000) > now) {
                    continue;
                }
                if (notifyObservers('before-asset-updated', {assetKey: key})) {
                    return key;
                }

                gcOne(key);
            }

            return undefined;
        };

        let onUpdate = function (details) {
            if (details.content !== '') {
                updated.push(details.assetKey);
                if (details.assetKey === 'asset.json') {
                    updateSourceRegistry(details.content);
                }
            } else {
                notifyObservers('asset-update-failed', {
                    assetKey: details.assetKey,
                });
            }

            if (findOne() !== undefined) {
                vAPI.setTimeout(updateNext, updateDelay);
            } else {
                updateEnd();
            }
        };

        let updateOne = function () {
            let key = findOne();
            if (key === undefined) {
                updateEnd();
                return;
            }

            updateFetch.add(key);
            getRemote(key, onUpdate);
        };

        let onSourceReady = function (registry) {
            updateOne();
        };

        let onCacheReady = function (registry) {
            getSourceRegistry(onSourceReady);
        };

        getCacheRegistry(onCacheReady);
    };

    let updateEnd = function () {
        let keys = updated.slice(0);
        updateFetch.clear();
        updateStatus = 'stop';
        updateDelay = updateDefaultDelay;
        notifyObservers('after-asset-updated', {
            assetKeys: keys,
        });
    };

    // Assets API
    api.addObserver = function (observer) {
        if (observers.indexOf(observer) === -1) {
            observers.push(observer);
        }
    };

    api.removeObserver = function (observer) {
        let pos = observers.indexOf(observer);
        if (pos !== -1) {
            observers.splice(pos, 1);
        }
    };

    api.fetchText = function (url, onLoad, onError, tries) {
        let iurl = externalPathRegex.test(url) ? url : vAPI.getURL(url);
        let tr = (tries === undefined) ? 10 : tries;

        if (typeof onError !== 'function') {
            onError = onLoad;
        }

        let onResponseReceived = function () {
            this.onload = this.onerror = this.ontimeout = null;

            let details = {
                url: url,
                content: '',
                // On local files this.status is 0, but the request
                // is successful
                statusCode: this.status || 200,
                statusText: this.statusText || '',
            };

            if (details.statusCode < 200 || details.statusCode >= 300) {
                onError.call(null, details, tr);
                return;
            }

            if (isEmptyString(this.responseText) === true) {
                onError.call(null, details, tr);
                return;
            }

            let t = this.responseText.trim();

            // Discard HTML as it's probably an error
            // (the request expects plain text as a response)
            if (t.startsWith('<') && t.endsWith('>')) {
                onError.call(null, details, tr);
                return;
            }

            details.content = t;
            onLoad.call(null, details, tr);
        };

        let onErrorReceived = function () {
            this.onload = this.onerror = this.ontimeout = null;

            ηMatrix.logger.writeOne('', 'error',
                                    connectionError.replace('{{url}}', iurl));

            onError.call(null, {
                url: url,
                content: '',
            }, tr);
        };

        let req = new XMLHttpRequest();
        req.open('GET', iurl, true);
        req.timeout = 30000;
        req.onload = onResponseReceived;
        req.onerror = onErrorReceived;
        req.ontimeout = onErrorReceived;
        req.responseType = 'text';

        try {
            // This can throw in some cases
            req.send();
        } catch (e) {
            onErrorReceived.call(req);
        }
    };

    api.registerAssetSource = function (key, details) {
        getSourceRegistry(function () {
            registerSource(key, details);
            saveSourceRegistry(true);
        });
    };

    api.unregisterAssetSource = function (key) {
        getSourceRegistry(function () {
            unregisterSource(key);
            saveSourceRegistry(true);
        });
    };

    api.get = function (key, options, callback) {
        let cb;
        let opt;

        if (typeof options === 'function') {
            cb = options;
            opt = {};
        } else if (typeof callback !== 'function') {
            cb = noOp;
            opt = options;
        } else {
            cb = callback;
            opt = options;
        }

        let assetDetails = {};
        let contentUrl = undefined;

        let report = function (content, error) {
            let details = {
                assetKey: key,
                content: content,
            };

            if (error) {
                details.error = assetDetails.error = error;
            } else {
                assetDetails.error = undefined;
            }

            cb(details);
        };

        let onContentNotLoaded = function (details, tries) {
            let external;
            let urls = [];

            let tr = (tries === undefined) ? 10 : tries;

            if (tr <= 0) {
                console.warn('ηMatrix couldn\'t download the asset '
                             +assetDetails.title);
                return;
            }

            if (typeof assetDetails.contentURL === 'string') {
                urls = [assetDetails.contentURL];
            } else if (Array.isArray(assetDetails.contentURL)) {
                urls = assetDetails.contentURL.slice(0);
            }

            while ((contentUrl = urls.shift())) {
                external = externalPathRegex.test(contentUrl);
                if (external === true && assetDetails.loaded !== true) {
                    break;
                }
                if (external === false || assetDetails.hasLocalURL !== true) {
                    break;
                }
            }

            if (!contentUrl) {
                report('', 'E_NOTFOUND');
                return;
            }

            api.fetchText(contentUrl, onContentLoaded, onContentNotLoaded,
                          tr-1);
        };

        let onContentLoaded = function (details, tries) {
            if (isEmptyString(details.content) === true) {
                onContentNotLoaded(undefined, tries);
                return;
            }

            if (externalPathRegex.test(details.url)
                && opt.dontCache !== true) {
                writeCache(key, {
                    content: details.content,
                    url: contentUrl,
                });
            }

            assetDetails.loaded = true;

            report(details.content);
        };

        let onCachedContentLoad = function (details) {
            if (details.content !== '') {
                report(details.content);
                return;
            }

            let onReady = function (registry) {
                assetDetails = registry[key] || {};
                onContentNotLoaded();
            }

            getSourceRegistry(onReady);
        };

        readCache(key, onCachedContentLoad);
    };

    api.put = function (key, content, callback) {
        writeCache(key, content, callback);
    };

    api.metadata = function (callback) {
        let onSourceReady = function (registry) {
            let source = JSON.parse(JSON.stringify(registry));
            let cache = cacheRegistry;
            let sourceEntry;
            let cacheEntry;
            let now = Date.now();
            let obsoleteAfter;

            for (let key in source) {
                sourceEntry = source[key];
                cacheEntry = cache[key];

                if (cacheEntry) {
                    sourceEntry.cached = true;
                    sourceEntry.writeTime = cacheEntry.writeTime;
                    obsoleteAfter = cacheEntry.writeTime
                        + sourceEntry.updateAfter * 86400000;
                    sourceEntry.obsolete = obsoleteAfter < now;
                    sourceEntry.remoteURL = cacheEntry.remoteURL;
                } else {
                    sourceEntry.writeTime = 0;
                    obsoleteAfter = 0;
                    sourceEntry.obsolete = true;
                }
            }

            callback(source);
        }

        let onCacheReady = function () {
            getSourceRegistry(onSourceReady);
        }

        getCacheRegistry(onCacheReady);
    };

    api.purge = function (pattern, exclude, callback) {
        markDirtyCache(pattern, exclude, callback);
    };

    api.remove = function (pattern, callback) {
        cacheRemove(pattern, callback);
    };

    api.rmrf = function () {
        cacheRemove(/./);
    };

    api.updateStart = function (details) {
        let oldDelay = updateDelay;
        let newDelay = details.delay || updateDefaultDelay;

        updateDelay = Math.min(oldDelay, newDelay);

        if (updateStatus === 'running') {
            if (newDelay < oldDelay) {
                clearTimeout(updateTimer);
                updateTimer = vAPI.setTimeout(updateNext, updateDelay);
            }
            return;
        }

        updateStart();
    };

    api.updateStop = function () {
        if (updateTimer) {
            clearTimeout(updateTimer);
            updateTimer = undefined;
        }
        if (updateStatus === 'running') {
            updateEnd();
        }
    };

    return api;
})();

/******************************************************************************/