ematrix/js/assets.js

/*******************************************************************************

    ηMatrix - a browser extension to black/white list requests.
    Copyright (C) 2013-2019 Raymond Hill
    Copyright (C) 2019-2022 Alessio Vanni

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://gitlab.com/vannilla/ematrix
    uMatrix Home: https://github.com/gorhill/uMatrix
*/

'use strict';

ηMatrix.assets = (function() {

    let api = {};
    let observers = [];
    let externalPathRegex = /^(?:[a-z-]+):\/\//;
    let connectionError = vAPI.i18n('errorCantConnectTo');

    let notifyObservers = function (topic, details) {
	let result;

	for (let i=0; i<observers.length; ++i) {
	    result = observers[i](topic, details);
	}

	return result;
    }

    function isEmptyString(s) {
	return (typeof s === 'string' && s === '');
    }

    function noOp() {
	return;
    }

    // Cache Registry
    var cacheRegistry = new Object();
    let cacheRegistryReady = false;
    let cacheRegistryCallbacks = undefined;
    let cacheRegistryStart = Date.now();

    let saveCacheRegistry = (function () {
	let timer;

	let save = function () {
	    timer = undefined;
	    vAPI.cacheStorage.set({
		assetCacheRegistry: cacheRegistry,
	    });
	}

	return function (lazy) {
	    if (timer !== undefined) {
		clearTimeout(timer);
	    }

	    if (lazy === true) {
		timer = vAPI.setTimeout(save, 500);
	    } else {
		save();
	    }
	};
    })();

    let getCacheRegistry = function (callback) {
	if (cacheRegistryReady == true) {
	    callback(cacheRegistry);
	    return;
	}

	if (cacheRegistryCallbacks !== undefined) {
	    // If it's not undefined it's always an array
	    //
	    // eMatrix: this block in particular is probably never
	    // called: originally, it was used because uMatrix called
	    // some code in a callback that could wait
	    //
	    // While waiting, more elements could've been pushed in
	    // the array
	    //
	    // Since the waiting callback is not used here, any
	    // further callback are likely to be handled by the
	    // condition above
	    // This block is left here just in case
	    cacheRegistryCallbacks.push(callback);
	    return;
	}

	cacheRegistryCallbacks = [callback];

	let onRead = function (bin) {
	    if (!bin || !bin['assetCacheRegistry']) {
		cacheRegistry = {};
	    } else {
		cacheRegistry = bin['assetCacheRegistry'];
	    }

	    cacheRegistryReady = true;

	    let f;
	    while ((f = cacheRegistryCallbacks.shift())) {
		f(cacheRegistry);
	    }
	};

	vAPI.cacheStorage.get('assetCacheRegistry', onRead);
    };

    let readCache = function (key, callback) {
	let fullkey = 'cache/' + key;
	let cache = { placeholder: true };

	let report = function (content, error) {
	    let details = {
		assetKey: key,
		content: content,
	    };

	    if (error) {
		details.error = error;
	    }

	    return callback(details);
	};

	let onRead = function (bin) {
	    if (!bin || !bin[fullkey]) {
		return report('', 'E_NOTFOUND');
	    }

	    let entry = cache[key];
	    if (entry === undefined) {
		return report('', 'E_NOTFOUND');
	    }

	    entry.readTime = Date.now();
	    saveCacheRegistry(true);

	    report(bin[fullkey]);
	};

	let onReady = function (registry) {
	    cache = registry;
	    vAPI.cacheStorage.get(fullkey, onRead);
	};

	getCacheRegistry(onReady);
    };

    let writeCache = function (key, details, callback) {
	let fullkey = 'cache/' + key;
	let content = '';

	if (typeof details === 'string') {
	    content = details;
	} else if (details instanceof Object) {
	    content = details.content || '';
	}

	if (content === '') {
	    return removeCache(key, callback);
	}

	let report = function (content) {
	    let details = {
		assetKey: key,
		content: content,
	    };

	    if (typeof callback === 'function') {
		callback(details);
	    }

	    notifyObservers('after-asset-updated', details);
	};

	let onReady = function (registry) {
	    let entry = registry[key];
	    if (entry === undefined) {
		entry = registry[key] = {};
	    }

	    entry.writeTime = entry.readTime = Date.now();
	    if (details instanceof Object && typeof details.url === 'string') {
		entry.remoteURL = details.url;
	    }

	    let bin = {
		assetCacheRegistry: cacheRegistry,
	    };

	    bin[fullkey] = content;
	    vAPI.cacheStorage.set(bin);
	    report(content);
	};

	getCacheRegistry(onReady);
    };

    let removeCache = function (pattern, callback) {
	let onReady = function (cache) {
	    let entries = [];
	    let contents = [];

	    for (let key in cache) {
		if (pattern instanceof RegExp && !pattern.test(key)) {
		    continue;
		}

		if (typeof pattern === 'string' && key !== pattern) {
		    continue;
		}

		entries.push(key);
		contents.push('cache/' + key);

		delete cache[key];
	    }

	    if (contents.length !== 0) {
		vAPI.cacheStorage.remove(contents);

		let bin = {
		    assetCacheRegistry: cache,
		};
		vAPI.cacheStorage.set(bin);
	    }

	    if (typeof callback === 'function') {
		callback();
	    }

	    for (let i=0; i<entries.length; ++i) {
		notifyObservers('after-asset-updated', {
		    assetKey: entries[i],
		});
	    }
	};

	getCacheRegistry(onReady);
    };

    let markDirtyCache = function (pattern, exclude, callback) {
	let onReady = function (registry) {
	    let entry;
	    let mustSave = false;

	    for (let key in registry) {
		if (pattern instanceof RegExp && pattern.test(key) === false) {
		    continue;
		} else if (typeof pattern === 'string' && key !== pattern) {
		    continue;
		} else if (Array.isArray(pattern)
			   && pattern.indexOf(key) === -1) {
		    continue;
		}

		if (exclude instanceof RegExp && exclude.test(key)) {
		    continue;
		} else if (typeof exclude === 'string' && key === exclude) {
		    continue;
		} else if (Array.isArray(exclude)
			   && exclude.indexOf(key) !== -1) {
		    continue;
		}

		entry = registry[key];
		if (!entry.writeTime) {
		    continue;
		}

		registry[key].writeTime = 0;
		mustSave = true;
	    }

	    if (mustSave) {
		let bin = {
		    assetCacheRegistry: registry,
		};
		vAPI.cacheStorage.set(bin);
	    }

	    if (typeof callback === 'function') {
		callback();
	    }
	};

	if (typeof exclude === 'function') {
	    callback = exclude;
	    exclude = undefined;
	}

	getCacheRegistry(onReady);
    };

    // Source Registry
    var sourceRegistry = Object.create(null);
    let sourceRegistryReady = false;
    let sourceRegistryCallbacks = undefined;

    let saveSourceRegistry = (function () {
	let timer;

	let save = function () {
	    timer = undefined;
	    vAPI.cacheStorage.set({
		assetSourceRegistry: sourceRegistry,
	    });
	}

	return function (lazy) {
	    if (timer !== undefined) {
		clearTimeout(timer);
	    }

	    if (lazy === true) {
		timer = vAPI.setTimeout(save, 500);
	    } else {
		save();
	    }
	};
    })();

    let registerSource = function (key, details) {
	let entry = sourceRegistry[key] || {};

	for (let p in details) {
	    if (details.hasOwnProperty(p) === false) {
		continue;
	    }

	    if (details[p] !== undefined) {
		entry[p] = details[p];
	    } else {
		delete entry[p];
	    }
	}

	let contentUrl = details.contentURL;
	if (contentUrl) {
	    if (typeof contentUrl === 'string') {
		contentUrl = entry.contentURL = [contentUrl];
	    } else if (Array.isArray(contentUrl) === false) {
		contentUrl = entry.contentURL = [];
	    }

	    let remoteCount = 0;

	    for (let i=0; i<contentUrl.length; ++i) {
		if (externalPathRegex.test(contentUrl[i])) {
		    ++remoteCount;
		}
	    }

	    entry.hasLocalURL = (remoteCount !== contentUrl.length);
	    entry.hasRemoteURL = (remoteCount !== 0);
	} else {
	    entry.contentURL = [];
	}

	if (typeof entry.updateAfter !== 'number') {
	    entry.updateAfter = 13;
	}

	if (entry.submitter) {
	    entry.submitTime = Date.now(); // Detects stale entries
	}

	sourceRegistry[key] = entry;
    };

    let unregisterSource = function (key) {
	removeCache(key);
	delete sourceRegistry[key];
	saveSourceRegistry();
    };

    let updateSourceRegistry = function (string, silent) {
	let json;

	try {
	    json = JSON.parse(string);
	} catch (e) {
	    return;
	}

	if (json instanceof Object === false) {
	    return;
	}

	for (let key in sourceRegistry) {
	    if (json[key] === undefined
		&& sourceRegistry[key].submitter === undefined) {
		unregisterSource(key);
	    }
	}

	for (let key in json) {
	    if (sourceRegistry[key] === undefined && !silent) {
		notifyObservers('builtin-asset-source-added', {
		    assetKey: key,
		    entry: json[key],
		});
	    }

	    registerSource(key, json[key]);
	}

	saveSourceRegistry();
    };

    let getSourceRegistry = function (callback) {
	if (sourceRegistryReady === true) {
	    callback(sourceRegistry);
	    return;
	}

	if (sourceRegistryCallbacks !== undefined) {
	    // If it's not undefined it's always an array
	    sourceRegistryCallbacks.push(callback);
	    return;
	}

	sourceRegistryCallbacks = [callback];

	let onReady = function () {
	    sourceRegistryReady = true;

	    let f;
	    while ((f = sourceRegistryCallbacks.shift())) {
		f(sourceRegistry);
	    }
	};

	let createRegistry = function () {
	    api.fetchText
	    (ηMatrix.assetsBootstrapLocation || 'assets/assets.json',
	     function (details) {
		 updateSourceRegistry(details.content, true);
		 onReady();
	     });
	};

	let onRead = function (bin) {
	    if (!bin || !bin.assetSourceRegistry
		|| Object.keys(bin.assetSourceRegistry).length == 0) {
		createRegistry();
		return;
	    }

	    sourceRegistry = bin.assetSourceRegistry;
	    onReady();
	};

	vAPI.cacheStorage.get('assetSourceRegistry', onRead);
    };

    // Remote
    let getRemote = function (key, callback) {
	let assetDetails = {};
	let contentUrl = '';
	let urls = [];

	let report = function (content, error) {
	    let details = {
		assetKey: key,
		content: content,
	    };
	    if (error) {
		details.error = assetDetails.lastError = error;
	    } else {
		assetDetails.lastError = undefined;
	    }
	    callback(details);
	};

	let tryLoad = function (tries) {
	    let tr = (tries === undefined) ? 10 : tries;

	    if (tr <= 0) {
		console.warn('ηMatrix could not load the asset '
			     +assetDetails.title);
		return;
	    }

	    while ((contentUrl = urls.shift())) {
		if (externalPathRegex.test(contentUrl)) {
		    break;
		}
	    }

	    if (!contentUrl) {
		report('', 'E_NOTFOUND');
		return;
	    }

	    api.fetchText(contentUrl,
			  onRemoteContentLoad,
			  onRemoteContentError,
			  tr-1);
	};

	let onRemoteContentLoad = function (details, tries) {
	    if (isEmptyString(details.content) === true) {
		registerSource(key, {
		    error: {
			time: Date.now(),
			error: 'No content'
		    }
		});
		return tryLoad(tries);
	    }

	    writeCache(key, {
		content: details.content,
		url: contentUrl,
	    });

	    registerSource(key, {error: undefined});
	    report(details.content);
	};

	let onRemoteContentError = function (details, tries) {
	    let text = details.statusText;
	    if (details.statusCode === 0) {
		text = 'network error';
	    }
	    registerSource(key, {
		error: {
		    time: Date.now(),
		    error: text,
		}
	    });
	    tryLoad(tries);
	};

	let onReady = function (registry) {
	    assetDetails = registry[key] || {};
	    if (typeof assetDetails.contentURL === 'string') {
		urls = [assetDetails.contentURL];
	    } else if (Array.isArray(assetDetails.contentURL)) {
		urls = assetDetails.contentURL.slice(0);
	    }
	    tryLoad();
	};

	getSourceRegistry(onReady);
    };

    // Updater
    let updateStatus = 'stop';
    let updateDefaultDelay = 120000;
    let updateDelay = updateDefaultDelay;
    let updateTimer;
    let updated = [];
    let updateFetch = new Set();

    let updateStart = function () {
	updateStatus = 'running';
	updateFetch.clear();
	updated = [];
	notifyObservers('before-assets-updated');
	updateNext();
    };

    let updateNext = function () {
	let cacheReg = undefined;
	let sourceReg = undefined;

	let gcOne = function (key) {
	    let entry = cacheReg[key];
	    if (entry && entry.readTime < cacheRegistryStart) {
		removeCache(key);
	    }
	};

	let findOne = function () {
	    let now = Date.now();
	    let sourceEntry;
	    let cacheEntry;

	    for (let key in sourceReg) {
		sourceEntry = sourceReg[key];
		if (sourceEntry.hasRemoteURL !== true) {
		    continue;
		}
		if (updateFetch.has(key)) {
		    continue;
		}

		cacheEntry = cacheReg[key];
		if (cacheEntry
		    && (cacheEntry.writeTime
			+ sourceEntry.updateAfter*86400000) > now) {
		    continue;
		}
		if (notifyObservers('before-asset-updated', {assetKey: key})) {
		    return key;
		}

		gcOne(key);
	    }

	    return undefined;
	};

	let onUpdate = function (details) {
	    if (details.content !== '') {
		updated.push(details.assetKey);
		if (details.assetKey === 'assets.json') {
		    updateSourceRegistry(details.content);
		}
	    } else {
		notifyObservers('asset-update-failed', {
		    assetKey: details.assetKey,
		});
	    }

	    if (findOne() !== undefined) {
		vAPI.setTimeout(updateNext, updateDelay);
	    } else {
		updateEnd();
	    }
	};

	let updateOne = function () {
	    let key = findOne();
	    if (key === undefined) {
		updateEnd();
		return;
	    }

	    updateFetch.add(key);
	    getRemote(key, onUpdate);
	};

	let onSourceReady = function (registry) {
	    sourceReg = registry;
	    updateOne();
	};

	let onCacheReady = function (registry) {
	    cacheReg = registry;
	    getSourceRegistry(onSourceReady);
	};

	getCacheRegistry(onCacheReady);
    };

    let updateEnd = function () {
	let keys = updated.slice(0);
	updateFetch.clear();
	updateStatus = 'stop';
	updateDelay = updateDefaultDelay;
	notifyObservers('after-asset-updated', {
	    assetKeys: keys,
	});
    };

    // Assets API
    api.addObserver = function (observer) {
	if (observers.indexOf(observer) === -1) {
	    observers.push(observer);
	}
    };

    api.removeObserver = function (observer) {
	let pos = observers.indexOf(observer);
	while (pos !== -1) {
	    observers.splice(pos, 1);
	    pos = observers.indexOf(observer);
	}
    };

    api.fetchText = function (url, onLoad, onError, tries) {
	let iurl = externalPathRegex.test(url) ? url : vAPI.getURL(url);
	let tr = (tries === undefined) ? 10 : tries;

	if (typeof onError !== 'function') {
	    onError = onLoad;
	}

	let onResponseReceived = function () {
	    this.onload = this.onerror = this.ontimeout = null;

	    let details = {
		url: url,
		content: '',
		// On local files this.status is 0, but the request
		// is successful
		statusCode: this.status || 200,
		statusText: this.statusText || '',
	    };

	    if (details.statusCode < 200 || details.statusCode >= 300) {
		return onError.call(null, details, tr);
	    }

	    if (isEmptyString(this.responseText) === true) {
		return onError.call(null, details, tr);
	    }

	    let t = this.responseText.trim();

	    // Discard HTML as it's probably an error
	    // (the request expects plain text as a response)
	    if (t.startsWith('<') && t.endsWith('>')) {
		return onError.call(null, details, tr);
	    }

	    details.content = t;
	    return onLoad.call(null, details, tr);
	};

	let onErrorReceived = function () {
	    this.onload = this.onerror = this.ontimeout = null;

	    ηMatrix.logger.writeOne('', 'error',
				    connectionError.replace('{{url}}', iurl));

	    onError.call(null, {
		url: url,
		content: '',
	    }, tr);
	};

	let req = new XMLHttpRequest();
	req.open('GET', iurl, true);
	req.timeout = 30000;
	req.onload = onResponseReceived;
	req.onerror = onErrorReceived;
	req.ontimeout = onErrorReceived;
	req.responseType = 'text';

	try {
	    // This can throw in some cases
	    req.send();
	} catch (e) {
	    onErrorReceived.call(req);
	}
    };

    api.registerAssetSource = function (key, details) {
	getSourceRegistry(function () {
	    registerSource(key, details);
	    saveSourceRegistry(true);
	});
    };

    api.unregisterAssetSource = function (key) {
	getSourceRegistry(function () {
	    unregisterSource(key);
	    saveSourceRegistry(true);
	});
    };

    api.get = function (key, options, callback) {
	let cb;
	let opt;

	if (typeof options === 'function') {
	    cb = options;
	    opt = {};
	} else if (typeof callback !== 'function') {
	    cb = noOp;
	    opt = options;
	} else {
	    cb = callback;
	    opt = options;
	}

	let assetDetails = {};
	let urls = [];
	let contentUrl = '';

	let report = function (content, error) {
	    let details = {
		assetKey: key,
		content: content,
	    };

	    if (error) {
		details.error = assetDetails.error = error;
	    } else {
		assetDetails.error = undefined;
	    }

	    cb(details);
	};

	let onContentNotLoaded = function (details, tries) {
	    let external;

	    let tr = (tries === undefined) ? 10 : tries;

	    if (tr <= 0) {
		console.warn('ηMatrix couldn\'t download the asset '
			     +assetDetails.title);
		return;
	    }

	    while ((contentUrl = urls.shift())) {
		external = externalPathRegex.test(contentUrl);
		if (external === true && assetDetails.loaded !== true) {
		    break;
		}
		if (external === false || assetDetails.hasLocalURL !== true) {
		    break;
		}
	    }

	    if (!contentUrl) {
		return report('', 'E_NOTFOUND');
	    }

	    api.fetchText(contentUrl,
			  onContentLoaded,
			  onContentNotLoaded,
			  tr-1);
	};

	let onContentLoaded = function (details, tries) {
	    if (isEmptyString(details.content) === true) {
		return onContentNotLoaded(undefined, tries);
	    }

	    if (externalPathRegex.test(details.url)
		&& opt.dontCache !== true) {
		writeCache(key, {
		    content: details.content,
		    url: contentUrl,
		});
	    }

	    assetDetails.loaded = true;

	    report(details.content);
	};

	let onCachedContentLoad = function (details) {
	    if (details.content !== '') {
		return report(details.content);
	    }

	    let onRead = function (details) {
		console.debug(details);
		report(details.content || 'missing contents');
	    }

	    let onReady = function (registry) {
		assetDetails = registry[key] || {};
		if (typeof assetDetails.contentURL === 'string') {
		    urls = [assetDetails.contentURL];
		} else if (Array.isArray(assetDetails.contentURL)) {
		    urls = assetDetails.contentURL.slice(0);
		}
		if (true === assetDetails.loaded) {
		    readCache(key, onRead);
		} else {
		    onContentNotLoaded();
		}
	    }

	    getSourceRegistry(onReady);
	};

	readCache(key, onCachedContentLoad);
    };

    api.put = function (key, content, callback) {
	writeCache(key, content, callback);
    };

    api.metadata = function (callback) {
	let onSourceReady = function (registry) {
	    let source = JSON.parse(JSON.stringify(registry));
	    let cache = cacheRegistry;
	    let sourceEntry;
	    let cacheEntry;
	    let now = Date.now();
	    let obsoleteAfter;

	    for (let key in source) {
		sourceEntry = source[key];
		cacheEntry = cache[key];

		if (cacheEntry) {
		    sourceEntry.cached = true;
		    sourceEntry.writeTime = cacheEntry.writeTime;
		    obsoleteAfter = cacheEntry.writeTime
			+ sourceEntry.updateAfter * 86400000;
		    sourceEntry.obsolete = obsoleteAfter < now;
		    sourceEntry.remoteURL = cacheEntry.remoteURL;
		} else {
		    sourceEntry.writeTime = 0;
		    obsoleteAfter = 0;
		    sourceEntry.obsolete = true;
		}
	    }

	    callback(source);
	}

	let onCacheReady = function () {
	    getSourceRegistry(onSourceReady);
	}

	getCacheRegistry(onCacheReady);
    };

    api.purge = function (pattern, exclude, callback) {
	markDirtyCache(pattern, exclude, callback);
    };

    api.remove = function (pattern, callback) {
	removeCache(pattern, callback);
    };

    api.rmrf = function () {
	removeCache(/./);
    };

    api.updateStart = function (details) {
	let oldDelay = updateDelay;
	let newDelay = details.delay || updateDefaultDelay;

	updateDelay = Math.min(oldDelay, newDelay);

	if (updateStatus !== 'stop') {
	    if (newDelay < oldDelay) {
		clearTimeout(updateTimer);
		updateTimer = vAPI.setTimeout(updateNext, updateDelay);
	    }
	    return;
	}

	updateStart();
    };

    api.updateStop = function () {
	if (updateTimer) {
	    clearTimeout(updateTimer);
	    updateTimer = undefined;
	}
	if (updateStatus === 'running') {
	    updateEnd();
	}
    };

    api.checkVersion = function () {
	let cache;

	let onSourceReady = function (registry) {
	    let source = JSON.parse(JSON.stringify(registry));
	    let version = ηMatrix.userSettings.assetVersion;

	    if (!version) {
		ηMatrix.userSettings.assetVersion = 1;
		version = 1;
	    }

	    if (!source["assets.json"].version
		|| version > source["assets.json"].version) {
		for (let key in source) {
		    switch (key) {
		    case "hphosts":
		    case "malware-0":
		    case "malware-1":
			delete source[key];
			api.remove(key, function () {});
			break;
		    default:
			break;
		    }

		    source["assets.json"].version = version;
		}

		let createRegistry = function () {
		    api.fetchText
		    (ηMatrix.assetsBootstrapLocation || 'assets/assets.json',
		     function (details) {
			 updateSourceRegistry(details.content, true);
		     });
		};

		createRegistry();
	    }
	};

	let onCacheReady = function (registry) {
	    cache = JSON.parse(JSON.stringify(registry));

	    getSourceRegistry(onSourceReady);
	};

	getCacheRegistry(onCacheReady);
    };

    return api;
})();