Haketilo

 * IMPORT is_privileged_url

 * IMPORT query_best

 * IMPORT gen_nonce

 * IMPORT inject_csp_headers

 * IMPORT apply_stream_filter

 * IMPORT is_chrome

let storage;

function on_headers_received(details)

{

    const url = details.url;

    if (is_privileged_url(details.url))

	return;

    const [pattern, settings] = query_best(storage, details.url);

    const allow = !!(settings && settings.allow);

    const nonce = gen_nonce();

    const policy = {allow, url, nonce};

    let headers = details.responseHeaders;

    let skip = false;

    for (const header of headers) {

	if ((header.name.toLowerCase().trim() === "content-disposition" &&

	     /^\s*attachment\s*(;.*)$/i.test(header.value)))

	    skip = true;

    }

    headers = inject_csp_headers(details, headers, policy);

    skip = skip || (details.statusCode >= 300 && details.statusCode < 400);

    if (!skip) {

	/* Check for API availability. */

	if (browser.webRequest.filterResponseData)

	    headers = apply_stream_filter(details, headers, policy);

    }

    return {responseHeaders: headers};

}

async function start_webRequest_operations()

{

    storage = await get_storage();

    const extra_opts = ["blocking", "responseHeaders"];

    if (is_chrome)

	extra_opts.push("extraHeaders");

    browser.webRequest.onHeadersReceived.addListener(

	on_headers_received,

	{urls: ["<all_urls>"], types: ["main_frame", "sub_frame"]},

	extra_opts

    );

}

start_webRequest_operations();

function inject_csp_headers(details, headers, policy)

	old_signature = old_data.policy_sig;

    headers = headers.filter(h => !is_csp_header_name(h.name, !policy.allow));

    const sanitizer = h => sanitize_csp_header(h, policy);

    const policy_str = encodeURIComponent(JSON.stringify(policy));

    const policy_sig = sign_data(policy_str, new Date());

	value: `hachette-${policy_sig}=${policy_str}; Expires=${later_30sec};`

    let hachette_data = {csp_headers: orig_csp_headers, policy_sig, url};

    if (!policy.allow)

	headers.push({

	    name: "content-security-policy",

	    value: csp_rule(policy.nonce)

	});

    return headers;

 * EXPORT inject_csp_headers

/**

 * Hachette modifying a web page using the StreamFilter API

 *

 * Copyright (C) 2018 Giorgio Maone <giorgio@maone.net>

 * Copyright (C) 2021 Wojtek Kosior

 * Redistribution terms are gathered in the `copyright' file.

 *

 * Derived from `bg/ResponseProcessor.js' and `bg/ResponseMetaData.js'

 * in LibreJS.

 */

/*

 * IMPORTS_START

 * IMPORT browser

 * IMPORTS_END

 */

function validate_encoding(charset)

{

    try {

	new TextDecoder();

	return charset;

    } catch(e) {

	return undefined;

    }

}

function is_content_type_header(header)

{

    header.name.toLowerCase().trim() === "content-type";

}

const charset_reg = /;\s*charset\s*=\s*([\w-]+)/i;

function properties_from_headers(headers)

{

    const properties = {};

    for (const header of headers.filter(is_content_type_header)) {

	const match = charset_reg.exec(header.value);

	if (!properties.detected_charset && validate_encoding(match[1]))

	    properties.detected_charset = match[1];

	if (/html/i.test(header.value))

	    properties.html = true;

    }

    return properties;

}

const UTF8_BOM = [0xef, 0xbb, 0xbf];

const BOMs = [

    [UTF8_BOM, "utf-8"],

    [[0xfe, 0xff], "utf-16be"],

    [[0xff, 0xfe], "utf-16le"]

];

function charset_from_BOM(data)

{

    for (const [BOM, charset] of BOMs) {

	if (BOM.reduce((ac, byte, i) => ac && byte === data[i], true))

	    return charset;

    }

    return "";

}

const charset_attrs =

      ['charset', 'http-equiv="content-type"', 'content*="charset"'];

const charset_meta_selector =

      charset_attrs.map(a => `head>meta[${a}]`).join(", ");

function charset_from_meta_tags(doc)

{

    for (const meta of doc.querySelectorAll(charset_meta_selector)) {

	const maybe_charset = meta.getAttribute("charset");

	if (maybe_charset && validate_encoding(maybe_charset))

	    return maybe_charset;

        const match = charset_reg.exec(meta.getAttribute("content"));

        if (match && validate_encoding(match[1]))

	    return match[1];

    }

    return undefined;

}

function create_decoder(properties, data)

{

    let charset = charset_from_BOM(data) || properties.detected_charset;

    if (!charset && data.indexOf(0) !== -1) {

        console.debug("Warning: zeroes in bytestream, probable cached encoding mismatch. Trying to decode it as UTF-16.",

		      properties);

	return new TextDecoder("utf-16be");

    }

    /* Missing HTTP charset, sniffing in content... */

    /*

     * TODO: I recall there is some standard saying how early in the doc the

     * charset has to be specified. We could process just this part of data.

     */

    const text = new TextDecoder("latin1").decode(data, {stream: true});

    properties.html = properties.html || /html/i.test(text);

    if (properties.html) {

	const tmp_doc = new DOMParser().parseFromString(text, "text/html");

	charset = charset_from_meta_tags(tmp_doc);

    }

    return new TextDecoder(charset || "latin1");

}

function filter_data(properties, event)

{

    const data = new Uint8Array(event.data);

    let first_chunk = false;

    if (!properties.decoder) {

	first_chunk = true;

	properties.decoder = create_decoder(properties, data);

	properties.encoder = new TextEncoder();

	/* Force UTF-8, this is the only encoding we can produce. */

	properties.filter.write(new Uint8Array(UTF8_BOM));

    }

    let decoded = properties.decoder.decode(data);

    if (first_chunk) {

	/*

	 * HAX! Our content scripts that execute at `document_start' will always

	 * run before the first script in the document, but under Mozilla some

	 * `<meta>' tags might already be loaded at that point. Here we inject a

	 * dummy `<script>' at the beginning (before any `<meta>' tags) that

	 * will force `document_start' to happen earlier. This way our content

	 * scripts will be able to sanitize `http-equiv' tags with CSP rules

	 * that would otherwise stop our injected scripts from executing.

	 */

	const dummy_script =

	      `<script data-hachette-deleteme="${properties.policy.nonce}" nonce="${properties.policy.nonce}">null</script>`;

	const doctype_decl = /^(\s*<!doctype[^<>"']*>)?/i.exec(decoded)[0];

	decoded = doctype_decl + dummy_script +

	    decoded.substring(doctype_decl.length);

    }

    properties.filter.write(properties.encoder.encode(decoded));

    if (properties.decoder.encoding === "utf-8")

	properties.filter.disconnect();

}

function apply_stream_filter(details, headers, policy)

{

    if (policy.allow)

	return headers;

    const properties = properties_from_headers(headers);

    properties.policy = policy;

    properties.filter =

	browser.webRequest.filterResponseData(details.requestId);

    properties.filter.ondata = event => filter_data(properties, event);

    properties.filter.onstop = () => properties.filter.close();

    /*

     * In the future we might consider modifying the headers that specify

     * encoding. For now we are not yet doing it, though. However, we

     * prepend the data with UTF-8 BOM which should be enough.

     */

    return headers;

}

/*

 * EXPORTS_START

 * EXPORT apply_stream_filter

 * EXPORTS_END

 */

    if (!policy.allow) {

function sanitize_script(script, data)

    if (script.getAttribute("data-hachette-deleteme") === data.policy.nonce) {

	script.remove();

	script.hachette_deleted = true;

	script.hachette_ignore = true;

    }

    if (data.policy.allow)

function inject_csp(head, data)

    if (data.policy.allow)

    meta.setAttribute("content", csp_rule(data.policy.nonce));

    data.new_added.unshift([meta, head]);

function sanitize_http_equiv_csp_rule(meta, data)

    const value = meta.content;

    if (!value || !is_csp_header_name(http_equiv, !data.policy.allow))

    block_attribute(meta, "content");

    if (data.policy.allow || is_csp_header_name(http_equiv, false))

	meta.content = sanitize_csp_header({value}, data.policy).value;

function sanitize_node(node, data)

	sanitize_script(node, data);

	inject_csp(node, data);

	sanitize_http_equiv_csp_rule(node, data);

    if (!data.policy.allow)

	sanitize_attributes(node, data);

}

/*

 * Instead of calling writer directly with multiple small chunks of reconstruced

 * HTML code, we utilize `setTimeout()' to only have it called once,

 * asynchronously.

 */

function do_write_callback(data)

{

    data.writer(data.chunks.join(""));

    data.chunks = [];

    if (data.finished && data.finisher)

	data.finisher();

}

function do_write(chunk, data)

{

    data.chunks.push(chunk);

    clearTimeout(data.write_timeout);

    data.write_timeout = setTimeout(() => do_write_callback(data), 0);

function start_serializing_node(node, data)

    node.hachette_started = true;

    do_write(data.uniq_reg.exec(clone.outerHTML)[1], data);

function finish_serializing_node(node, data)

	node.hachette_ignore = true;

	    do_write(data.uniq_reg.exec(node.outerHTML)[2], data);

	    continue;

	}

	do_write(node.outerHTML || serializer.serializeToString(node), data);

    }

}

function process_initial_nodes(node, data)

{

    if (data.processed_initial_nodes)

	return;

    data.processed_initial_nodes = true;

    start_serializing_node(data.html_root, data);

    const new_added = [];

    const nodes_to_process = [data.html_root];

    let i = 0;

    while (nodes_to_process.length > 0) {

	let current = nodes_to_process.shift();

	if (current.firstChild) {

	    if (current.firstChild === node)

		break;

	    nodes_to_process.unshift(current.firstChild, current);

	    new_added.push([current.firstChild, current]);

	while (current && !current.nextSibling)

	    current = nodes_to_process.shift();

	if (!current || current.nextSibling === node)

	    break;

	nodes_to_process.unshift(current.nextSibling);

	new_added.push([current.nextSibling, nodes_to_process[1]]);

    data.new_added.unshift(...new_added);

 * in MutationRecords under Chromium. Track node relations manually.

    /*

     * Functions we call here might cause new nodes to be injected or found

     * that require processing before the one we got in function argument.

     * We rely on those functions putting the node(s) they create/find at the

     * very beginning of the `new_added' queue and (for created nodes) setting

     * their `hachette_ignore' property, based on which their MutationRecord

     * will not be processed. A function can also mark a node already in the

     * `new_added' queue as not eligible for processing by setting its

     * `hachette_deleted' property.

     */

    process_initial_nodes(node, data);

    data.new_added.push([node, true_parent]);

    while (data.new_added.length > 0) {

	[node, true_parent] = data.new_added.shift();

	if (true_parent.hachette_deleted)

	    node.hachette_deleted = true;

	if (node.hachette_deleted)

	    continue;

	if (!true_parent.hachette_started)

	    start_serializing_node(true_parent, data)

	if (!node.hachette_ignore)

	    sanitize_node(node, data);

	if (node.hachette_deleted)

	    continue;

	if (data.node_eater)

	    data.node_eater(node, true_parent);

	finish_serializing_node(true_parent.hachette_last_added, data);

	true_parent.hachette_last_added = node;

    }

	for (const node of mutation.addedNodes) {

	    /* Check for nodes added by ourselves. */

	    if (mutation.target.hachette_ignore)

		node.hachette_ignore = true;

	    if (node.hachette_ignore)

		continue;

	}

    process_initial_nodes(undefined, data);

    /*

     * The `finisher' callback should be called, if provided. Normally our

     * function that performs the last write does it after seeing `finished'

     * set to `true'. If, however, there's no `writer' callback and hence no

     * writes to perform, we need to take care of calling `finisher' here.

     */

    data.finished = true;

    /*

     * Additional whitespace that was after `</body>' gets appended to body.

     * Although it's a minor issue, it is not what we want. There's no way to

     * tell exactly what part of that whitespace was after `</body>' and what

     * was before, so we just replace it with a single newline which looks good

     * when printed.

     */

    const body = data.html_root.lastChild;

    const text = body && body.tagName === "BODY" && body.lastChild;

    if (text && text.nodeName === "#text") {

	const new_content = /^([\S\s]*\S)?\s*$/.exec(text.textContent)[1] || "";

	text.textContent = new_content + "\n";

    }

    finish_serializing_node(data.html_root, data);

    if (!data.writer && data.finisher)

	setTimeout(data.finisher, 0);

/*

 * This function sanitizes `html_root' according to `policy'. It is capable of

 * working on an HTML document that is being written to, sanitizing new nodes

 * as they appear.

 *

 * `consumers' object may contain 3 optional callback functions: `writer',

 * `node_eater' and `finisher'. The first one, if present, is called with chunks

 * of reconstructed HTML code. The second one, if present, gets called for every

 * added node with 2 arguments: that node and its parent. The third one is

 * called at the end, after all processing has been done.

 *

 * `modify_on_the_fly()' returns a callback that should be called (with no

 * arguments) once the document of html_root has finished being written to.

 * Unfortunately, due to specifics behavior of document that has had its

 * documentElement replaced

 */

function modify_on_the_fly(html_root, policy, consumers)

    const uniq_reg = new RegExp(`^([\\s\\S]*)${uniq}([\\s\\S]*)$`);

    const data = {policy, html_root, uniq, uniq_reg, chunks: [], new_added: []};

    Object.assign(data, consumers);

    observer.observe(data.html_root, {

Files: background/stream_filter.js

Copyright: 2018 Giorgio Maone <giorgio@maone.net>

   2021 Wojtek Kosior <koszko@koszko.org>

License: GPL-3+-javascript or Alicense-1.0, and GPL-3+

Comment: Code by Wojtek is dual-licensed under GPL-3+-javascript and

 Alicense-1.0. Giorgio's code is under GPL-3+.