Haketilo

 *           requests through this background script using one-off messages.

#FROM common/misc.js    IMPORT uint8_to_hex, error_data_jsonifiable

/*

 * In this file we implement a fetch()-from-background-script service. Code in

 * other parts of the extension shall call sendMessage() with arguments to

 * fetch() and code here will call fetch() with those arguments and send back

 * the response.

 *

 * We have to convert the Response from fetch() into a JSON-ifiable value and

 * then convert it back (using Response() constructor) due (among others) the

 * limitations of Chromium's messaging API:

 * https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Chrome_incompatibilities#data_cloning_algorithm

 *

 * We also catch possible errors from fetch() (e.g. in case of an invalid URL)

 * and send their data in JSON-ifiable form so that the error object can be

 * later re-created.

 */

/* Make it possible to serialize Response object. */

async function response_data_jsonifiable(response) {

    return {

	status:     response.status,

	statusText: response.statusText,

	headers:    [...response.headers.entries()],

	body:       uint8_to_hex(new Uint8Array(await response.arrayBuffer()))

    };

async function perform_download(fetch_data) {

	const response = await fetch(fetch_data.url, fetch_data.init);

	return response_data_jsonifiable(response);

	return {error: error_data_jsonifiable(e)};

/*

 * uint8_to_hex is a separate function used in cryptographic functions and when

 * dealing with binary data.

 */

#EXPORT uint8_to_hex

function gen_nonce(length=16) {

      /^\s*(content-security-policy|x-webkit-csp|x-content-security-policy)\s*$/i;

/* Make it possible to serialize en Error object. */

function error_data_jsonifiable(error) {

    const jsonifiable = {};

    for (const property of ["name", "message", "fileName", "lineNumber"])

	jsonifiable[property] = error[property];

    return jsonifiable;

}

#EXPORT error_data_jsonifiable

#FROM common/misc.js    IMPORT error_data_jsonifiable

 * resolving to those. The use of promises helps us prevent multiple requests

 * for the same resource from starting concurrently.

	var result = await browser.runtime.sendMessage(["CORS_bypass", {url}]);

	    throw new Error("Couldn't communicate with background script.");

	return {error: error_data_jsonifiable(e)};

#FROM html/repo_query_cacher_client.js IMPORT indirect_fetch

	if (!this.is_ok)

	    return;

	try {

	    var response = await indirect_fetch(tab_id, url);

	} catch(e) {

	    return work.err(e, "Failure to communicate with repository :(");

	if (!work.is_ok)

	    return;

	try {

	    var json = await response.json();

	} catch(e) {

	    return work.err(e, "Repository's response is not valid JSON :(");

	if (!work.is_ok)

	    return;

	const result = haketilo_validator.validate(json, schema);

	    if (json.$schema && !reg.test(json.$schema)) {

	const scripts = item_type === "resource" && json.scripts;

	const files = json.source_copyright.concat(scripts || []);

	    for (const res_ref of Object.values(json.payloads || {}))

	    for (const res_ref of (json.dependencies || []))

	if (!db_def || db_def.version < json.version)

	    work.result.push({def: json, db_def});

	    const text = await response.text();

	    if (!work.is_ok)

		return;

#FROM html/repo_query_cacher_client.js IMPORT indirect_fetch

    const encoded_queried_url = encodeURIComponent(query_view.url);

    const query_url = `${repo_url}query?url=${encoded_queried_url}`;

    const query_results = async () => {

	try {

	    var response = await indirect_fetch(query_view.tab_id, query_url);

	} catch(e) {

	    console.error("Haketilo:", e);

	}

	try {

	    var json = await response.json();

	} catch(e) {

	    console.error("Haketilo:", e);

	}

	const result = haketilo_validator.validate(json, schema);

	    if (json.$schema && !reg.test(json.$schema))

	return json.mappings;

/**

 * This file is part of Haketilo.

 *

 * Function: Making requests to remote repositories through a response cache in

 *           operating in the content script of a browser tab.

 *

 * Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>

 *

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * As additional permission under GNU GPL version 3 section 7, you

 * may distribute forms of that code without the copy of the GNU

 * GPL normally required by section 4, provided you include this

 * license notice and, in case of non-source distribution, a URL

 * through which recipients can access the Corresponding Source.

 * If you modify file(s) with this exception, you may extend this

 * exception to your version of the file(s), but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * As a special exception to the GPL, any HTML file which merely

 * makes function calls to this code, and for that purpose

 * includes it by reference shall be deemed a separate work for

 * copyright law purposes. If you modify this code, you may extend

 * this exception to your version of the code, but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 *

 * I, Wojtek Kosior, thereby promise not to sue for violation of this file's

 * license. Although I request that you do not make use of this code in a

 * proprietary program, I am not going to enforce this in court.

 */

#FROM common/browser.js IMPORT browser

/*

 * This is a complementary function to error_data_jsonifiable() in

 * common/misc.js.

 */

function error_from_jsonifiable_data(jsonifiable) {

    const arg_props = ["message", "fileName", "lineNumber"];

    return new window[jsonifiable.name](...arg_props.map(p => jsonifiable[p]));

}

/*

 * Make it possible to recover a Response object. This is a complementary

 * function to response_data_jsonifiable() in background/CORS_bypass_server.js.

 */

function response_from_jsonifiable_data(jsonifiable) {

    const body = jsonifiable.body, body_len = body.length / 2;

    const body_buf = new Uint8Array(body_len);

    for (let i = 0; i < body_len; i++)

	body_buf[i] = parseInt(`0x${body.substring(i * 2, i * 2 + 2)}`);

    const init = {

	status:     jsonifiable.status,

	statusText: jsonifiable.statusText,

	headers:    new Headers(jsonifiable.headers)

    };

    return new Response(body_buf, init);

}

async function indirect_fetch(tab_id, url) {

    const msg = ["repo_query", url];

    const jsonifiable = await browser.tabs.sendMessage(tab_id, msg);

    if ("error" in jsonifiable)

	throw error_from_jsonifiable_data(jsonifiable);

    return response_from_jsonifiable_data(jsonifiable);

}

#EXPORT indirect_fetch

datas = {

    'resource':       'https://anotherdoma.in/resource/blocked/by/CORS.json',

    'nonexistent':    'https://nxdoma.in/resource.json',

    'invalid':        'w3csucks://invalid.url/',

    'redirected_ok':  'https://site.with.scripts.block.ed',

    'redirected_err': 'https://site.with.scripts.block.ed'

for name, url in [*datas.items()]:

    datas[name] = {'url': url}

datas['redirected_ok']['init']  = {'redirect': 'follow'}

datas['redirected_err']['init'] = {'redirect': 'error'}

const datas = %s;

    for (const [name, data] of Object.entries(datas)) {

        const sending = browser.runtime.sendMessage(["CORS_bypass", data]);

content_script = content_script % json.dumps(datas);

        for (const [name, data] of Object.entries(arguments[0])) {

            promises.push(fetch(data.url).then(ok_cb, err_cb));

        {**datas, 'sameorigin': './nonexistent_resource'})

    assert results == dict([*[(k, 'err') for k in datas.keys()],

    assert results['invalid']['error']['fileName'].endswith('background.js')

    assert type(results['invalid']['error']['lineNumber']) is int

    assert type(results['invalid']['error']['message']) is str

    assert results['invalid']['error']['name'] == 'TypeError'

    assert results['nonexistent']['statusText'] == 'Not Found'

    assert any([name.lower() == 'content-length'

                for name, value in results['nonexistent']['headers']])

    assert bytes.fromhex(results['nonexistent']['body']) == \

        b'Handler for this URL not found.'

    assert results['resource']['statusText'] == 'OK'

    assert any([name.lower() == 'content-length'

                for name, value in results['resource']['headers']])

    assert bytes.fromhex(results['resource']['body']) == b'{"some": "data"}'

    assert results['redirected_ok']['status'] == 200

    assert results['redirected_err']['error']['name'] == 'TypeError'

    execute_in_page(mock_cacher_code)

            window.mock_cacher_fetch = () => new Promise(cb => {});

            window.mock_cacher_fetch =

                () => {throw new Error("Something happened :o")};

            const response = new Response("", {status: 404});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            const response = new Response("sth", {status: 200});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            const newer_schema_url =

                "https://hydrilla.koszko.org/schemas/api_mapping_description-255.1.schema.json";

            const mocked_json_data = JSON.stringify({$schema: newer_schema_url});

            const response = new Response(mocked_json_data, {status: 200});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            window.mock_cacher_fetch = async function(...args) {

                const response = await fetch(...args);

                const json = await response.json();

                /* identifier is no longer a string as it should be. */

                json.identifier = 1234567;

                return new Response(JSON.stringify(json), {

                    status:     response.status,

                    statusText: response.statusText,

                    headers:    [...response.headers.entries()]

                });

            fetch = () => {throw new Error("some error");};

const old_sendMessage = browser.tabs.sendMessage;

    if (msg[0] === "page_info")

    else if (msg[0] === "repo_query")

        return old_sendMessage(tab_id, msg);

    else

tab_mock_js = mock_cacher_code + tab_mock_js

    execute_in_page(mock_cacher_code)

            '''

            window.mock_cacher_fetch =

                () => {throw new Error("Something happened :o")};

            ''')

            const response = new Response("", {status: 405});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            const response = new Response("sth", {status: 200});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            const newer_schema_url =

                "https://hydrilla.koszko.org/schemas/api_query_result-255.2.1.schema.json";

            const mocked_json_data = JSON.stringify({$schema: newer_schema_url});

            const response = new Response(mocked_json_data, {status: 200});

            window.mock_cacher_fetch = () => Promise.resolve(response);

            window.mock_cacher_fetch = async function(...args) {

                const response = await fetch(...args);

                const json = await response.json();

                /* $schema is no longer a string as it should be. */

                json.$schema = null;

                return new Response(JSON.stringify(json), {

                    status:     response.status,

                    statusText: response.statusText,

                    headers:    [...response.headers.entries()]

                });

            }

            'window.mock_cacher_fetch = () => new Promise(cb => {});'

            const schema_url =

                "https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json";

            const mocked_json_data =

                JSON.stringify({$schema: schema_url, mappings: []});

            const response = new Response(mocked_json_data, {status: 200});

            window.mock_cacher_fetch = () => Promise.resolve(response);

def test_repo_query_cacher_normal_use(driver):

    assert set(result.keys()) == {'status', 'statusText', 'headers', 'body'}

    counter_initial = json.loads(bytes.fromhex(result['body']))['counter']

        assert json.loads(bytes.fromhex(result['body'])) \

            == {'counter': counter_initial}

    assert json.loads(bytes.fromhex(result['body'])) \

        == {'counter': counter_initial + 1}

        assert result['error']['name'] == 'TypeError'

    assert set(result['error'].keys()) == \

        {'name', 'message', 'fileName', 'lineNumber'}

    assert result['error']['message'] == \

        "Couldn't communicate with background script."

"""

Some parts of code depend on content/repo_query_cacher.js and

background/CORS_bypass_server.js running in their appropriate contexts. This

snippet modifies the relevant browser.runtime.sendMessage function to perform

fetch(), bypassing the cacher.

"""

mock_cacher_code = '''{

const uint8_to_hex =

    array => [...array].map(b => ("0" + b.toString(16)).slice(-2)).join("");

const old_sendMessage = browser.tabs.sendMessage;

window.mock_cacher_fetch = fetch;

browser.tabs.sendMessage = async function(tab_id, msg) {

    if (msg[0] !== "repo_query")

        return old_sendMessage(tab_id, msg);

    /*

     * Use snapshotted fetch() under the name window.mock_cacher_fetch,

     * allow other test code to override it.

     */

    try {

        const response = await window.mock_cacher_fetch(msg[1]);

        const buf = await response.arrayBuffer();

        return {

            status:     response.status,

            statusText: response.statusText,

            headers:    [...response.headers.entries()],

            body:       uint8_to_hex(new Uint8Array(buf))

    } catch(e) {

        return {error: {name: e.name, message: e.message}};

    }

}

}'''