Haketilo

/**

 * This file is part of Haketilo.

 *

 * Function: Allow content scripts to query IndexedDB through messages to

 *           background script.

 *

 * Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>

 *

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * As additional permission under GNU GPL version 3 section 7, you

 * may distribute forms of that code without the copy of the GNU

 * GPL normally required by section 4, provided you include this

 * license notice and, in case of non-source distribution, a URL

 * through which recipients can access the Corresponding Source.

 * If you modify file(s) with this exception, you may extend this

 * exception to your version of the file(s), but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * As a special exception to the GPL, any HTML file which merely

 * makes function calls to this code, and for that purpose

 * includes it by reference shall be deemed a separate work for

 * copyright law purposes. If you modify this code, you may extend

 * this exception to your version of the code, but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 *

 * I, Wojtek Kosior, thereby promise not to sue for violation of this file's

 * license. Although I request that you do not make use of this code in a

 * proprietary program, I am not going to enforce this in court.

 */

#IMPORT common/indexeddb.js AS haketilodb

#FROM common/browser.js IMPORT browser

async function get_resource_files(getting, id) {

    if (getting.defs_by_res_id.has(id))

	return;

    getting.defs_by_res_id.set(id, null);

    const definition = await haketilodb.idb_get(getting.tx, "resource", id);

    if (!definition)

	throw {haketilo_error_type: "missing", id};

    getting.defs_by_res_id.set(id, definition);

    const file_proms = (definition.scripts || [])

	  .map(s => haketilodb.idb_get(getting.tx, "files", s.hash_key));

    const deps_proms = (definition.dependencies || [])

	  .map(dep_id => get_resource_files(getting, dep_id));

    const files = (await Promise.all(file_proms)).map(f => f.contents);

    getting.files_by_res_id.set(id, files);

    await Promise.all(deps_proms);

}

function get_files_list(defs_by_res_id, files_by_res_id, root_id) {

    const processed = new Set(), to_process = [["start", root_id]],

	  trace = new Set(), files = [];

    while (to_process.length > 0) {

	const [what, id] = to_process.pop();

	if (what === "end") {

	    trace.delete(id);

	    files.push(...files_by_res_id.get(id));

	    continue;

	}

	if (trace.has(id))

	    throw {haketilo_error_type: "circular", id};

	if (processed.has(id))

	    continue;

	trace.add(id);

	to_process.push(["end", id]);

	processed.add(id);

	const ds = (defs_by_res_id.get(id).dependencies || []).reverse();

	ds.forEach(dep_id => to_process.push(["start", dep_id]));

    }

    return files;

}

async function send_resource_files(root_resource_id, send_cb) {

    const db = await haketilodb.get();

    const getting = {

	defs_by_res_id:  new Map(),

	files_by_res_id: new Map(),

	tx:              db.transaction(["files", "resource"])

    };

    let prom_cbs, prom = new Promise((...cbs) => prom_cbs = cbs);

    getting.tx.onerror = e => prom_cbs[1]({haketilo_error_type: "db", e});

    get_resource_files(getting, root_resource_id, new Set()).then(...prom_cbs);

    try {

	await prom;

	const files = get_files_list(

	    getting.defs_by_res_id,

	    getting.files_by_res_id,

	    root_resource_id

	);

	var to_send = {files};

    } catch(e) {

	if (typeof e === "object" && "haketilo_error_type" in e) {

	    if (e.haketilo_error_type === "db") {

		console.error(e.e);

		delete e.e;

	    }

	    var to_send = {error: e};

	} else {

	    console.error(e);

	    var to_send = {error: {haketilo_error_type: "other"}};

	}

    }

    send_cb(to_send);

}

function on_indexeddb_files_request([type, resource_id], sender, respond_cb) {

    if (type !== "indexeddb_files")

	return;

    send_resource_files(resource_id, respond_cb);

    return true;

}

function start() {

    browser.runtime.onMessage.addListener(on_indexeddb_files_request);

}

#EXPORT start

# SPDX-License-Identifier: CC0-1.0

"""

Haketilo unit tests - serving indexeddb resource script files to content scripts

"""

# This file is part of Haketilo

#

# Copyright (C) 2021,2022 Wojtek Kosior <koszko@koszko.org>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the CC0 1.0 Universal License as published by

# the Creative Commons Corporation.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# CC0 1.0 Universal License for more details.

import pytest

import copy

from uuid import uuid4

from selenium.webdriver.support.ui import WebDriverWait

from ..script_loader import load_script

from .utils import *

"""

How many test resources we're going to have.

"""

count = 15

sample_files_list = [(f'file_{n}_{i}', f'contents {n} {i}')

                       for n in range(count) for i in range(2)]

sample_files = dict(sample_files_list)

sample_files, sample_files_by_hash = make_sample_files(sample_files)

def make_sample_resource_with_deps(n):

    resource = make_sample_resource(with_files=False)

    resource['identifier'] = f'res-{n}'

    resource['dependencies'] = [f'res-{m}'

                                for m in range(max(n - 4, 0), n)]

    resource['scripts'] = [sample_file_ref(f'file_{n}_{i}', sample_files)

                           for i in range(2)]

    return resource

resources = [make_sample_resource_with_deps(n) for n in range(count)]

sample_data = {

    'resources': sample_data_dict(resources),

    'mapping': {},

    'files': sample_files_by_hash

}

def prepare_test_page(initial_indexeddb_data, execute_in_page):

    js = load_script('background/indexeddb_files_server.js',

                     code_to_add='#IMPORT common/broadcast.js')

    execute_in_page(js)

    mock_broadcast(execute_in_page)

    clear_indexeddb(execute_in_page)

    execute_in_page(

        '''

        let registered_listener;

        const new_addListener = cb => registered_listener = cb;

        browser = {runtime: {onMessage: {addListener: new_addListener}}};

        haketilodb.save_items(arguments[0]);

        start();

        ''',

        initial_indexeddb_data)

@pytest.mark.get_page('https://gotmyowndoma.in')

def test_indexeddb_files_server_normal_usage(driver, execute_in_page):

    """

    Test querying resource files (with resource dependency resolution)

    from IndexedDB and serving them in messages to content scripts.

    """

    prepare_test_page(sample_data, execute_in_page)

    # Verify other types of messages are ignored.

    function_returned_value = execute_in_page(

        '''

        returnval(registered_listener(["???"], {},

                  () => location.reload()));

        ''')

    assert function_returned_value == None

    # Verify single resource's files get properly resolved.

    function_returned_value = execute_in_page(

        '''

        var result_cb, contents_prom = new Promise(cb => result_cb = cb);

        returnval(registered_listener(["indexeddb_files", "res-0"],

                                      {}, result_cb));

        ''')

    assert function_returned_value == True

    assert execute_in_page('returnval(contents_prom);') == \

        {'files': [tuple[1] for tuple in sample_files_list[0:2]]}

    # Verify multiple resources' files get properly resolved.

    function_returned_value = execute_in_page(

        '''

        var result_cb, contents_prom = new Promise(cb => result_cb = cb);

        returnval(registered_listener(["indexeddb_files", arguments[0]],

                                      {}, result_cb));

        ''',

        f'res-{count - 1}')

    assert function_returned_value == True

    assert execute_in_page('returnval(contents_prom);') == \

        {'files': [tuple[1] for tuple in sample_files_list]}

@pytest.mark.get_page('https://gotmyowndoma.in')

@pytest.mark.parametrize('error', [

    'missing',

    'circular',

    'db',

    'other'

])

def test_indexeddb_files_server_errors(driver, execute_in_page, error):

    """

    Test reporting of errors when querying resource files (with resource

    dependency resolution) from IndexedDB and serving them in messages to

    content scripts.

    """

    sample_data_copy = copy.deepcopy(sample_data)

    if error == 'missing':

        del sample_data_copy['resources']['res-3']

    elif error == 'circular':

        res3_defs = sample_data_copy['resources']['res-3'].values()

        next(iter(res3_defs))['dependencies'].append('res-8')

    prepare_test_page(sample_data_copy, execute_in_page)

    if error == 'db':

        execute_in_page('haketilodb.idb_get = t => t.onerror("oooops");')

    elif error == 'other':

        execute_in_page('haketilodb.idb_get = () => {throw "oooops"};')

    response = execute_in_page(

        '''

        var result_cb, contents_prom = new Promise(cb => result_cb = cb);

        registered_listener(["indexeddb_files", arguments[0]],

                            {}, result_cb);

        returnval(contents_prom);

        ''',

        f'res-{count - 1}')

    assert response['error']['haketilo_error_type'] == error

    if error == 'missing':

        assert response['error']['id'] == 'res-3'

    elif error == 'circular':

        assert response['error']['id'] in ('res-3', 'res-8')

    elif error not in ('db', 'other'):

        raise Exception('made a typo in test function params?')

def make_sample_files(names_contents):

    """

    Take a dict mapping file names to file contents. Return, as a tuple, dicts

    mapping file names to file objects (dicts) and file hash keys to file

    contents.

    """

    sample_files = dict([(name, sample_file(contents))

                         for name, contents in names_contents.items()])

    sample_files_by_hash = dict([[file['hash_key'], file['contents']]

                                 for file in sample_files.values()])

    return sample_files, sample_files_by_hash

sample_files, sample_files_by_hash = make_sample_files({

    'report.spdx':              '<!-- dummy report -->',

    'LICENSES/somelicense.txt': 'Permission is granted...',

    'LICENSES/CC0-1.0.txt':     'Dummy Commons...',

    'hello.js':                 'console.log("uńićódę hello!");\n',

    'bye.js':                   'console.log("bye!");\n',

    'combined.js':              'console.log("hello!\\nbye!");\n',

    'README.md':                '# Python Frobnicator\n...'

})

def sample_file_ref(file_name, sample_files_dict=sample_files):

    """

    Return a dictionary suitable for using as file reference in resource/mapping

    definition.

    """

    return {

        'file': file_name,

        'hash_key': sample_files_dict[file_name]['hash_key']

    }

def make_sample_mapping(with_files=True):

    """

    Procude a sample mapping definition that can be dumped to JSON and put into

    Haketilo's IndexedDB.

    """

        ] if with_files else [],

def make_sample_resource(with_files=True):

    """

    Procude a sample resource definition that can be dumped to JSON and put into

    Haketilo's IndexedDB.

    """

        ] if with_files else [],

        ] if with_files else []

    Some IndexedDB functions expect saved items to be provided in a nested dict

def mock_broadcast(execute_in_page):

    """

    Make all broadcast operations no-ops (broadcast must be imported).

    """

    execute_in_page(

        'Object.keys(broadcast).forEach(k => broadcast[k] = () => {});'

    )