Haketilo

	        haketilo-$(version).tar exports_init.js

 * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>

 * Copyright (C) 2021 Jahoti <jahoti@envs.net>

 * IMPORT initial_data

    /* Below we add sample settings to the extension. */

    for (let setting of initial_data) {

    ./write_exports_init.sh "$BROWSER" "$BUILDDIR" default_settings.json

/**

 * This file is part of Haketilo.

 *

 * Function: Operations on resources and mappings.

 *

 * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>

 *

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * As additional permission under GNU GPL version 3 section 7, you

 * may distribute forms of that code without the copy of the GNU

 * GPL normally required by section 4, provided you include this

 * license notice and, in case of non-source distribution, a URL

 * through which recipients can access the Corresponding Source.

 * If you modify file(s) with this exception, you may extend this

 * exception to your version of the file(s), but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * As a special exception to the GPL, any HTML file which merely

 * makes function calls to this code, and for that purpose

 * includes it by reference shall be deemed a separate work for

 * copyright law purposes. If you modify this code, you may extend

 * this exception to your version of the code, but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 *

 * I, Wojtek Kosior, thereby promise not to sue for violation of this file's

 * license. Although I request that you do not make use this code in a

 * proprietary program, I am not going to enforce this in court.

 */

/*

 * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"

 * return [4, 6, 13, 0].

 */

const parse_version = ver_str => ver_str.split(".").map(parseInt);

/*

 * ver is an array of integers. rev is an optional integer. Produce string

 * representation of version (optionally with revision number), like:

 *     1.2.3-5

 * No version normalization is performed.

 */

const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : "");

/* vers should be an array of comparable values. Return the greatest one. */

const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2);

/*

 * versioned_item should be a dict with keys being version strings and values

 * being definitions of the respective versions of a single resource/mapping.

 * Example:

 *     {

 *         "1": {

 *             version: [1]//,

 *             // more stuff

 *         },

 *         "1.1": {

 *             version: [1, 1]//,

 *             // more stuff

 *         }

 *      }

 *

 * Returns the definition with the highest version.

 */

function get_newest_version(versioned_item)

{

    const best_ver = max(Object.keys(versioned_item).map(parse_version));

    return versioned_item[version_string(best_ver)];

}

/*

 * item is a definition of a resource or mapping. Yield all file references

 * (objects with `file` and `sha256` properties) this definition has.

 */

function* get_used_files(item)

{

    for (const file of item.source_copyright)

	yield file;

    if (item.type === "resource") {

	for (const file of item.scripts || [])

	    yield file;

    }

}

const entities = {

    get_newest: get_newest_version,

    get_files:  get_used_files

};

/*

 * EXPORTS_START

 * EXPORT entities

 * EXPORTS_END

 */

/*

 * Note: the functions below were overeagerly written and are not used now but

 * might prove useful to once we add more functionalities and are hence kept...

 */

/*

 * Clone recursively all objects. Leave other items (arrays, strings) untouched.

 */

function deep_object_copy(object)

{

    const orig = {object};

    const result = {};

    const to_copy = [[orig, {}]];

    while (to_copy.length > 0) {

	const [object, copy] = to_copy.pop();

	for (const [key, value] of Object.entries(object)) {

	    copy[key] = value;

	    if (typeof value === "object" && !Array.isArray(value)) {

		const value_copy = {};

		to_copy.push([value, value_copy]);

		copy[key] = value_copy;

	    }

	}

    }

    return result.orig;

}

/* helper function for normalize_version() */

const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc];

/*

 * ver is an array of integers. Strip right-most zeroes from ver.

 *

 * Returns a *new* array. Doesn't modify its argument.

 */

const normalize_version = ver => Array.reduceRight(ver, version_reductor, []);

/**

 * This file is part of Haketilo.

 *

 * Function: Facilitate use of IndexedDB within Haketilo.

 *

 * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>

 *

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * As additional permission under GNU GPL version 3 section 7, you

 * may distribute forms of that code without the copy of the GNU

 * GPL normally required by section 4, provided you include this

 * license notice and, in case of non-source distribution, a URL

 * through which recipients can access the Corresponding Source.

 * If you modify file(s) with this exception, you may extend this

 * exception to your version of the file(s), but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * As a special exception to the GPL, any HTML file which merely

 * makes function calls to this code, and for that purpose

 * includes it by reference shall be deemed a separate work for

 * copyright law purposes. If you modify this code, you may extend

 * this exception to your version of the code, but you are not

 * obligated to do so. If you do not wish to do so, delete this

 * exception statement from your version.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 *

 * I, Wojtek Kosior, thereby promise not to sue for violation of this file's

 * license. Although I request that you do not make use this code in a

 * proprietary program, I am not going to enforce this in court.

 */

/*

 * IMPORTS_START

 * IMPORT initial_data

 * IMPORT entities

 * IMPORTS_END

 */

/* Update when changes are made to database schema. Must have 3 elements */

const db_version = [1, 0, 0];

const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i];

const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1];

const stores = 	[

    ["files",     {keyPath: "sha256"}],

    ["file_uses", {keyPath: "sha256"}],

    ["resources", {keyPath: "identifier"}],

    ["mappings",  {keyPath: "identifier"}]

];

let db = null;

/* Generate a Promise that resolves when an IndexedDB request succeeds. */

async function wait_request(idb_request)

{

    let resolve, reject;

    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);

    [idb_request.onsuccess, idb_request.onerror] = [resolve, reject];

    return waiter;

}

/* asynchronous wrapper for IDBObjectStore's get() method. */

async function idb_get(transaction, store_name, key)

{

    const req = transaction.objectStore(store_name).get(key);

    return (await wait_request(req)).target.result;

}

/* asynchronous wrapper for IDBObjectStore's put() method. */

async function idb_put(transaction, store_name, object)

{

    return wait_request(transaction.objectStore(store_name).put(object));

}

/* asynchronous wrapper for IDBObjectStore's delete() method. */

async function idb_del(transaction, store_name, key)

{

    return wait_request(transaction.objectStore(store_name).delete(key));

}

/* Open haketilo database, asynchronously return an IDBDatabase object. */

async function get_db(initialization_data=initial_data)

{

    if (db)

	return db;

    let resolve, reject;

    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);

    const request = indexedDB.open("haketilo", version_nr(db_version));

    request.onsuccess       = resolve;

    request.onerror         = ev => reject("db error: " + ev.target.errorCode);

    request.onupgradeneeded = resolve;

    const event = await waiter;

    const opened_db = event.target.result;

    if (event instanceof IDBVersionChangeEvent) {

	/*

	 * When we move to a new database schema, we will add upgrade logic

	 * here.

	 */

	if (event.oldVersion > 0)

	    throw "bad db version: " + event.oldVersion;

	let store;

	for (const [store_name, key_mode] of stores)

	    store = opened_db.createObjectStore(store_name, key_mode);

	await new Promise(resolve => store.transaction.oncomplete = resolve);

	save_items(db, initialization_data);

    }

    db = opened_db;

    return db;

}

/*

 * How a sample data argument to the function below might look like:

 *

 * data = {

 *     resources: {

 *         "resource1": {

 *             "1": {

 *                 // some stuff

 *             },

 *             "1.1": {

 *                 // some stuff

 *             }

 *         },

 *         "resource2": {

 *             "0.4.3": {

 *                 // some stuff

 *             }

 *         },

 *     },

 *     mappings: {

 *         "mapping1": {

 *             "2": {

 *                 // some stuff

 *             }

 *         },

 *         "mapping2": {

 *             "0.1": {

 *                 // some stuff

 *             }

 *         },

 *     },

 *     files: {

 *         "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n",

 *         "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");"

 *     }

 * }

 */

async function save_items(db, data)

{

    const files = data.files;

    const resources =

	  Object.values(data.resources || []).map(entities.get_newest);

    const mappings =

	  Object.values(data.mappings || []).map(entities.get_newest);

    resources.concat(mappings).forEach(i => save_item(i, data.files, db));

}

/* helper function of save_item() */

async function get_file_uses(transaction, file_uses_sha256, file_ref)

{

    let uses = file_uses_sha256[file_ref.sha256];

    if (uses === undefined) {

	uses = await idb_get(transaction, "file_uses", file_ref.sha256);

	if (uses)

	    [uses.new, uses.initial] = [false, uses.uses];

	else

	    uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0};

	file_uses_sha256[file_ref.sha256] = uses;

    }

    return uses;

}

/*

 * Save given definition of a resource/mapping to IndexedDB. If the definition

 * (passed as `item`) references files that are not already present in

 * IndexedDB, those files should be present as values of the `files_sha256`

 * object with keys being their sha256 sums.

 */

async function save_item(item, files_sha256, db)

{

    const store_name = {resource: "resources", mapping: "mappings"}[item.type];

    const transaction =

	  db.transaction([store_name, "files", "file_uses"], "readwrite");

    let resolve, reject;

    const result = new Promise((...cbs) => [resolve, reject] = cbs);

    transaction.oncomplete = resolve;

    transaction.onerror = reject;

    const uses_sha256 = {};

    for (const file_ref of entities.get_files(item))

	(await get_file_uses(transaction, uses_sha256, file_ref)).uses++;

    const old_item = await idb_get(transaction, store_name, item.identifier);

    if (old_item !== undefined) {

	for (const file_ref of entities.get_files(old_item))

	    (await get_file_uses(transaction, uses_sha256, file_ref)).uses--;

    }

    for (const uses of Object.values(uses_sha256)) {

	if (uses.uses < 0)

	    console.error("internal error: uses < 0 for file " + uses.sha256);

	const [is_new, initial_uses] = [uses.new, uses.initial];

	delete uses.new;

	delete uses.initial;

	if (uses.uses < 1) {

	    if (!is_new) {

		idb_del(transaction, "file_uses", uses.sha256);

		idb_del(transaction, "files",     uses.sha256);

	    }

	    continue;

	}

	if (uses.uses === initial_uses)

	    continue;

	const file = files_sha256[uses.sha256];

	if (file === undefined)

	    throw "file not present: " + uses.sha256;

	idb_put(transaction, "files", {sha256: uses.sha256, contents: file});

	idb_put(transaction, "file_uses", uses);

    }

    idb_put(transaction, store_name, item);

    return result;

}

const haketilodb = {

    get:       get_db,

    save_item: save_item

};

/*

 * EXPORTS_START

 * EXPORT haketilodb

 * EXPORT idb_get

 * EXPORT idb_put

 * EXPORT idb_del

 * EXPORTS_END

 */

    provides["browser"]      = "exports_init.js"

    provides["is_chrome"]    = "exports_init.js"

    provides["is_mozilla"]   = "exports_init.js"

    provides["initial_data"] = "exports_init.js"

        if not (script_root / 'exports_init.js').exists():

            subprocess.run([str(script_root / 'write_exports_init.sh'),

                            'mozilla', '.', 'default_settings.json'],

                           cwd=script_root, check=True)

        with open(script_root / 'exports_init.js') as script:

# SPDX-License-Identifier: CC0-1.0

"""

Haketilo unit tests - IndexedDB access

"""

# This file is part of Haketilo

#

# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the CC0 1.0 Universal License as published by

# the Creative Commons Corporation.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# CC0 1.0 Universal License for more details.

import pytest

from hashlib import sha256

from ..script_loader import load_script

@pytest.fixture(scope="session")

def indexeddb_code():

    yield load_script('common/indexeddb.js', ['common'])

def sample_file(contents):

    return {

        'sha256': sha256(contents.encode()).digest().hex(),

        contents: contents

    }

sample_files = {

    'report.spdx':              sample_file('<!-- dummy report -->'),

    'LICENSES/somelicense.txt': sample_file('Permission is granted...'),

    'hello.js':                 sample_file('console.log("hello!");\n'),

    'bye.js':                   sample_file('console.log("bye!");\n'),

    'README.md':                sample_file('# Python Frobnicator\n...')

}

sample_files_sha256 = \

    dict([[file['sha256'], file] for file in sample_files.values()])

def file_ref(file_name):

    return {'file': file_name, 'sha256': sample_files[file_name]['sha256']}

def test_save_item(execute_in_page, indexeddb_code):

    """

    indexeddb.js facilitates operating on Haketilo's internal database.

    Verify database operations work properly.

    """

    execute_in_page(indexeddb_code, page='https://gotmyowndoma.in')

    # Don't use Haketilo's default initial data.

    execute_in_page(

        '''{

        const _get_db = haketilodb.get;

        get_db = () => _get_db({});

        haketilodb.get = get_db;

        }'''

    )

    # Start with no database.

    execute_in_page(

        '''{

        async function delete_db() {

            let resolve;

            const result = new Promise(_resolve => resolve = _resolve);

            const request = indexedDB.deleteDatabase("haketilo");

            [request.onsuccess, request.onerror] = [resolve, resolve];

            await result;

        }

        returnval(delete_db());

        }'''

    )

    # Facilitate retrieving all IndexedDB contents.

    execute_in_page(

        '''

        async function get_database_contents(promise=Promise.resolve())

        {

            if (promise)

                await promise;

            const db = await haketilodb.get();

            const transaction = db.transaction(db.objectStoreNames);

            const store_names_reqs = [...db.objectStoreNames]

                .map(sn => [sn, transaction.objectStore(sn).getAll()])

            const promises = store_names_reqs

                .map(([_, req]) => wait_request(req));

            await Promise.all(promises);

            const result = {};

            store_names_reqs.forEach(([sn, req]) => result[sn] = req.result);

            return result;

        }

        ''')

    # Sample resource definition. It'd normally contain more fields but here

    # we use a simplified version.

    sample_item = {

        'source_copyright': [

            file_ref('report.spdx'),

            file_ref('LICENSES/somelicense.txt')

        ],

        'type': 'resource',

        'identifier': 'helloapple',

        'scripts': [file_ref('hello.js'), file_ref('bye.js')],

        'type': 'resource'

    }

    next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True

    database_contents = execute_in_page(

        '''{

        const prom = haketilodb.get().then(db => save_item(...arguments, db));

        returnval(get_database_contents(prom));

        }''',

        sample_item, sample_files_sha256)

    assert len(database_contents['files']) == 4

    assert all([sample_files_sha256[file['sha256']] == file['contents']

                for file in database_contents['files']])

    assert all([len(file) == 2 for file in database_contents['files']])

    assert len(database_contents['file_uses']) == 4

    assert all([uses['uses'] == 1 for uses in database_contents['file_uses']])

    assert set([uses['sha256'] for uses in database_contents['file_uses']]) \

        == set([file['sha256'] for file in database_contents['files']])

    assert database_contents['mappings'] == []

    assert database_contents['resources'] == [sample_item]

#!/bin/sh

# This file is part of Haketilo

#

# Copyright (C) 2021, Wojtek Kosior

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the CC0 1.0 Universal License as published by

# the Creative Commons Corporation.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# CC0 1.0 Universal License for more details.

set -e

BROWSER="$1"

BUILDDIR="$2"

SETTINGS="$3"

if [ "chromium" = "$BROWSER" ]; then

    cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js

else

    cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js

fi

printf 'window.haketilo_exports.initial_data = %s;\n' "$(cat "$SETTINGS")" \

       >> "$BUILDDIR"/exports_init.js