Haketilo

        copyright default_settings.json manifest.json

metafiles = build.sh configure Makefile.in process_html_file.sh README.txt \

	$(srcdir)/build.sh $* $(srcdir) $(UPDATE_URL)

	"$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}"

test/:

	mkdir $@

test/certs/: | test/

test/certs/%.key: | test/certs/

test/certs/rootCA.pem: test/certs/rootCA.key

test: test/certs/rootCA.pem test/certs/site.key $(default_target)-build.zip

	MOZ_HEADLESS=whatever $(PYTHON) -m pytest

test-environment: test/certs/rootCA.pem test/certs/site.key

	python3 -m test

test-environment-with-haketilo: test/certs/rootCA.pem test/certs/site.key \

	python3 -m test --load-haketilo

	rm -rf test/certs

	rm -rf $$(find . -name geckodriver.log)

	rm -rf $$(find . -type d -name __pycache__)

	rm -rf $$(find . -type d -name injected_scripts)

dist: $(extension_files) $(metafiles) icons/haketilo16.png

	test -d haketilo-$(version) || mkdir haketilo-$(version)

	for file in $(extension_files) $(metafiles) icons/; do \

	    cp -R "$(srcdir)"/$$file haketilo-$(version); \

	done

	tar cf haketilo-$(version).tar haketilo-$(version)

	gzip haketilo-$(version).tar

. ./shell_utils.sh

#!/usr/bin/env pytest

# This file is part of Haketilo

#

# Copyright (C) 2021, Wojtek Kosior

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the CC0 1.0 Universal License as published by

# the Creative Commons Corporation.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# CC0 1.0 Universal License for more details.

[pytest]

markers =

    ext_data: define a custom testing extension for `webextension` fixture.

    get_page: define a url the `driver` fixture should navigate the browser to.

    second_driver: tell `driver` fixture to spawn a separate browser instance fr this test.

# SPDX-License-Identifier: CC0-1.0

# Copyright (C) 2021 Wojtek Kosior

# SPDX-License-Identifier: AGPL-3.0-or-later

"""

Run a Firefox-type browser with WebDriver attached and Python console open

"""

# This file is part of Haketilo.

#

# Copyright (C) 2021 jahoti <jahoti@tilde.team>

# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU Affero General Public License as

# published by the Free Software Foundation, either version 3 of the

# License, or (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <https://www.gnu.org/licenses/>.

#

#

# I, Wojtek Kosior, thereby promise not to sue for violation of this

# file's license. Although I request that you do not make use of this code

# in a proprietary program, I am not going to enforce this in court.

import sys

import time

import code

from rlcompleter import Completer

import readline

from .server import do_an_internet

from .misc_constants import *

from .profiles import firefox_safe_mode

from .extension_crafting import get_extension_base_url

def fail(msg, error_code):

    print('Error:', msg)

    print('Usage:', sys.argv[0], '[--load-haketilo]', '[certificates_directory] [proxy_port]')

    sys.exit(error_code)

load_haketilo = False

argv_idx = 1

if len(sys.argv) > argv_idx and sys.argv[argv_idx] == '--load-haketilo':

    load_haketilo = True

    argv_idx += 1

certdir = Path(sys.argv[argv_idx]).resolve() if len(sys.argv) > argv_idx \

    else default_cert_dir

if not certdir.is_dir():

    fail('selected certificate directory does not exist.', 2)

argv_idx += 1

port = sys.argv[argv_idx] if len(sys.argv) > argv_idx \

    else str(default_proxy_port)

if not port.isnumeric():

    fail('port must be an integer.', 3)

httpd = do_an_internet(certdir, int(port))

driver = firefox_safe_mode(proxy_port=int(port))

if load_haketilo:

    driver.install_addon(str(here.parent / 'mozilla-build.zip'), temporary=True)

    driver.get(get_extension_base_url(driver) + 'html/settings.html')

print("You can now control the browser through 'driver' object")

# Here we enable readline-enhanced editing:

# https://stackoverflow.com/questions/35115208/is-there-any-way-to-combine-readline-rlcompleter-and-interactiveconsole-in-pytho#answer-35116399

readline.parse_and_bind('tab: complete');

console_locals = globals()

readline.set_completer(Completer(console_locals).complete)

code.InteractiveConsole(locals=globals()).interact()

driver.quit()

httpd.shutdown()

# SPDX-License-Identifier: GPL-3.0-or-later

"""

Common fixtures for Haketilo unit tests

"""

# This file is part of Haketilo.

#

# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <https://www.gnu.org/licenses/>.

#

# I, Wojtek Kosior, thereby promise not to sue for violation of this file's

# license. Although I request that you do not make use of this code in a

# proprietary program, I am not going to enforce this in court.

import pytest

from pathlib import Path

from tempfile import TemporaryDirectory

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from .profiles           import firefox_safe_mode

from .server             import do_an_internet

from .extension_crafting import make_extension

from .world_wide_library import start_serving_script, dump_scripts

from .misc_constants     import here

@pytest.fixture(scope="session")

def proxy():

    httpd = do_an_internet()

    yield httpd

    httpd.shutdown()

@pytest.fixture(scope="session")

def _driver(proxy):

    with firefox_safe_mode() as driver:

        yield driver

        driver.quit()

def close_all_but_one_window(driver):

    while len(driver.window_handles) > 1:

        driver.switch_to.window(driver.window_handles[-1])

        driver.close()

    driver.switch_to.window(driver.window_handles[0])

@pytest.fixture()

def driver(_driver, request):

    nav_target = request.node.get_closest_marker('get_page')

    nav_target = nav_target.args[0] if nav_target else 'about:blank'

    second_driver = request.node.get_closest_marker('second_driver')

    if second_driver:

        with firefox_safe_mode() as _driver:

            _driver.get(nav_target)

            yield _driver

            _driver.quit()

    else:

        close_all_but_one_window(_driver)

        _driver.get(nav_target)

        _driver.implicitly_wait(0)

        yield _driver

@pytest.fixture()

def webextension(driver, request):

    ext_data = request.node.get_closest_marker('ext_data')

    if ext_data is None:

        raise Exception('"webextension" fixture requires "ext_data" marker to be set')

    ext_data = ext_data.args[0].copy()

    navigate_to = ext_data.get('navigate_to')

    if navigate_to is not None:

        del ext_data['navigate_to']

    driver.get('https://gotmyowndoma.in/')

    ext_path = make_extension(Path(driver.firefox_profile.path), **ext_data)

    addon_id = driver.install_addon(str(ext_path), temporary=True)

    get_url = lambda d: d.execute_script('return window.ext_page_url;')

    ext_page_url = WebDriverWait(driver, 10).until(get_url)

    driver.get(ext_page_url)

    if navigate_to is not None:

        driver.get(driver.current_url.replace('testpage.html', navigate_to))

    yield

    # Unloading an extension might cause its windows to vanish. Make sure

    # there's at least one window navigated to some other page before

    # uninstalling the addon. Otherwise, we could be left with a windowless

    # browser :c

    driver.switch_to.window(driver.window_handles[-1])

    driver.get('https://gotmyowndoma.in/')

    driver.uninstall_addon(addon_id)

    ext_path.unlink()

@pytest.fixture()

def haketilo(driver):

    addon_id = driver.install_addon(str(here.parent / 'mozilla-build.zip'),

                                    temporary=True)

    yield

    driver.uninstall_addon(addon_id)

script_injector_script = '''\

/*

 * Selenium by default executes scripts in some weird one-time context. We want

 * separately-loaded scripts to be able to access global variables defined

 * before, including those declared with `const` or `let`. To achieve that, we

 * run our scripts by injecting them into the page with a <script> tag that runs

 * javascript served by our proxy. We use custom properties of the `window`

 * object to communicate with injected code.

 */

const inject = async () => {

    delete window.haketilo_selenium_return_value;

    delete window.haketilo_selenium_exception;

    window.returnval = val => window.haketilo_selenium_return_value = val;

    const injectee = document.createElement('script');

    injectee.src = arguments[0];

    injectee.type = "application/javascript";

    injectee.async = true;

    const prom = new Promise(cb => injectee.onload = cb);

    window.arguments = arguments[1];

    document.body.append(injectee);

    await prom;

    /*

     * To ease debugging, we want this script to signal all exceptions from the

     * injectee.

     */

    if (window.haketilo_selenium_exception !== false)

        throw ['haketilo_selenium_error',

               'Error in injected script! Check your geckodriver.log and ./injected_scripts/!'];

    return window.haketilo_selenium_return_value;

}

return inject();

'''

def _execute_in_page_context(driver, script, args):

    script = script + '\n;\nwindow.haketilo_selenium_exception = false;'

    script_url = start_serving_script(script)

    try:

        result = driver.execute_script(script_injector_script, script_url, args)

        if type(result) is list and len(result) == 2 and \

           result[0] == 'haketilo_selenium_error':

            raise Exception(result[1])

        return result

    except Exception as e:

        dump_scripts()

        raise e from None

# Some fixtures here just define functions that operate on driver. We should

# consider making them into webdriver wrapper class methods.

@pytest.fixture()

def execute_in_page(driver):

    def do_execute(script, *args):

        return _execute_in_page_context(driver, script, args)

    yield do_execute

@pytest.fixture()

def wait_elem_text(driver):

    def do_wait(id, text):

        WebDriverWait(driver, 10).until(

            EC.text_to_be_present_in_element((By.ID, id), text)

        )

    yield do_wait

<!DOCTYPE html>

<!--

    SPDX-License-Identifier: AGPL-3.0-or-later

    Sample testing page

    This file is part of Haketilo.

    Copyright (C) 2021 jahoti <jahoti@tilde.team>

    This program is free software: you can redistribute it and/or modify

    it under the terms of the GNU Affero General Public License as

    published by the Free Software Foundation, either version 3 of the

    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License

    along with this program.  If not, see <https://www.gnu.org/licenses/>.

  -->

<html>

	<head>

		<meta name=charset value="latin1">

		<title>Schrodinger's Document</title>

	</head>

	<body>

		A nice, simple page for testing.

		<script>

			document.write('<p><b>Or so you thought...</b></p>');

		</script>

	</body>

</html>

<!DOCTYPE html>

<!--

    SPDX-License-Identifier: AGPL-3.0-or-later

    Sample testing page to serve over HTTPS

    This file is part of Haketilo.

    Copyright (C) 2021 jahoti <jahoti@tilde.team>

    This program is free software: you can redistribute it and/or modify

    it under the terms of the GNU Affero General Public License as

    published by the Free Software Foundation, either version 3 of the

    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License

    along with this program.  If not, see <https://www.gnu.org/licenses/>.

  -->

<html>

	<head>

		<meta name="charset" value="latin1">

		<title>Schrodinger's Document</title>

	</head>

	<body>

		A nice, simple page for testing (using HTTPS).

		<script>

			document.write('<p><b>Or so you thought...</b></p>');

		</script>

	</body>

</html>

<!DOCTYPE html>

<!--

    SPDX-License-Identifier: CC0-1.0

    A testing page with various scripts that need to get blocked.

    This file is part of Haketilo.

    Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>

    This program is free software: you can redistribute it and/or modify

    it under the terms of the CC0 1.0 Universal License as published by

    the Creative Commons Corporation.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    CC0 1.0 Universal License for more details.

  -->

<html>

  <head>

    <meta name="charset" value="latin1">

    <script>

      window.__run = [...(window.__run || []), 'inline'];

    </script>

    <!-- the one below shall not execute even when blocking is off... -->

    <script type="application/json">

      window.__run = [...(window.__run || []), 'json'];

    </script>

  </head>

  <body>

    <button id="clickme1"

	    onclick="window.__run = [...(window.__run || []), 'on'];">

      Click Meee!

    </button>

    <a id="clickme2"

       href="javascript:window.__run = [...(window.__run || []), 'href'];void(0);">

      Click Meee!

    </a>

    <iframe src="javascript:void(window.parent.__run = [...(window.parent.__run || []), 'src']);">

    </iframe>

    <object data="javascript:window.__run = [...(window.__run || []), 'data'];">

    </object>

  </body>

</html>

{"schemaVersion":25,"addons":[{"id":"jid1-KtlZuoiikVfFew@jetpack","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/jid1-KtlZuoiikVfFew@jetpack"},{"id":"uBlock0@raymondhill.net","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/uBlock0@raymondhill.net.xpi"},{"id":"SubmitMe@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SubmitMe@0xbeef.coffee"},{"id":"FreeUSPS@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/FreeUSPS@0xbeef.coffee"},{"id":"tortm-browser-button@jeremybenthum","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tortm-browser-button@jeremybenthum"},{"id":"tprb.addon@searxes.danwin1210.me","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tprb.addon@searxes.danwin1210.me"},{"id":"SimpleSumOfUs@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SimpleSumOfUs@0xbeef.coffee"}]}

# SPDX-License-Identifier: GPL-3.0-or-later

"""

Making temporary WebExtensions for use in the test suite

"""

# This file is part of Haketilo.

#

# Copyright (C) 2021, 2022 Wojtek Kosior <koszko@koszko.org>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <https://www.gnu.org/licenses/>.

#

# I, Wojtek Kosior, thereby promise not to sue for violation of this file's

# license. Although I request that you do not make use of this code in a

# proprietary program, I am not going to enforce this in court.

import json

import zipfile

import re

import shutil

import subprocess

from pathlib import Path

from uuid import uuid4

from tempfile import TemporaryDirectory

from selenium.webdriver.support.ui import WebDriverWait

from selenium.common.exceptions import NoSuchElementException

from .misc_constants import *

class ManifestTemplateValueToFill:

    pass

def manifest_template():

    return {

        'manifest_version': 2,

        'name': 'Haketilo test extension',

        'version': '1.0',

        'applications': {

	    'gecko': {

	        'id': ManifestTemplateValueToFill(),

	        'strict_min_version': '60.0'

	    }

        },

        'permissions': [

	    'contextMenus',

	    'webRequest',

	    'webRequestBlocking',

	    'activeTab',

	    'notifications',

	    'sessions',

	    'storage',

	    'tabs',

	    '<all_urls>',

	    'unlimitedStorage'

        ],

        'content_security_policy': "object-src 'none'; script-src 'self' https://serve.scrip.ts;",

        'web_accessible_resources': ['testpage.html'],

        'options_ui': {

	    'page': 'testpage.html',

	    'open_in_tab': True

        },

        'background': {

	    'persistent': True,

	    'scripts': ['__open_test_page.js', 'background.js']

        },

        'content_scripts': [

	    {

	        'run_at': 'document_start',

	        'matches': ['<all_urls>'],

	        'match_about_blank': True,

	        'all_frames': True,

	        'js': ['content.js']

	    }

        ]

    }

class ExtraHTML:

    def __init__(self, html_path, append={}, wrap_into_htmldoc=True):

        self.html_path = html_path

        self.append = append

        self.wrap_into_htmldoc = wrap_into_htmldoc

    def add_to_xpi(self, xpi, tmpdir=None):

        if tmpdir is None:

            with TemporaryDirectory() as tmpdir:

                return self.add_to_xpi(xpi, tmpdir)

        append_flags = []

        for filename, code in self.append.items():

            append_flags.extend(['-A', f'{filename}:{code}'])

        awk = subprocess.run(

            ['awk', '-f', awk_script_name, '--', *unit_test_defines,

             *append_flags, '-H', self.html_path, '--write-js-deps',

             '--output=files-to-copy', f'--output-dir={tmpdir}'],

            stdout=subprocess.PIPE, cwd=script_root, check=True

        )

        for path in filter(None, awk.stdout.decode().split('\n')):

            xpi.write(script_root / path, path)

        tmpdir = Path(tmpdir)

        for path in tmpdir.rglob('*'):

            relpath = str(path.relative_to(tmpdir))

            if not path.is_dir() and relpath != self.html_path:

                xpi.write(path, relpath)

        with open(tmpdir / self.html_path, 'rt') as html_file:

            html = html_file.read()

            if self.wrap_into_htmldoc:

                html = f'<!DOCTYPE html><html><body>{html}</body></html>'

            xpi.writestr(self.html_path, html)

default_background_script = ''

default_content_script = ''

default_test_page = '''

<!DOCTYPE html>

<html>

  <head>

    <title>Extension's options page for testing</title>

  </head>

  <body>

    <h1>Extension's options page for testing</h1>

  </body>

</html>

'''

open_test_page_script = '''(() => {

const page_url = browser.runtime.getURL("testpage.html");

const execute_details = {

    code: `window.wrappedJSObject.ext_page_url=${JSON.stringify(page_url)};`

};

browser.tabs.query({currentWindow: true, active: true})

    .then(t => browser.tabs.executeScript(t.id, execute_details));

})();'''

def make_extension(destination_dir,

                   background_script=default_background_script,

                   content_script=default_content_script,

                   test_page=default_test_page,

                   extra_files={}, extra_html=[]):

    if not hasattr(extra_html, '__iter__'):

        extra_html = [extra_html]

    manifest = manifest_template()

    extension_id = '{%s}' % uuid4()

    manifest['applications']['gecko']['id'] = extension_id

    files = {

        'manifest.json'      : json.dumps(manifest),

        '__open_test_page.js': open_test_page_script,

        'background.js'      : background_script,

        'content.js'         : content_script,

        'testpage.html'      : test_page,

        **extra_files

    }

    destination_path = destination_dir / f'{extension_id}.xpi'

    with zipfile.ZipFile(destination_path, 'x') as xpi:

        for filename, contents in files.items():

            if hasattr(contents, '__call__'):

                contents = contents()

            xpi.writestr(filename, contents)

        for html in extra_html:

            html.add_to_xpi(xpi)

    return destination_path

extract_base_url_re = re.compile(r'^(.*)manifest.json$')

def get_extension_base_url(driver):

    """

    Extension's internall UUID is not directly exposed in Selenium. Instead, we

    can navigate to about:debugging and inspect the manifest URL present there

    to get the base url like:

        moz-extension://b225c78f-d108-4caa-8406-f38b37d8dee5/

    which can then be used to navigate to extension-bundled pages.

    """

    # For newer Firefoxes

    driver.get('about:debugging#/runtime/this-firefox')

    def get_manifest_link_newer_ff(driver):

        try:

            return driver.find_element_by_class_name('qa-manifest-url')

        except NoSuchElementException:

            pass

        try:

            details = driver.find_element_by_class_name('error-page-details')

        except NoSuchElementException:

            return False

        if '#/runtime/this-firefox' in details.text:

            return "not_newer_ff"

    manifest_link = WebDriverWait(driver, 10).until(get_manifest_link_newer_ff)

    if manifest_link == "not_newer_ff":

        driver.get("about:debugging#addons")

        driver.implicitly_wait(10)

        manifest_link = driver.find_element_by_class_name('manifest-url')

        driver.implicitly_wait(0)

    manifest_url = manifest_link.get_attribute('href')

    return extract_base_url_re.match(manifest_url).group(1)