Project

General

Profile

« Previous | Next » 

Revision 22c722bf

Added by koszko about 1 year ago

  • ID 22c722bf59e59246f47491c7229b17f9ef783614
  • Parent 6bc04f80

New upstream version 1.1~beta1

View differences:

MANIFEST.in
4 4
#
5 5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6 6

  
7
include src/hydrilla/schemas/*.schema.json*
7
include src/hydrilla/schemas/*/*.schema.json*
8 8
include src/hydrilla/builder/locales/*/LC_MESSAGES/hydrilla-messages.po
9 9
include tests/source-package-example/*
10 10
include tests/source-package-example/LICENSES/*
PKG-INFO
1 1
Metadata-Version: 2.1
2 2
Name: hydrilla.builder
3
Version: 1.0
3
Version: 1.1b1
4 4
Summary: Hydrilla package builder
5 5
Home-page: https://git.koszko.org/hydrilla-builder
6 6
Author: Wojtek Kosior
......
24 24
        
25 25
        ### Build
26 26
        
27
        * build (a PEP517 package builder)
27 28
        * setuptools
28 29
        * wheel
29 30
        * setuptools_scm
30
        * babel
31
        * babel (Python library)
31 32
        
32 33
        ### Test
33 34
        
34 35
        * pytest
35
        * reuse
36 36
        
37
        ## Building & testing
37
        ## Building & testing & installation from wheel
38 38
        
39
        Build and test processed are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
39
        Build, test and installation processes are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
40 40
        
41 41
        ## Running
42 42
        
......
50 50
        ```
51 51
        
52 52
        You might as well like to run from sources, without installation:
53
        
54
        ``` shell
53 55
        mkdir /tmp/bananowarzez/
54 56
        ./setup.py compile_catalog # generate the necessary .po files
55 57
        PYTHONPATH=src python3 -m hydrilla.builder -s src/test/source-package-example/ \
......
86 88
Requires-Python: >=3.7
87 89
Description-Content-Type: text/markdown
88 90
Provides-Extra: setup
91
Provides-Extra: spdx
89 92
Provides-Extra: test
README.md
15 15

  
16 16
### Build
17 17

  
18
* build (a PEP517 package builder)
18 19
* setuptools
19 20
* wheel
20 21
* setuptools_scm
21
* babel
22
* babel (Python library)
22 23

  
23 24
### Test
24 25

  
25 26
* pytest
26
* reuse
27 27

  
28
## Building & testing
28
## Building & testing & installation from wheel
29 29

  
30
Build and test processed are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
30
Build, test and installation processes are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
31 31

  
32 32
## Running
33 33

  
......
41 41
```
42 42

  
43 43
You might as well like to run from sources, without installation:
44

  
45
``` shell
44 46
mkdir /tmp/bananowarzez/
45 47
./setup.py compile_catalog # generate the necessary .po files
46 48
PYTHONPATH=src python3 -m hydrilla.builder -s src/test/source-package-example/ \
conftest.py
1
# SPDX-License-Identifier: CC0-1.0
2

  
3
# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
4
#
5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6

  
7
import sys
8
from pathlib import Path
9

  
10
import pytest
11
import pkgutil
12
import importlib
13
from tempfile import TemporaryDirectory
14
from typing import Iterable
15

  
16
here = Path(__file__).resolve().parent
17
sys.path.insert(0, str(here / 'src'))
18

  
19
@pytest.fixture(autouse=True)
20
def no_requests(monkeypatch):
21
    """Remove requests.sessions.Session.request for all tests."""
22
    if importlib.util.find_spec("requests") is not None:
23
        monkeypatch.delattr('requests.sessions.Session.request')
24

  
25
@pytest.fixture
26
def mock_subprocess_run(monkeypatch, request):
27
    """
28
    Temporarily replace subprocess.run() with a function supplied through pytest
29
    marker 'subprocess_run'.
30

  
31
    The marker excepts 2 arguments:
32
    * the module inside which the subprocess attribute should be mocked and
33
    * a run() function to use.
34
    """
35
    where, mocked_run = request.node.get_closest_marker('subprocess_run').args
36

  
37
    class MockedSubprocess:
38
        """Minimal mocked version of the subprocess module."""
39
        run = mocked_run
40

  
41
    monkeypatch.setattr(where, 'subprocess', MockedSubprocess)
42

  
43
@pytest.fixture(autouse=True)
44
def no_gettext(monkeypatch, request):
45
    """
46
    Make gettext return all strings untranslated unless we request otherwise.
47
    """
48
    if request.node.get_closest_marker('enable_gettext'):
49
        return
50

  
51
    import hydrilla
52
    modules_to_process = [hydrilla]
53

  
54
    def add_child_modules(parent):
55
        """
56
        Recursuvely collect all modules descending from 'parent' into an array.
57
        """
58
        try:
59
            load_paths = parent.__path__
60
        except AttributeError:
61
            return
62

  
63
        for module_info in pkgutil.iter_modules(load_paths):
64
            if module_info.name != '__main__':
65
                __import__(f'{parent.__name__}.{module_info.name}')
66
                modules_to_process.append(getattr(parent, module_info.name))
67
                add_child_modules(getattr(parent, module_info.name))
68

  
69
    add_child_modules(hydrilla)
70

  
71
    for module in modules_to_process:
72
        if hasattr(module, '_'):
73
            monkeypatch.setattr(module, '_', lambda message: message)
74

  
75
@pytest.fixture
76
def tmpdir() -> Iterable[Path]:
77
    """
78
    Provide test case with a temporary directory that will be automatically
79
    deleted after the test.
80
    """
81
    with TemporaryDirectory() as tmpdir:
82
        yield Path(tmpdir)
doc/man/man1/hydrilla-builder.1
6 6
.\"
7 7
.\" Available under the terms of Creative Commons Zero v1.0 Universal.
8 8

  
9
.TH HYDRILLA-BUILDER 1 2022-04-22 "Hydrilla 1.0" "Hydrilla Manual"
9
.TH HYDRILLA-BUILDER 1 2022-06-14 "Hydrilla 1.1" "Hydrilla Manual"
10 10

  
11 11
.SH NAME
12
hydrilla-builder \- Generate packages to be served by Hydrilla
12
hydrilla\-builder \- Generate packages to be served by Hydrilla
13 13

  
14 14
.SH SYNOPSIS
15 15
.B "hydrilla\-builder \-\-help"
......
21 21
names.)
22 22

  
23 23
.SH DESCRIPTION
24
.I hydrilla-builder
24
.I hydrilla\-builder
25 25
is a tool which takes a Hydrilla source package and generates files of a
26 26
built package, suitable for serving by the Hydrilla server.
27 27

  
28
As of Hydrilla version 1.0
29
.I hydrilla-builder
30
does not yet perform nor trigger actions like compilation, minification or
31
bundling of source code files. Its main function is to automate the process
32
of computing SHA256 cryptographic sums of package files and including them
33
in JSON definitions.
28
The main function of
29
.I hydrilla\-builder
30
is to automate the process of computing SHA256 cryptographic sums of package
31
files and including them in JSON definitions.
32

  
33
This tool does not perform nor trigger actions like compilation, minification or
34
bundling of source code files. When this is needed,
35
.I hydrilla\-builder
36
instead relies on facilities already provided by other software distribution
37
systems like APT and extracts the requested files from .deb packages. This
38
feature is called \*(lqpiggybacking\*(rq.
34 39

  
35 40
In addition,
36
.B hydrilla\-builder
41
.I hydrilla\-builder
37 42
can generate an SPDX report from source package if the
38 43
\*(lqreuse_generate_spdx_report\*(rq property is set to true in index.json.
39 44

  
......
64 69
\*(lqindex.json\*(rq, substituting any file with such name that could be
65 70
present in the source directory.
66 71

  
72
.TP
73
.BI \-p " PIGGYBACK_PATH" "\fR,\fP \-\^\-piggyback\-files=" PIGGYBACK_PATH
74
Read and write foreign package archives under
75
.IR PIGGYBACK_PATH .
76
If not specified, a default value is computed by appending
77
\*(lq.foreign-packages\*(rq to the
78
.I SOURCE
79
directory path.
80

  
67 81
.TP
68 82
.BI \-d " DESTINATION" "\fR,\fP \-\^\-dstdir=" DESTINATION
69 83
Write generated files under
70 84
.IR DESTINATION .
71
Files are written in such way that
72 85
.I DESTINATION
73
is valid for being passed to Hydrilla to serve packages from.
86
can then be passed to Hydrilla to serve packages from.
74 87

  
75 88
.TP
76 89
.B \-\^\-version
77 90
Show version information for this instance of
78
.I hydrilla-builder
91
.I hydrilla\-builder
79 92
on the standard output and exit successfully.
80 93

  
81 94
.SH "EXIT STATUS"
pyproject.toml
13 13

  
14 14
[tool.pytest.ini_options]
15 15
minversion = "6.0"
16
addopts = "-ra -q"
16
addopts = "-ra"
17 17
testpaths = [
18 18
    "tests"
19 19
]
20
markers = [
21
    "subprocess_run: define how mocked subprocess.run should behave"
22
]
setup.cfg
24 24
zip_safe = False
25 25
package_dir = 
26 26
	= src
27
packages = find:
27
packages = find_namespace:
28 28
include_package_data = True
29 29
python_requires = >= 3.7
30 30
install_requires = 
......
36 36

  
37 37
[options.extras_require]
38 38
test = pytest
39
setup = setuptools_scm
39
setup = setuptools_scm; babel
40
spdx = reuse
40 41

  
41 42
[options.packages.find]
42 43
where = src
setup.py
8 8
import setuptools
9 9

  
10 10
from setuptools.command.build_py import build_py
11
from setuptools.command.sdist import sdist
12

  
13
from pathlib import Path
14

  
15
here = Path(__file__).resolve().parent
11 16

  
12 17
class CustomBuildCommand(build_py):
13
    '''
14
    The build command but runs babel before build.
15
    '''
18
    """The build command but runs babel before build."""
16 19
    def run(self, *args, **kwargs):
20
        """Wrapper around build_py's original run() method."""
17 21
        self.run_command('compile_catalog')
22

  
23
        super().run(*args, **kwargs)
24

  
25
class CustomSdistCommand(sdist):
26
    """
27
    The sdist command but prevents compiled message catalogs from being included
28
    in the archive.
29
    """
30
    def run(self, *args, **kwargs):
31
        """Wrapper around sdist's original run() method."""
32
        locales_dir = here / 'src/hydrilla/builder/locales'
33
        locale_files = {}
34

  
35
        for path in locales_dir.rglob('*.mo'):
36
            locale_files[path] = path.read_bytes()
37

  
38
        for path in locale_files:
39
            path.unlink()
40

  
18 41
        super().run(*args, **kwargs)
19 42

  
20
setuptools.setup(cmdclass={'build_py': CustomBuildCommand})
43
        for path, contents in locale_files.items():
44
            path.write_bytes(contents)
45

  
46
setuptools.setup(cmdclass = {
47
    'build_py': CustomBuildCommand,
48
    'sdist': CustomSdistCommand
49
})
src/hydrilla.builder.egg-info/PKG-INFO
1 1
Metadata-Version: 2.1
2 2
Name: hydrilla.builder
3
Version: 1.0
3
Version: 1.1b1
4 4
Summary: Hydrilla package builder
5 5
Home-page: https://git.koszko.org/hydrilla-builder
6 6
Author: Wojtek Kosior
......
24 24
        
25 25
        ### Build
26 26
        
27
        * build (a PEP517 package builder)
27 28
        * setuptools
28 29
        * wheel
29 30
        * setuptools_scm
30
        * babel
31
        * babel (Python library)
31 32
        
32 33
        ### Test
33 34
        
34 35
        * pytest
35
        * reuse
36 36
        
37
        ## Building & testing
37
        ## Building & testing & installation from wheel
38 38
        
39
        Build and test processed are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
39
        Build, test and installation processes are analogous to those described in the [README of Hydrilla server part](https://git.koszko.org/pydrilla/about).
40 40
        
41 41
        ## Running
42 42
        
......
50 50
        ```
51 51
        
52 52
        You might as well like to run from sources, without installation:
53
        
54
        ``` shell
53 55
        mkdir /tmp/bananowarzez/
54 56
        ./setup.py compile_catalog # generate the necessary .po files
55 57
        PYTHONPATH=src python3 -m hydrilla.builder -s src/test/source-package-example/ \
......
86 88
Requires-Python: >=3.7
87 89
Description-Content-Type: text/markdown
88 90
Provides-Extra: setup
91
Provides-Extra: spdx
89 92
Provides-Extra: test
src/hydrilla.builder.egg-info/SOURCES.txt
1 1
MANIFEST.in
2 2
README.md
3 3
README.md.license
4
conftest.py
4 5
pyproject.toml
5 6
setup.cfg
6 7
setup.py
......
21 22
src/hydrilla/builder/__main__.py
22 23
src/hydrilla/builder/_version.py
23 24
src/hydrilla/builder/build.py
25
src/hydrilla/builder/common_errors.py
26
src/hydrilla/builder/local_apt.py
27
src/hydrilla/builder/piggybacking.py
24 28
src/hydrilla/builder/locales/en_US/LC_MESSAGES/hydrilla-messages.po
25
src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json
26
src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json.license
27
src/hydrilla/schemas/api_query_result-1.0.1.schema.json
28
src/hydrilla/schemas/api_query_result-1.0.1.schema.json.license
29
src/hydrilla/schemas/api_resource_description-1.0.1.schema.json
30
src/hydrilla/schemas/api_resource_description-1.0.1.schema.json.license
31
src/hydrilla/schemas/api_source_description-1.0.1.schema.json
32
src/hydrilla/schemas/api_source_description-1.0.1.schema.json.license
33
src/hydrilla/schemas/common_definitions-1.0.1.schema.json
34
src/hydrilla/schemas/common_definitions-1.0.1.schema.json.license
35
src/hydrilla/schemas/package_source-1.0.1.schema.json
36
src/hydrilla/schemas/package_source-1.0.1.schema.json.license
29
src/hydrilla/schemas/1.x/api_mapping_description-1.0.1.schema.json
30
src/hydrilla/schemas/1.x/api_mapping_description-1.0.1.schema.json.license
31
src/hydrilla/schemas/1.x/api_query_result-1.0.1.schema.json
32
src/hydrilla/schemas/1.x/api_query_result-1.0.1.schema.json.license
33
src/hydrilla/schemas/1.x/api_resource_description-1.0.1.schema.json
34
src/hydrilla/schemas/1.x/api_resource_description-1.0.1.schema.json.license
35
src/hydrilla/schemas/1.x/api_source_description-1.0.1.schema.json
36
src/hydrilla/schemas/1.x/api_source_description-1.0.1.schema.json.license
37
src/hydrilla/schemas/1.x/common_definitions-1.0.1.schema.json
38
src/hydrilla/schemas/1.x/common_definitions-1.0.1.schema.json.license
39
src/hydrilla/schemas/1.x/package_source-1.0.1.schema.json
40
src/hydrilla/schemas/1.x/package_source-1.0.1.schema.json.license
41
src/hydrilla/schemas/2.x/api_mapping_description-2.schema.json
42
src/hydrilla/schemas/2.x/api_mapping_description-2.schema.json.license
43
src/hydrilla/schemas/2.x/api_query_result-2.schema.json
44
src/hydrilla/schemas/2.x/api_query_result-2.schema.json.license
45
src/hydrilla/schemas/2.x/api_resource_description-2.schema.json
46
src/hydrilla/schemas/2.x/api_resource_description-2.schema.json.license
47
src/hydrilla/schemas/2.x/api_source_description-2.schema.json
48
src/hydrilla/schemas/2.x/api_source_description-2.schema.json.license
49
src/hydrilla/schemas/2.x/common_definitions-2.schema.json
50
src/hydrilla/schemas/2.x/common_definitions-2.schema.json.license
51
src/hydrilla/schemas/2.x/package_source-2.schema.json
52
src/hydrilla/schemas/2.x/package_source-2.schema.json.license
37 53
src/hydrilla/util/__init__.py
38 54
src/hydrilla/util/_util.py
39
tests/test_hydrilla_builder.py
55
tests/__init__.py
56
tests/helpers.py
57
tests/test_build.py
58
tests/test_local_apt.py
40 59
tests/source-package-example/README.txt
41 60
tests/source-package-example/README.txt.license
42 61
tests/source-package-example/bye.js
src/hydrilla.builder.egg-info/requires.txt
2 2
jsonschema>=3.0
3 3

  
4 4
[setup]
5
babel
5 6
setuptools_scm
6 7

  
8
[spdx]
9
reuse
10

  
7 11
[test]
8 12
pytest
src/hydrilla/builder/_version.py
1 1
# coding: utf-8
2 2
# file generated by setuptools_scm
3 3
# don't change, don't track in version control
4
version = '1.0'
5
version_tuple = (1, 0)
4
version = '1.1b1'
5
version_tuple = (1, '1b1')
src/hydrilla/builder/build.py
30 30
import json
31 31
import re
32 32
import zipfile
33
from pathlib import Path
33
import subprocess
34
from pathlib import Path, PurePosixPath
34 35
from hashlib import sha256
35 36
from sys import stderr
37
from contextlib import contextmanager
38
from tempfile import TemporaryDirectory, TemporaryFile
39
from typing import Optional, Iterable, Union
36 40

  
37 41
import jsonschema
38 42
import click
39 43

  
40 44
from .. import util
41 45
from . import _version
46
from . import local_apt
47
from .piggybacking import Piggybacked
48
from .common_errors import *
42 49

  
43 50
here = Path(__file__).resolve().parent
44 51

  
45 52
_ = util.translation(here / 'locales').gettext
46 53

  
47
index_validator = util.validator_for('package_source-1.0.1.schema.json')
48

  
49 54
schemas_root = 'https://hydrilla.koszko.org/schemas'
50 55

  
51 56
generated_by = {
......
53 58
    'version': _version.version
54 59
}
55 60

  
56
class FileReferenceError(Exception):
57
    """
58
    Exception used to report various problems concerning files referenced from
59
    source package's index.json.
60
    """
61

  
62
class ReuseError(Exception):
61
class ReuseError(SubprocessError):
63 62
    """
64 63
    Exception used to report various problems when calling the REUSE tool.
65 64
    """
66 65

  
67
class FileBuffer:
68
    """
69
    Implement a file-like object that buffers data written to it.
70
    """
71
    def __init__(self):
72
        """
73
        Initialize FileBuffer.
74
        """
75
        self.chunks = []
76

  
77
    def write(self, b):
78
        """
79
        Buffer 'b', return number of bytes buffered.
80

  
81
        'b' is expected to be an instance of 'bytes' or 'str', in which case it
82
        gets encoded as UTF-8.
83
        """
84
        if type(b) is str:
85
            b = b.encode()
86
        self.chunks.append(b)
87
        return len(b)
88

  
89
    def flush(self):
90
        """
91
        A no-op mock of file-like object's flush() method.
92
        """
93
        pass
94

  
95
    def get_bytes(self):
96
        """
97
        Return all data written so far concatenated into a single 'bytes'
98
        object.
99
        """
100
        return b''.join(self.chunks)
101

  
102
def generate_spdx_report(root):
66
def generate_spdx_report(root: Path) -> bytes:
103 67
    """
104 68
    Use REUSE tool to generate an SPDX report for sources under 'root' and
105 69
    return the report's contents as 'bytes'.
106 70

  
107
    'root' shall be an instance of pathlib.Path.
108

  
109 71
    In case the directory tree under 'root' does not constitute a
110
    REUSE-compliant package, linting report is printed to standard output and
111
    an exception is raised.
72
    REUSE-compliant package, as exception is raised with linting report
73
    included in it.
112 74

  
113
    In case the reuse package is not installed, an exception is also raised.
75
    In case the reuse tool is not installed, an exception is also raised.
114 76
    """
115
    try:
116
        from reuse._main import main as reuse_main
117
    except ModuleNotFoundError:
118
        raise ReuseError(_('couldnt_import_reuse_is_it_installed'))
119

  
120
    mocked_output = FileBuffer()
121
    if reuse_main(args=['--root', str(root), 'lint'], out=mocked_output) != 0:
122
        stderr.write(mocked_output.get_bytes().decode())
123
        raise ReuseError(_('spdx_report_from_reuse_incompliant'))
124

  
125
    mocked_output = FileBuffer()
126
    if reuse_main(args=['--root', str(root), 'spdx'], out=mocked_output) != 0:
127
        stderr.write(mocked_output.get_bytes().decode())
128
        raise ReuseError("Couldn't generate an SPDX report for package.")
129

  
130
    return mocked_output.get_bytes()
77
    for command in [
78
            ['reuse', '--root', str(root), 'lint'],
79
            ['reuse', '--root', str(root), 'spdx']
80
    ]:
81
        try:
82
            cp = subprocess.run(command, capture_output=True, text=True)
83
        except FileNotFoundError:
84
            msg = _('couldnt_execute_{}_is_it_installed').format('reuse')
85
            raise ReuseError(msg)
86

  
87
        if cp.returncode != 0:
88
            msg = _('command_{}_failed').format(' '.join(command))
89
            raise ReuseError(msg, cp)
90

  
91
    return cp.stdout.encode()
131 92

  
132 93
class FileRef:
133 94
    """Represent reference to a file in the package."""
134
    def __init__(self, path: Path, contents: bytes):
95
    def __init__(self, path: PurePosixPath, contents: bytes) -> None:
135 96
        """Initialize FileRef."""
136
        self.include_in_distribution = False
137
        self.include_in_zipfile      = True
138
        self.path                    = path
139
        self.contents                = contents
97
        self.include_in_distribution   = False
98
        self.include_in_source_archive = True
99
        self.path                      = path
100
        self.contents                  = contents
140 101

  
141 102
        self.contents_hash = sha256(contents).digest().hex()
142 103

  
143
    def make_ref_dict(self, filename: str):
104
    def make_ref_dict(self) -> dict[str, str]:
144 105
        """
145 106
        Represent the file reference through a dict that can be included in JSON
146 107
        defintions.
147 108
        """
148 109
        return {
149
            'file':   filename,
110
            'file':   str(self.path),
150 111
            'sha256': self.contents_hash
151 112
        }
152 113

  
114
@contextmanager
115
def piggybacked_system(piggyback_def: Optional[dict],
116
                       piggyback_files: Optional[Path]) \
117
                       -> Iterable[Piggybacked]:
118
    """
119
    Resolve resources from a foreign software packaging system. Optionally, use
120
    package files (.deb's, etc.) from a specified directory instead of resolving
121
    and downloading them.
122
    """
123
    if piggyback_def is None:
124
        yield Piggybacked()
125
    else:
126
        # apt is the only supported system right now
127
        assert piggyback_def['system'] == 'apt'
128

  
129
        with local_apt.piggybacked_system(piggyback_def, piggyback_files) \
130
             as piggybacked:
131
            yield piggybacked
132

  
153 133
class Build:
154 134
    """
155 135
    Build a Hydrilla package.
156 136
    """
157
    def __init__(self, srcdir, index_json_path):
137
    def __init__(self, srcdir: Path, index_json_path: Path,
138
                 piggyback_files: Optional[Path]=None):
158 139
        """
159 140
        Initialize a build. All files to be included in a distribution package
160 141
        are loaded into memory, all data gets validated and all necessary
161 142
        computations (e.g. preparing of hashes) are performed.
162

  
163
        'srcdir' and 'index_json' are expected to be pathlib.Path objects.
164 143
        """
165 144
        self.srcdir          = srcdir.resolve()
166
        self.index_json_path = index_json_path
145
        self.piggyback_files = piggyback_files
146
        if piggyback_files is None:
147
            piggyback_default_path = \
148
                srcdir.parent / f'{srcdir.name}.foreign-packages'
149
            if piggyback_default_path.exists():
150
                self.piggyback_files = piggyback_default_path
167 151
        self.files_by_path   = {}
168 152
        self.resource_list   = []
169 153
        self.mapping_list    = []
170 154

  
171 155
        if not index_json_path.is_absolute():
172
            self.index_json_path = (self.srcdir / self.index_json_path)
156
            index_json_path = (self.srcdir / index_json_path)
173 157

  
174
        self.index_json_path = self.index_json_path.resolve()
158
        index_obj, major = util.load_instance_from_file(index_json_path)
175 159

  
176
        with open(self.index_json_path, 'rt') as index_file:
177
            index_json_text = index_file.read()
160
        if major not in (1, 2):
161
            msg = _('unknown_schema_package_source_{}')\
162
                .format(index_json_path)
163
            raise util.UnknownSchemaError(msg)
178 164

  
179
        index_obj = json.loads(util.strip_json_comments(index_json_text))
165
        index_desired_path = PurePosixPath('index.json')
166
        self.files_by_path[index_desired_path] = \
167
            FileRef(index_desired_path, index_json_path.read_bytes())
180 168

  
181
        self.files_by_path[self.srcdir / 'index.json'] = \
182
            FileRef(self.srcdir / 'index.json', index_json_text.encode())
169
        self._process_index_json(index_obj, major)
183 170

  
184
        self._process_index_json(index_obj)
185

  
186
    def _process_file(self, filename: str, include_in_distribution: bool=True):
171
    def _process_file(self, filename: Union[str, PurePosixPath],
172
                      piggybacked: Piggybacked,
173
                      include_in_distribution: bool=True):
187 174
        """
188 175
        Resolve 'filename' relative to srcdir, load it to memory (if not loaded
189 176
        before), compute its hash and store its information in
190 177
        'self.files_by_path'.
191 178

  
192
        'filename' shall represent a relative path using '/' as a separator.
179
        'filename' shall represent a relative path withing package directory.
193 180

  
194 181
        if 'include_in_distribution' is True it shall cause the file to not only
195 182
        be included in the source package's zipfile, but also written as one of
196 183
        built package's files.
197 184

  
185
        For each file an attempt is made to resolve it using 'piggybacked'
186
        object. If a file is found and pulled from foreign software packaging
187
        system this way, it gets automatically excluded from inclusion in
188
        Hydrilla source package's zipfile.
189

  
198 190
        Return file's reference object that can be included in JSON defintions
199 191
        of various kinds.
200 192
        """
201
        path = self.srcdir
202
        for segment in filename.split('/'):
203
            path /= segment
204

  
205
        path = path.resolve()
206
        if not path.is_relative_to(self.srcdir):
207
            raise FileReferenceError(_('loading_{}_outside_package_dir')
208
                                     .format(filename))
209

  
210
        if str(path.relative_to(self.srcdir)) == 'index.json':
211
            raise FileReferenceError(_('loading_reserved_index_json'))
193
        include_in_source_archive = True
194

  
195
        desired_path = PurePosixPath(filename)
196
        if '..' in desired_path.parts:
197
            msg = _('path_contains_double_dot_{}').format(filename)
198
            raise FileReferenceError(msg)
199

  
200
        path = piggybacked.resolve_file(desired_path)
201
        if path is None:
202
            path = (self.srcdir / desired_path).resolve()
203
            try:
204
                rel_path = path.relative_to(self.srcdir)
205
            except ValueError:
206
                raise FileReferenceError(_('loading_{}_outside_package_dir')
207
                                         .format(filename))
208

  
209
            if str(rel_path) == 'index.json':
210
                raise FileReferenceError(_('loading_reserved_index_json'))
211
        else:
212
            include_in_source_archive = False
212 213

  
213
        file_ref = self.files_by_path.get(path)
214
        file_ref = self.files_by_path.get(desired_path)
214 215
        if file_ref is None:
215
            with open(path, 'rb') as file_handle:
216
                contents = file_handle.read()
216
            if not path.is_file():
217
                msg = _('referenced_file_{}_missing').format(desired_path)
218
                raise FileReferenceError(msg)
217 219

  
218
            file_ref = FileRef(path, contents)
219
            self.files_by_path[path] = file_ref
220
            file_ref = FileRef(desired_path, path.read_bytes())
221
            self.files_by_path[desired_path] = file_ref
220 222

  
221 223
        if include_in_distribution:
222 224
            file_ref.include_in_distribution = True
223 225

  
224
        return file_ref.make_ref_dict(filename)
226
        if not include_in_source_archive:
227
            file_ref.include_in_source_archive = False
228

  
229
        return file_ref.make_ref_dict()
225 230

  
226
    def _prepare_source_package_zip(self, root_dir_name: str):
231
    def _prepare_source_package_zip(self, source_name: str,
232
                                    piggybacked: Piggybacked) -> str:
227 233
        """
228 234
        Create and store in memory a .zip archive containing files needed to
229 235
        build this source package.
230 236

  
231
        'root_dir_name' shall not contain any slashes ('/').
237
        'src_dir_name' shall not contain any slashes ('/').
232 238

  
233 239
        Return zipfile's sha256 sum's hexstring.
234 240
        """
235
        fb = FileBuffer()
236
        root_dir_path = Path(root_dir_name)
241
        tf = TemporaryFile()
242
        source_dir_path      = PurePosixPath(source_name)
243
        piggybacked_dir_path = PurePosixPath(f'{source_name}.foreign-packages')
237 244

  
238
        def zippath(file_path):
239
            file_path = root_dir_path / file_path.relative_to(self.srcdir)
240
            return file_path.as_posix()
241

  
242
        with zipfile.ZipFile(fb, 'w') as xpi:
245
        with zipfile.ZipFile(tf, 'w') as zf:
243 246
            for file_ref in self.files_by_path.values():
244
                if file_ref.include_in_zipfile:
245
                    xpi.writestr(zippath(file_ref.path), file_ref.contents)
247
                if file_ref.include_in_source_archive:
248
                    zf.writestr(str(source_dir_path / file_ref.path),
249
                                file_ref.contents)
250

  
251
            for desired_path, real_path in piggybacked.archive_files():
252
                zf.writestr(str(piggybacked_dir_path / desired_path),
253
                            real_path.read_bytes())
246 254

  
247
        self.source_zip_contents = fb.get_bytes()
255
        tf.seek(0)
256
        self.source_zip_contents = tf.read()
248 257

  
249 258
        return sha256(self.source_zip_contents).digest().hex()
250 259

  
251
    def _process_item(self, item_def: dict):
260
    def _process_item(self, as_what: str, item_def: dict,
261
                      piggybacked: Piggybacked):
252 262
        """
253
        Process 'item_def' as definition of a resource/mapping and store in
254
        memory its processed form and files used by it.
263
        Process 'item_def' as definition of a resource or mapping (determined by
264
        'as_what' param) and store in memory its processed form and files used
265
        by it.
255 266

  
256 267
        Return a minimal item reference suitable for using in source
257 268
        description.
258 269
        """
259
        copy_props = ['type', 'identifier', 'long_name', 'description']
260
        for prop in ('comment', 'uuid'):
261
            if prop in item_def:
262
                copy_props.append(prop)
270
        resulting_schema_version = [1]
271

  
272
        copy_props = ['identifier', 'long_name', 'description',
273
                      *filter(lambda p: p in item_def, ('comment', 'uuid'))]
263 274

  
264
        if item_def['type'] == 'resource':
275
        if as_what == 'resource':
265 276
            item_list = self.resource_list
266 277

  
267 278
            copy_props.append('revision')
268 279

  
269
            script_file_refs = [self._process_file(f['file'])
280
            script_file_refs = [self._process_file(f['file'], piggybacked)
270 281
                                for f in item_def.get('scripts', [])]
271 282

  
272 283
            deps = [{'identifier': res_ref['identifier']}
273 284
                    for res_ref in item_def.get('dependencies', [])]
274 285

  
275 286
            new_item_obj = {
276
                'dependencies': deps,
287
                'dependencies': [*piggybacked.resource_must_depend, *deps],
277 288
                'scripts':      script_file_refs
278 289
            }
279 290
        else:
......
287 298
                'payloads': payloads
288 299
            }
289 300

  
290
        new_item_obj.update([(p, item_def[p]) for p in copy_props])
291

  
292 301
        new_item_obj['version'] = util.normalize_version(item_def['version'])
293
        new_item_obj['$schema'] = f'{schemas_root}/api_{item_def["type"]}_description-1.schema.json'
302

  
303
        if as_what == 'mapping' and item_def['type'] == "mapping_and_resource":
304
            new_item_obj['version'].append(item_def['revision'])
305

  
306
        if self.source_schema_ver >= [2]:
307
            # handle 'required_mappings' field
308
            required = [{'identifier': map_ref['identifier']}
309
                        for map_ref in item_def.get('required_mappings', [])]
310
            if required:
311
                resulting_schema_version = max(resulting_schema_version, [2])
312
                new_item_obj['required_mappings'] = required
313

  
314
            # handle 'permissions' field
315
            permissions = item_def.get('permissions', {})
316
            processed_permissions = {}
317

  
318
            if permissions.get('cors_bypass'):
319
                processed_permissions['cors_bypass'] = True
320
            if permissions.get('eval'):
321
                processed_permissions['eval'] = True
322

  
323
            if processed_permissions:
324
                new_item_obj['permissions'] = processed_permissions
325
                resulting_schema_version = max(resulting_schema_version, [2])
326

  
327
            # handle '{min,max}_haketilo_version' fields
328
            for minmax, default in ('min', [1]), ('max', [65536]):
329
                constraint = item_def.get(f'{minmax}_haketilo_version')
330
                if constraint in (None, default):
331
                    continue
332

  
333
                copy_props.append(f'{minmax}_haketilo_version')
334
                resulting_schema_version = max(resulting_schema_version, [2])
335

  
336
        new_item_obj.update((p, item_def[p]) for p in copy_props)
337

  
338
        new_item_obj['$schema'] = ''.join([
339
            schemas_root,
340
            f'/api_{as_what}_description',
341
            '-',
342
            util.version_string(resulting_schema_version),
343
            '.schema.json'
344
        ])
345
        new_item_obj['type']             = as_what
294 346
        new_item_obj['source_copyright'] = self.copyright_file_refs
295
        new_item_obj['source_name'] = self.source_name
296
        new_item_obj['generated_by'] = generated_by
347
        new_item_obj['source_name']      = self.source_name
348
        new_item_obj['generated_by']     = generated_by
297 349

  
298 350
        item_list.append(new_item_obj)
299 351

  
300 352
        props_in_ref = ('type', 'identifier', 'version', 'long_name')
301 353
        return dict([(prop, new_item_obj[prop]) for prop in props_in_ref])
302 354

  
303
    def _process_index_json(self, index_obj: dict):
355
    def _process_index_json(self, index_obj: dict,
356
                            major_schema_version: int) -> None:
304 357
        """
305 358
        Process 'index_obj' as contents of source package's index.json and store
306 359
        in memory this source package's zipfile as well as package's individual
307 360
        files and computed definitions of the source package and items defined
308 361
        in it.
309 362
        """
310
        index_validator.validate(index_obj)
363
        schema_name = f'package_source-{major_schema_version}.schema.json';
364

  
365
        util.validator_for(schema_name).validate(index_obj)
311 366

  
312
        schema = f'{schemas_root}/api_source_description-1.schema.json'
367
        match = re.match(r'.*-((([1-9][0-9]*|0)\.)+)schema\.json$',
368
                         index_obj['$schema'])
369
        self.source_schema_ver = \
370
            [int(n) for n in filter(None, match.group(1).split('.'))]
371

  
372
        out_schema = f'{schemas_root}/api_source_description-1.schema.json'
313 373

  
314 374
        self.source_name = index_obj['source_name']
315 375

  
316 376
        generate_spdx = index_obj.get('reuse_generate_spdx_report', False)
317 377
        if generate_spdx:
318 378
            contents  = generate_spdx_report(self.srcdir)
319
            spdx_path = (self.srcdir / 'report.spdx').resolve()
379
            spdx_path = PurePosixPath('report.spdx')
320 380
            spdx_ref  = FileRef(spdx_path, contents)
321 381

  
322
            spdx_ref.include_in_zipfile = False
382
            spdx_ref.include_in_source_archive = False
323 383
            self.files_by_path[spdx_path] = spdx_ref
324 384

  
325
        self.copyright_file_refs = \
326
            [self._process_file(f['file']) for f in index_obj['copyright']]
385
        piggyback_def = None
386
        if self.source_schema_ver >= [1, 1] and 'piggyback_on' in index_obj:
387
            piggyback_def = index_obj['piggyback_on']
327 388

  
328
        if generate_spdx and not spdx_ref.include_in_distribution:
329
            raise FileReferenceError(_('report_spdx_not_in_copyright_list'))
389
        with piggybacked_system(piggyback_def, self.piggyback_files) \
390
             as piggybacked:
391
            copyright_to_process = [
392
                *(file_ref['file'] for file_ref in index_obj['copyright']),
393
                *piggybacked.package_license_files
394
            ]
395
            self.copyright_file_refs = [self._process_file(f, piggybacked)
396
                                        for f in copyright_to_process]
330 397

  
331
        item_refs = [self._process_item(d) for d in index_obj['definitions']]
398
            if generate_spdx and not spdx_ref.include_in_distribution:
399
                raise FileReferenceError(_('report_spdx_not_in_copyright_list'))
332 400

  
333
        for file_ref in index_obj.get('additional_files', []):
334
            self._process_file(file_ref['file'], include_in_distribution=False)
401
            item_refs = []
402
            for item_def in index_obj['definitions']:
403
                if 'mapping' in item_def['type']:
404
                    ref = self._process_item('mapping', item_def, piggybacked)
405
                    item_refs.append(ref)
406
                if 'resource' in item_def['type']:
407
                    ref = self._process_item('resource', item_def, piggybacked)
408
                    item_refs.append(ref)
335 409

  
336
        root_dir_path = Path(self.source_name)
410
            for file_ref in index_obj.get('additional_files', []):
411
                self._process_file(file_ref['file'], piggybacked,
412
                                   include_in_distribution=False)
337 413

  
338
        source_archives_obj = {
339
            'zip' : {
340
                'sha256': self._prepare_source_package_zip(root_dir_path)
341
            }
342
        }
414
            zipfile_sha256 = self._prepare_source_package_zip\
415
                (self.source_name, piggybacked)
416

  
417
            source_archives_obj = {'zip' : {'sha256': zipfile_sha256}}
343 418

  
344 419
        self.source_description = {
345
            '$schema':            schema,
420
            '$schema':            out_schema,
346 421
            'source_name':        self.source_name,
347 422
            'source_copyright':   self.copyright_file_refs,
348 423
            'upstream_url':       index_obj['upstream_url'],
......
398 473

  
399 474
dir_type = click.Path(exists=True, file_okay=False, resolve_path=True)
400 475

  
476
@click.command(help=_('build_package_from_srcdir_to_dstdir'))
401 477
@click.option('-s', '--srcdir', default='./', type=dir_type, show_default=True,
402 478
              help=_('source_directory_to_build_from'))
403 479
@click.option('-i', '--index-json', default='index.json', type=click.Path(),
404 480
              help=_('path_instead_of_index_json'))
481
@click.option('-p', '--piggyback-files', type=click.Path(),
482
              help=_('path_instead_for_piggyback_files'))
405 483
@click.option('-d', '--dstdir', type=dir_type, required=True,
406 484
              help=_('built_package_files_destination'))
407 485
@click.version_option(version=_version.version, prog_name='Hydrilla builder',
408 486
                      message=_('%(prog)s_%(version)s_license'),
409 487
                      help=_('version_printing'))
410
def perform(srcdir, index_json, dstdir):
411
    """<this will be replaced by a localized docstring for Click to pick up>"""
412
    build = Build(Path(srcdir), Path(index_json))
413
    build.write_package_files(Path(dstdir))
414

  
415
perform.__doc__ = _('build_package_from_srcdir_to_dstdir')
488
def perform(srcdir, index_json, piggyback_files, dstdir):
489
    """
490
    Execute Hydrilla builder to turn source package into a distributable one.
416 491

  
417
perform = click.command()(perform)
492
    This command is meant to be the entry point of hydrilla-builder command
493
    exported by this package.
494
    """
495
    build = Build(Path(srcdir), Path(index_json),
496
                  piggyback_files and Path(piggyback_files))
497
    build.write_package_files(Path(dstdir))
src/hydrilla/builder/common_errors.py
1
# SPDX-License-Identifier: AGPL-3.0-or-later
2

  
3
# Error classes.
4
#
5
# This file is part of Hydrilla
6
#
7
# Copyright (C) 2022 Wojtek Kosior
8
#
9
# This program is free software: you can redistribute it and/or modify
10
# it under the terms of the GNU Affero General Public License as
11
# published by the Free Software Foundation, either version 3 of the
12
# License, or (at your option) any later version.
13
#
14
# This program is distributed in the hope that it will be useful,
15
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
# GNU Affero General Public License for more details.
18
#
19
# You should have received a copy of the GNU Affero General Public License
20
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
21
#
22
#
23
# I, Wojtek Kosior, thereby promise not to sue for violation of this
24
# file's license. Although I request that you do not make use this code
25
# in a proprietary program, I am not going to enforce this in court.
26

  
27
"""
28
This module defines error types for use in other parts of Hydrilla builder.
29
"""
30

  
31
# Enable using with Python 3.7.
32
from __future__ import annotations
33

  
34
from pathlib import Path
35

  
36
from .. import util
37

  
38
here = Path(__file__).resolve().parent
39

  
40
_ = util.translation(here / 'locales').gettext
41

  
42
class DistroError(Exception):
43
    """
44
    Exception used to report problems when resolving an OS distribution.
45
    """
46

  
47
class FileReferenceError(Exception):
48
    """
49
    Exception used to report various problems concerning files referenced from
50
    source package.
51
    """
52

  
53
class SubprocessError(Exception):
54
    """
55
    Exception used to report problems related to execution of external
56
    processes, includes. various problems when calling apt-* and dpkg-*
57
    commands.
58
    """
59
    def __init__(self, msg: str, cp: Optional[CP]=None) -> None:
60
        """Initialize this SubprocessError"""
61
        if cp and cp.stdout:
62
            msg = '\n\n'.join([msg, _('STDOUT_OUTPUT_heading'), cp.stdout])
63

  
64
        if cp and cp.stderr:
65
            msg = '\n\n'.join([msg, _('STDERR_OUTPUT_heading'), cp.stderr])
66

  
67
        super().__init__(msg)
src/hydrilla/builder/local_apt.py
1
# SPDX-License-Identifier: AGPL-3.0-or-later
2

  
3
# Using a local APT.
4
#
5
# This file is part of Hydrilla
6
#
7
# Copyright (C) 2022 Wojtek Kosior
8
#
9
# This program is free software: you can redistribute it and/or modify
10
# it under the terms of the GNU Affero General Public License as
11
# published by the Free Software Foundation, either version 3 of the
12
# License, or (at your option) any later version.
13
#
14
# This program is distributed in the hope that it will be useful,
15
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
# GNU Affero General Public License for more details.
18
#
19
# You should have received a copy of the GNU Affero General Public License
20
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
21
#
22
#
23
# I, Wojtek Kosior, thereby promise not to sue for violation of this
24
# file's license. Although I request that you do not make use this code
25
# in a proprietary program, I am not going to enforce this in court.
26

  
27
# Enable using with Python 3.7.
28
from __future__ import annotations
29

  
30
import zipfile
31
import shutil
32
import re
33
import subprocess
34
CP = subprocess.CompletedProcess
35
from pathlib import Path, PurePosixPath
36
from tempfile import TemporaryDirectory, NamedTemporaryFile
37
from hashlib import sha256
38
from urllib.parse import unquote
39
from contextlib import contextmanager
40
from typing import Optional, Iterable
41

  
42
from .. import util
43
from .piggybacking import Piggybacked
44
from .common_errors import *
45

  
46
here = Path(__file__).resolve().parent
47

  
48
_ = util.translation(here / 'locales').gettext
49

  
50
"""
51
Default cache directory to save APT configurations and downloaded GPG keys in.
52
"""
53
default_apt_cache_dir = Path.home() / '.cache' / 'hydrilla' / 'builder' / 'apt'
54

  
55
"""
56
Default keyserver to use.
57
"""
58
default_keyserver = 'hkps://keyserver.ubuntu.com:443'
59

  
60
"""
61
Default keys to download when using a local APT.
62
"""
63
default_keys = [
64
    # Trisquel
65
    'E6C27099CA21965B734AEA31B4EFB9F38D8AEBF1',
66
    '60364C9869F92450421F0C22B138CA450C05112F',
67
    # Ubuntu
68
    '630239CC130E1A7FD81A27B140976EAF437D05B5',
69
    '790BC7277767219C42C86F933B4FE6ACC0B21F32',
70
    'F6ECB3762474EDA9D21B7022871920D1991BC93C',
71
    # Debian
72
    '6D33866EDD8FFA41C0143AEDDCC9EFBF77E11517',
73
    '80D15823B7FD1561F9F7BCDDDC30D7C23CBBABEE',
74
    'AC530D520F2F3269F5E98313A48449044AAD5C5D'
75
]
76

  
77
"""sources.list file contents for known distros."""
78
default_lists = {
79
    'nabia': [f'{type} http://archive.trisquel.info/trisquel/ nabia{suf} main'
80
              for type in ('deb', 'deb-src')
81
              for suf in ('', '-updates', '-security')]
82
}
83

  
84
class GpgError(Exception):
85
    """
86
    Exception used to report various problems when calling GPG.
87
    """
88

  
89
class AptError(SubprocessError):
90
    """
91
    Exception used to report various problems when calling apt-* and dpkg-*
92
    commands.
93
    """
94

  
95
def run(command, **kwargs):
96
    """A wrapped around subprocess.run that sets some default options."""
97
    return subprocess.run(command, **kwargs, env={'LANG': 'en_US'},
98
                          capture_output=True, text=True)
99

  
100
class Apt:
101
    """
102
    This class represents an APT instance and can be used to call apt-get
103
    commands with it.
104
    """
105
    def __init__(self, apt_conf: str) -> None:
106
        """Initialize this Apt object."""
107
        self.apt_conf = apt_conf
108

  
109
    def get(self, *args: str, **kwargs) -> CP:
110
        """
111
        Run apt-get with the specified arguments and raise a meaningful AptError
112
        when something goes wrong.
113
        """
114
        command = ['apt-get', '-c', self.apt_conf, *args]
115
        try:
116
            cp = run(command, **kwargs)
117
        except FileNotFoundError:
118
            msg = _('couldnt_execute_{}_is_it_installed').format('apt-get')
119
            raise AptError(msg)
120

  
121
        if cp.returncode != 0:
122
            msg = _('command_{}_failed').format(' '.join(command))
123
            raise AptError(msg, cp)
124

  
125
        return cp
126

  
127
def cache_dir() -> Path:
128
    """
129
    Return the directory used to cache data (APT configurations, keyrings) to
130
    speed up repeated operations.
131

  
132
    This function first ensures the directory exists.
133
    """
134
    default_apt_cache_dir.mkdir(parents=True, exist_ok=True)
135
    return default_apt_cache_dir
136

  
137
class SourcesList:
138
    """Representation of apt's sources.list contents."""
139
    def __init__(self, list: [str]=[], codename: Optional[str]=None) -> None:
140
        """Initialize this SourcesList."""
141
        self.codename = None
142
        self.list = [*list]
143
        self.has_extra_entries = bool(self.list)
144

  
145
        if codename is not None:
146
            if codename not in default_lists:
147
                raise DistroError(_('distro_{}_unknown').format(codename))
148

  
149
            self.codename = codename
150
            self.list.extend(default_lists[codename])
151

  
152
    def identity(self) -> str:
153
        """
154
        Produce a string that uniquely identifies this sources.list contents.
155
        """
156
        if self.codename and not self.has_extra_entries:
157
            return self.codename
158

  
159
        return sha256('\n'.join(sorted(self.list)).encode()).digest().hex()
160

  
161
def apt_conf(directory: Path) -> str:
162
    """
163
    Given local APT's directory, produce a configuration suitable for running
164
    APT there.
165

  
166
    'directory' must not contain any special characters including quotes and
167
    spaces.
168
    """
169
    return f'''
170
Architecture "amd64";
171
Dir "{directory}";
172
Dir::State "{directory}/var/lib/apt";
173
Dir::State::status "{directory}/var/lib/dpkg/status";
174
Dir::Etc::SourceList "{directory}/etc/apt.sources.list";
175
Dir::Etc::SourceParts "";
176
Dir::Cache "{directory}/var/cache/apt";
177
pkgCacheGen::Essential "none";
178
Dir::Etc::Trusted "{directory}/etc/trusted.gpg";
179
'''
180

  
181
def apt_keyring(keys: [str]) -> bytes:
182
    """
183
    Download the requested keys if necessary and export them as a keyring
184
    suitable for passing to APT.
185

  
186
    The keyring is returned as a bytes value that should be written to a file.
187
    """
188
    try:
189
        from gnupg import GPG
190
    except ModuleNotFoundError:
191
        raise GpgError(_('couldnt_import_{}_is_it_installed').format('gnupg'))
192

  
193
    gpg = GPG(keyring=str(cache_dir() / 'master_keyring.gpg'))
194
    for key in keys:
195
        if gpg.list_keys(keys=[key]) != []:
196
            continue
197

  
198
        if gpg.recv_keys(default_keyserver, key).imported == 0:
199
            raise GpgError(_('gpg_couldnt_recv_key_{}').format(key))
200

  
201
    return gpg.export_keys(keys, armor=False, minimal=True)
202

  
203
def cache_apt_root(apt_root: Path, destination_zip: Path) -> None:
204
    """
205
    Zip an APT root directory for later use and move the zipfile to the
206
    requested destination.
207
    """
208
    temporary_zip_path = None
209
    try:
210
        tmpfile = NamedTemporaryFile(suffix='.zip', prefix='tmp_',
211
                                     dir=cache_dir(), delete=False)
212
        temporary_zip_path = Path(tmpfile.name)
213

  
214
        to_skip = {Path('etc') / 'apt.conf', Path('etc') / 'trusted.gpg'}
215

  
216
        with zipfile.ZipFile(tmpfile, 'w') as zf:
217
            for member in apt_root.rglob('*'):
218
                relative = member.relative_to(apt_root)
219
                if relative not in to_skip:
220
                    # This call will also properly add empty folders to zip file
221
                    zf.write(member, relative, zipfile.ZIP_DEFLATED)
222

  
223
        shutil.move(temporary_zip_path, destination_zip)
224
    finally:
225
        if temporary_zip_path is not None and temporary_zip_path.exists():
226
            temporary_zip_path.unlink()
227

  
228
def setup_local_apt(directory: Path, list: SourcesList, keys: [str]) -> Apt:
229
    """
230
    Create files and directories necessary for running APT without root rights
231
    inside 'directory'.
232

  
233
    'directory' must not contain any special characters including quotes and
234
    spaces and must be empty.
235

  
236
    Return an Apt object that can be used to call apt-get commands.
237
    """
238
    apt_root = directory / 'apt_root'
239

  
240
    conf_text     = apt_conf(apt_root)
241
    keyring_bytes = apt_keyring(keys)
242

  
243
    apt_zipfile = cache_dir() / f'apt_{list.identity()}.zip'
244
    if apt_zipfile.exists():
245
        with zipfile.ZipFile(apt_zipfile) as zf:
246
            zf.extractall(apt_root)
247

  
248
    for to_create in (
249
            apt_root / 'var' / 'lib' / 'apt' / 'partial',
250
            apt_root / 'var' / 'lib' / 'apt' / 'lists',
251
            apt_root / 'var' / 'cache' / 'apt' / 'archives' / 'partial',
252
            apt_root / 'etc' / 'apt' / 'preferences.d',
253
            apt_root / 'var' / 'lib' / 'dpkg',
254
            apt_root / 'var' / 'log' / 'apt'
255
    ):
256
        to_create.mkdir(parents=True, exist_ok=True)
257

  
258
    conf_path    = apt_root / 'etc' / 'apt.conf'
259
    trusted_path = apt_root / 'etc' / 'trusted.gpg'
260
    status_path  = apt_root / 'var' / 'lib' / 'dpkg' / 'status'
261
    list_path    = apt_root / 'etc' / 'apt.sources.list'
262

  
263
    conf_path.write_text(conf_text)
264
    trusted_path.write_bytes(keyring_bytes)
265
    status_path.touch()
266
    list_path.write_text('\n'.join(list.list))
267

  
268
    apt = Apt(str(conf_path))
269
    apt.get('update')
270

  
271
    cache_apt_root(apt_root, apt_zipfile)
272

  
273
    return apt
274

  
275
@contextmanager
276
def local_apt(list: SourcesList, keys: [str]) -> Iterable[Apt]:
277
    """
278
    Create a temporary directory with proper local APT configuration in it.
279
    Yield an Apt object that can be used to issue apt-get commands.
280

  
281
    This function returns a context manager that will remove the directory on
282
    close.
283
    """
284
    with TemporaryDirectory() as td:
285
        td = Path(td)
286
        yield setup_local_apt(td, list, keys)
287

  
288
def download_apt_packages(list: SourcesList, keys: [str], packages: [str],
289
                          destination_dir: Path, with_deps: bool) -> [str]:
290
    """
291
    Set up a local APT, update it using the specified sources.list configuration
292
    and use it to download the specified packages.
293

  
294
    This function downloads .deb files of packages matching the amd64
295
    architecture (which includes packages with architecture 'all') as well as
296
    all their corresponding source package files and (if requested) the debs
297
    and source files of all their declared dependencies.
298

  
299
    Return value is a list of names of all downloaded files.
300
    """
301
    install_line_regex = re.compile(r'^Inst (?P<name>\S+) \((?P<version>\S+) ')
302

  
303
    with local_apt(list, keys) as apt:
304
        if with_deps:
305
            cp = apt.get('install', '--yes', '--just-print', *packages)
306

  
307
            lines = cp.stdout.split('\n')
308
            matches = [install_line_regex.match(l) for l in lines]
309
            packages = [f'{m.group("name")}={m.group("version")}'
310
                        for m in matches if m]
311

  
312
            if not packages:
313
                raise AptError(_('apt_install_output_not_understood'), cp)
314

  
315
        # Download .debs to indirectly to destination_dir by first placing them
316
        # in a temporary subdirectory.
317
        with TemporaryDirectory(dir=destination_dir) as td:
318
            td = Path(td)
319
            cp = apt.get('download', *packages, cwd=td)
320

  
321
            deb_name_regex = re.compile(
322
                r'''
323
                ^
324
                (?P<name>[^_]+)
325
                _
326
                (?P<ver>[^_]+)
327
                _
328
                .+              # architecture (or 'all')
329
                \.deb
330
                $
331
                ''',
332
                re.VERBOSE)
333

  
334
            names_vers = []
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff