Project

General

Profile

« Previous | Next » 

Revision 61f0aa75

Added by koszko over 1 year ago

support piggybacking on APT packages

View differences:

src/hydrilla/builder/build.py
30 30
import json
31 31
import re
32 32
import zipfile
33
from pathlib import Path
33
import subprocess
34
from pathlib import Path, PurePosixPath
34 35
from hashlib import sha256
35 36
from sys import stderr
37
from contextlib import contextmanager
38
from tempfile import TemporaryDirectory, TemporaryFile
39
from typing import Optional, Iterable, Union
36 40

  
37 41
import jsonschema
38 42
import click
39 43

  
40 44
from .. import util
41 45
from . import _version
46
from . import local_apt
47
from .piggybacking import Piggybacked
48
from .common_errors import *
42 49

  
43 50
here = Path(__file__).resolve().parent
44 51

  
45 52
_ = util.translation(here / 'locales').gettext
46 53

  
47
index_validator = util.validator_for('package_source-1.0.1.schema.json')
54
index_validator = util.validator_for('package_source-2.schema.json')
48 55

  
49 56
schemas_root = 'https://hydrilla.koszko.org/schemas'
50 57

  
......
53 60
    'version': _version.version
54 61
}
55 62

  
56
class FileReferenceError(Exception):
57
    """
58
    Exception used to report various problems concerning files referenced from
59
    source package's index.json.
60
    """
61

  
62
class ReuseError(Exception):
63
class ReuseError(SubprocessError):
63 64
    """
64 65
    Exception used to report various problems when calling the REUSE tool.
65 66
    """
66 67

  
67
class FileBuffer:
68
    """
69
    Implement a file-like object that buffers data written to it.
70
    """
71
    def __init__(self):
72
        """
73
        Initialize FileBuffer.
74
        """
75
        self.chunks = []
76

  
77
    def write(self, b):
78
        """
79
        Buffer 'b', return number of bytes buffered.
80

  
81
        'b' is expected to be an instance of 'bytes' or 'str', in which case it
82
        gets encoded as UTF-8.
83
        """
84
        if type(b) is str:
85
            b = b.encode()
86
        self.chunks.append(b)
87
        return len(b)
88

  
89
    def flush(self):
90
        """
91
        A no-op mock of file-like object's flush() method.
92
        """
93
        pass
94

  
95
    def get_bytes(self):
96
        """
97
        Return all data written so far concatenated into a single 'bytes'
98
        object.
99
        """
100
        return b''.join(self.chunks)
101

  
102
def generate_spdx_report(root):
68
def generate_spdx_report(root: Path) -> bytes:
103 69
    """
104 70
    Use REUSE tool to generate an SPDX report for sources under 'root' and
105 71
    return the report's contents as 'bytes'.
106 72

  
107
    'root' shall be an instance of pathlib.Path.
108

  
109 73
    In case the directory tree under 'root' does not constitute a
110
    REUSE-compliant package, linting report is printed to standard output and
111
    an exception is raised.
74
    REUSE-compliant package, as exception is raised with linting report
75
    included in it.
112 76

  
113
    In case the reuse package is not installed, an exception is also raised.
77
    In case the reuse tool is not installed, an exception is also raised.
114 78
    """
115
    try:
116
        from reuse._main import main as reuse_main
117
    except ModuleNotFoundError:
118
        raise ReuseError(_('couldnt_import_reuse_is_it_installed'))
79
    for command in [
80
            ['reuse', '--root', str(root), 'lint'],
81
            ['reuse', '--root', str(root), 'spdx']
82
    ]:
83
        try:
84
            cp = subprocess.run(command, capture_output=True, text=True)
85
        except FileNotFoundError:
86
            raise ReuseError(_('couldnt_execute_reuse_is_it_installed'))
119 87

  
120
    mocked_output = FileBuffer()
121
    if reuse_main(args=['--root', str(root), 'lint'], out=mocked_output) != 0:
122
        stderr.write(mocked_output.get_bytes().decode())
123
        raise ReuseError(_('spdx_report_from_reuse_incompliant'))
88
        if cp.returncode != 0:
89
            msg = _('reuse_command_{}_failed').format(' '.join(command))
90
            raise ReuseError(msg, cp)
124 91

  
125
    mocked_output = FileBuffer()
126
    if reuse_main(args=['--root', str(root), 'spdx'], out=mocked_output) != 0:
127
        stderr.write(mocked_output.get_bytes().decode())
128
        raise ReuseError("Couldn't generate an SPDX report for package.")
129

  
130
    return mocked_output.get_bytes()
92
    return cp.stdout.encode()
131 93

  
132 94
class FileRef:
133 95
    """Represent reference to a file in the package."""
134
    def __init__(self, path: Path, contents: bytes):
96
    def __init__(self, path: PurePosixPath, contents: bytes) -> None:
135 97
        """Initialize FileRef."""
136
        self.include_in_distribution = False
137
        self.include_in_zipfile      = True
138
        self.path                    = path
139
        self.contents                = contents
98
        self.include_in_distribution   = False
99
        self.include_in_source_archive = True
100
        self.path                      = path
101
        self.contents                  = contents
140 102

  
141 103
        self.contents_hash = sha256(contents).digest().hex()
142 104

  
143
    def make_ref_dict(self, filename: str):
105
    def make_ref_dict(self) -> dict[str, str]:
144 106
        """
145 107
        Represent the file reference through a dict that can be included in JSON
146 108
        defintions.
147 109
        """
148 110
        return {
149
            'file':   filename,
111
            'file':   str(self.path),
150 112
            'sha256': self.contents_hash
151 113
        }
152 114

  
115
@contextmanager
116
def piggybacked_system(piggyback_def: Optional[dict],
117
                       piggyback_files: Optional[Path]) \
118
                       -> Iterable[Piggybacked]:
119
    """
120
    Resolve resources from a foreign software packaging system. Optionally, use
121
    package files (.deb's, etc.) from a specified directory instead of resolving
122
    and downloading them.
123
    """
124
    if piggyback_def is None:
125
        yield Piggybacked()
126
    else:
127
        # apt is the only supported system right now
128
        assert piggyback_def['system'] == 'apt'
129

  
130
        with local_apt.piggybacked_system(piggyback_def, piggyback_files) \
131
             as piggybacked:
132
            yield piggybacked
133

  
153 134
class Build:
154 135
    """
155 136
    Build a Hydrilla package.
156 137
    """
157
    def __init__(self, srcdir, index_json_path):
138
    def __init__(self, srcdir: Path, index_json_path: Path,
139
                 piggyback_files: Optional[Path]=None):
158 140
        """
159 141
        Initialize a build. All files to be included in a distribution package
160 142
        are loaded into memory, all data gets validated and all necessary
161 143
        computations (e.g. preparing of hashes) are performed.
162

  
163
        'srcdir' and 'index_json' are expected to be pathlib.Path objects.
164 144
        """
165 145
        self.srcdir          = srcdir.resolve()
166
        self.index_json_path = index_json_path
146
        self.piggyback_files = piggyback_files
147
        # TODO: the piggyback files we set are ignored for now; use them
148
        if piggyback_files is None:
149
            piggyback_default_path = \
150
                srcdir.parent / f'{srcdir.name}.foreign-packages'
151
            if piggyback_default_path.exists():
152
                self.piggyback_files = piggyback_default_path
167 153
        self.files_by_path   = {}
168 154
        self.resource_list   = []
169 155
        self.mapping_list    = []
170 156

  
171 157
        if not index_json_path.is_absolute():
172
            self.index_json_path = (self.srcdir / self.index_json_path)
173

  
174
        self.index_json_path = self.index_json_path.resolve()
158
            index_json_path = (self.srcdir / index_json_path)
175 159

  
176
        with open(self.index_json_path, 'rt') as index_file:
160
        with open(index_json_path, 'rt') as index_file:
177 161
            index_json_text = index_file.read()
178 162

  
179 163
        index_obj = json.loads(util.strip_json_comments(index_json_text))
180 164

  
181
        self.files_by_path[self.srcdir / 'index.json'] = \
182
            FileRef(self.srcdir / 'index.json', index_json_text.encode())
165
        index_desired_path = PurePosixPath('index.json')
166
        self.files_by_path[index_desired_path] = \
167
            FileRef(index_desired_path, index_json_text.encode())
183 168

  
184 169
        self._process_index_json(index_obj)
185 170

  
186
    def _process_file(self, filename: str, include_in_distribution: bool=True):
171
    def _process_file(self, filename: Union[str, PurePosixPath],
172
                      piggybacked: Piggybacked,
173
                      include_in_distribution: bool=True):
187 174
        """
188 175
        Resolve 'filename' relative to srcdir, load it to memory (if not loaded
189 176
        before), compute its hash and store its information in
190 177
        'self.files_by_path'.
191 178

  
192
        'filename' shall represent a relative path using '/' as a separator.
179
        'filename' shall represent a relative path withing package directory.
193 180

  
194 181
        if 'include_in_distribution' is True it shall cause the file to not only
195 182
        be included in the source package's zipfile, but also written as one of
196 183
        built package's files.
197 184

  
185
        For each file an attempt is made to resolve it using 'piggybacked'
186
        object. If a file is found and pulled from foreign software packaging
187
        system this way, it gets automatically excluded from inclusion in
188
        Hydrilla source package's zipfile.
189

  
198 190
        Return file's reference object that can be included in JSON defintions
199 191
        of various kinds.
200 192
        """
201
        path = self.srcdir
202
        for segment in filename.split('/'):
203
            path /= segment
204

  
205
        path = path.resolve()
206
        if not path.is_relative_to(self.srcdir):
207
            raise FileReferenceError(_('loading_{}_outside_package_dir')
208
                                     .format(filename))
209

  
210
        if str(path.relative_to(self.srcdir)) == 'index.json':
211
            raise FileReferenceError(_('loading_reserved_index_json'))
193
        include_in_source_archive = True
194

  
195
        desired_path = PurePosixPath(filename)
196
        if '..' in desired_path.parts:
197
            msg = _('path_contains_double_dot_{}').format(filename)
198
            raise FileReferenceError(msg)
199

  
200
        path = piggybacked.resolve_file(desired_path)
201
        if path is None:
202
            path = (self.srcdir / desired_path).resolve()
203
            if not path.is_relative_to(self.srcdir):
204
                raise FileReferenceError(_('loading_{}_outside_package_dir')
205
                                         .format(filename))
206

  
207
            if str(path.relative_to(self.srcdir)) == 'index.json':
208
                raise FileReferenceError(_('loading_reserved_index_json'))
209
        else:
210
            include_in_source_archive = False
212 211

  
213
        file_ref = self.files_by_path.get(path)
212
        file_ref = self.files_by_path.get(desired_path)
214 213
        if file_ref is None:
215 214
            with open(path, 'rb') as file_handle:
216 215
                contents = file_handle.read()
217 216

  
218
            file_ref = FileRef(path, contents)
219
            self.files_by_path[path] = file_ref
217
            file_ref = FileRef(desired_path, contents)
218
            self.files_by_path[desired_path] = file_ref
220 219

  
221 220
        if include_in_distribution:
222 221
            file_ref.include_in_distribution = True
223 222

  
224
        return file_ref.make_ref_dict(filename)
223
        if not include_in_source_archive:
224
            file_ref.include_in_source_archive = False
225

  
226
        return file_ref.make_ref_dict()
225 227

  
226
    def _prepare_source_package_zip(self, root_dir_name: str):
228
    def _prepare_source_package_zip(self, source_name: str,
229
                                    piggybacked: Piggybacked) -> str:
227 230
        """
228 231
        Create and store in memory a .zip archive containing files needed to
229 232
        build this source package.
230 233

  
231
        'root_dir_name' shall not contain any slashes ('/').
234
        'src_dir_name' shall not contain any slashes ('/').
232 235

  
233 236
        Return zipfile's sha256 sum's hexstring.
234 237
        """
235
        fb = FileBuffer()
236
        root_dir_path = Path(root_dir_name)
238
        tf = TemporaryFile()
239
        source_dir_path      = PurePosixPath(source_name)
240
        piggybacked_dir_path = PurePosixPath(f'{source_name}.foreign-packages')
237 241

  
238
        def zippath(file_path):
239
            file_path = root_dir_path / file_path.relative_to(self.srcdir)
240
            return file_path.as_posix()
241

  
242
        with zipfile.ZipFile(fb, 'w') as xpi:
242
        with zipfile.ZipFile(tf, 'w') as zf:
243 243
            for file_ref in self.files_by_path.values():
244
                if file_ref.include_in_zipfile:
245
                    xpi.writestr(zippath(file_ref.path), file_ref.contents)
244
                if file_ref.include_in_source_archive:
245
                    zf.writestr(str(source_dir_path / file_ref.path),
246
                                file_ref.contents)
247

  
248
            for desired_path, real_path in piggybacked.archive_files():
249
                zf.writestr(str(piggybacked_dir_path / desired_path),
250
                            real_path.read_bytes())
246 251

  
247
        self.source_zip_contents = fb.get_bytes()
252
        tf.seek(0)
253
        self.source_zip_contents = tf.read()
248 254

  
249 255
        return sha256(self.source_zip_contents).digest().hex()
250 256

  
251
    def _process_item(self, item_def: dict):
257
    def _process_item(self, item_def: dict, piggybacked: Piggybacked):
252 258
        """
253 259
        Process 'item_def' as definition of a resource/mapping and store in
254 260
        memory its processed form and files used by it.
......
266 272

  
267 273
            copy_props.append('revision')
268 274

  
269
            script_file_refs = [self._process_file(f['file'])
275
            script_file_refs = [self._process_file(f['file'], piggybacked)
270 276
                                for f in item_def.get('scripts', [])]
271 277

  
272 278
            deps = [{'identifier': res_ref['identifier']}
273 279
                    for res_ref in item_def.get('dependencies', [])]
274 280

  
275 281
            new_item_obj = {
276
                'dependencies': deps,
282
                'dependencies': [*piggybacked.package_must_depend, *deps],
277 283
                'scripts':      script_file_refs
278 284
            }
279 285
        else:
......
308 314
        in it.
309 315
        """
310 316
        index_validator.validate(index_obj)
317
        match = re.match(r'.*-((([1-9][0-9]*|0)\.)+)schema\.json$',
318
                         index_obj['$schema'])
319
        self.source_schema_ver = \
320
            [int(n) for n in filter(None, match.group(1).split('.'))]
311 321

  
312
        schema = f'{schemas_root}/api_source_description-1.schema.json'
322
        out_schema = f'{schemas_root}/api_source_description-1.schema.json'
313 323

  
314 324
        self.source_name = index_obj['source_name']
315 325

  
316 326
        generate_spdx = index_obj.get('reuse_generate_spdx_report', False)
317 327
        if generate_spdx:
318 328
            contents  = generate_spdx_report(self.srcdir)
319
            spdx_path = (self.srcdir / 'report.spdx').resolve()
329
            spdx_path = PurePosixPath('report.spdx')
320 330
            spdx_ref  = FileRef(spdx_path, contents)
321 331

  
322
            spdx_ref.include_in_zipfile = False
332
            spdx_ref.include_in_source_archive = False
323 333
            self.files_by_path[spdx_path] = spdx_ref
324 334

  
325
        self.copyright_file_refs = \
326
            [self._process_file(f['file']) for f in index_obj['copyright']]
335
        piggyback_def = None
336
        if self.source_schema_ver >= [1, 1] and 'piggyback_on' in index_obj:
337
            piggyback_def = index_obj['piggyback_on']
327 338

  
328
        if generate_spdx and not spdx_ref.include_in_distribution:
329
            raise FileReferenceError(_('report_spdx_not_in_copyright_list'))
339
        with piggybacked_system(piggyback_def, self.piggyback_files) \
340
             as piggybacked:
341
            copyright_to_process = [
342
                *(file_ref['file'] for file_ref in index_obj['copyright']),
343
                *piggybacked.package_license_files
344
            ]
345
            self.copyright_file_refs = [self._process_file(f, piggybacked)
346
                                        for f in copyright_to_process]
330 347

  
331
        item_refs = [self._process_item(d) for d in index_obj['definitions']]
348
            if generate_spdx and not spdx_ref.include_in_distribution:
349
                raise FileReferenceError(_('report_spdx_not_in_copyright_list'))
332 350

  
333
        for file_ref in index_obj.get('additional_files', []):
334
            self._process_file(file_ref['file'], include_in_distribution=False)
351
            item_refs = [self._process_item(d, piggybacked)
352
                         for d in index_obj['definitions']]
335 353

  
336
        root_dir_path = Path(self.source_name)
354
            for file_ref in index_obj.get('additional_files', []):
355
                self._process_file(file_ref['file'], piggybacked,
356
                                   include_in_distribution=False)
337 357

  
338
        source_archives_obj = {
339
            'zip' : {
340
                'sha256': self._prepare_source_package_zip(root_dir_path)
341
            }
342
        }
358
            zipfile_sha256 = self._prepare_source_package_zip\
359
                (self.source_name, piggybacked)
360

  
361
            source_archives_obj = {'zip' : {'sha256': zipfile_sha256}}
343 362

  
344 363
        self.source_description = {
345
            '$schema':            schema,
364
            '$schema':            out_schema,
346 365
            'source_name':        self.source_name,
347 366
            'source_copyright':   self.copyright_file_refs,
348 367
            'upstream_url':       index_obj['upstream_url'],
......
398 417

  
399 418
dir_type = click.Path(exists=True, file_okay=False, resolve_path=True)
400 419

  
420
@click.command(help=_('build_package_from_srcdir_to_dstdir'))
401 421
@click.option('-s', '--srcdir', default='./', type=dir_type, show_default=True,
402 422
              help=_('source_directory_to_build_from'))
403 423
@click.option('-i', '--index-json', default='index.json', type=click.Path(),
404 424
              help=_('path_instead_of_index_json'))
425
@click.option('-p', '--piggyback-files', type=click.Path(),
426
              help=_('path_instead_for_piggyback_files'))
405 427
@click.option('-d', '--dstdir', type=dir_type, required=True,
406 428
              help=_('built_package_files_destination'))
407 429
@click.version_option(version=_version.version, prog_name='Hydrilla builder',
408 430
                      message=_('%(prog)s_%(version)s_license'),
409 431
                      help=_('version_printing'))
410
def perform(srcdir, index_json, dstdir):
411
    """<this will be replaced by a localized docstring for Click to pick up>"""
412
    build = Build(Path(srcdir), Path(index_json))
413
    build.write_package_files(Path(dstdir))
414

  
415
perform.__doc__ = _('build_package_from_srcdir_to_dstdir')
432
def perform(srcdir, index_json, piggyback_files, dstdir):
433
    """
434
    Execute Hydrilla builder to turn source package into a distributable one.
416 435

  
417
perform = click.command()(perform)
436
    This command is meant to be the entry point of hydrilla-builder command
437
    exported by this package.
438
    """
439
    build = Build(Path(srcdir), Path(index_json),
440
                  piggyback_files and Path(piggyback_files))
441
    build.write_package_files(Path(dstdir))

Also available in: Unified diff