Project

General

Profile

Download (16.7 KB) Statistics
| Branch: | Tag: | Revision:

hydrilla-builder / src / test / test_hydrilla_builder.py @ 136859ca

1
# SPDX-License-Identifier: CC0-1.0
2

    
3
# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
4
#
5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6

    
7
import pytest
8
import json
9
import shutil
10

    
11
from tempfile import TemporaryDirectory
12
from pathlib import Path
13
from hashlib import sha256, sha1
14
from zipfile import ZipFile
15
from typing import Callable, Optional, Iterable
16

    
17
from jsonschema import ValidationError
18

    
19
from hydrilla import util as hydrilla_util
20
from hydrilla.builder import build, _version
21

    
22
here = Path(__file__).resolve().parent
23

    
24
expected_generated_by = {
25
    'name': 'hydrilla.builder',
26
    'version': _version.version
27
}
28

    
29
default_srcdir = here / 'source-package-example'
30

    
31
default_js_filenames = ['bye.js', 'hello.js', 'message.js']
32
default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt']
33
default_src_filenames = [
34
    *default_dist_filenames,
35
    'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
36
]
37

    
38
default_sha1_hashes   = {}
39
default_sha256_hashes = {}
40
default_contents      = {}
41

    
42
for fn in default_src_filenames:
43
    with open(default_srcdir / fn, 'rb') as file_handle:
44
        default_contents[fn]      = file_handle.read()
45
        default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex()
46
        default_sha1_hashes[fn]   = sha1(default_contents[fn]).digest().hex()
47

    
48
class CaseSettings:
49
    """Gather parametrized values in a class."""
50
    def __init__(self):
51
        """Init CaseSettings with default values."""
52
        self.srcdir = default_srcdir
53
        self.index_json_path = Path('index.json')
54
        self.report_spdx_included = True
55

    
56
        self.js_filenames   = default_js_filenames.copy()
57
        self.dist_filenames = default_dist_filenames.copy()
58
        self.src_filenames  = default_src_filenames.copy()
59

    
60
        self.sha1_hashes   = default_sha1_hashes.copy()
61
        self.sha256_hashes = default_sha256_hashes.copy()
62
        self.contents      = default_contents.copy()
63

    
64
        self.expected_resources = [{
65
            '$schema': 'https://hydrilla.koszko.org/schemas/api_resource_description-1.schema.json',
66
            'source_name': 'hello',
67
            'source_copyright': [{
68
                'file': 'report.spdx',
69
                'sha256': '!!!!value to fill during test!!!!'
70
            }, {
71
                'file': 'LICENSES/CC0-1.0.txt',
72
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
73
            }],
74
            'type': 'resource',
75
            'identifier': 'helloapple',
76
            'long_name': 'Hello Apple',
77
            'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
78
            'version': [2021, 11, 10],
79
            'revision': 1,
80
            'description': 'greets an apple',
81
            'dependencies': [{'identifier': 'hello-message'}],
82
            'scripts': [{
83
                'file': 'hello.js',
84
                'sha256': self.sha256_hashes['hello.js']
85
            }, {
86
                'file': 'bye.js',
87
                'sha256': self.sha256_hashes['bye.js']
88
            }],
89
            'generated_by': expected_generated_by
90
        }, {
91
            '$schema': 'https://hydrilla.koszko.org/schemas/api_resource_description-1.schema.json',
92
            'source_name': 'hello',
93
            'source_copyright': [{
94
                'file': 'report.spdx',
95
                'sha256': '!!!!value to fill during test!!!!'
96
            }, {
97
                'file': 'LICENSES/CC0-1.0.txt',
98
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
99
            }],
100
            'type': 'resource',
101
            'identifier': 'hello-message',
102
            'long_name': 'Hello Message',
103
            'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
104
            'version': [2021, 11, 10],
105
            'revision': 2,
106
            'description': 'define messages for saying hello and bye',
107
            'dependencies': [],
108
            'scripts': [{
109
                'file': 'message.js',
110
                'sha256': self.sha256_hashes['message.js']
111
            }],
112
            'generated_by': expected_generated_by
113
        }]
114
        self.expected_mapping = {
115
            '$schema': 'https://hydrilla.koszko.org/schemas/api_mapping_description-1.schema.json',
116
            'source_name': 'hello',
117
            'source_copyright': [{
118
                'file': 'report.spdx',
119
                'sha256': '!!!!value to fill during test!!!!'
120
            }, {
121
                'file': 'LICENSES/CC0-1.0.txt',
122
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
123
            }],
124
            'type': 'mapping',
125
	    'identifier': 'helloapple',
126
	    'long_name': 'Hello Apple',
127
	    'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
128
	    'version': [2021, 11, 10],
129
	    'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
130
	    'payloads': {
131
	        'https://hydrillabugs.koszko.org/***': {
132
		    'identifier': 'helloapple'
133
	        },
134
	        'https://hachettebugs.koszko.org/***': {
135
		    'identifier': 'helloapple'
136
                }
137
            },
138
            'generated_by': expected_generated_by
139
        }
140
        self.expected_source_description = {
141
            '$schema': 'https://hydrilla.koszko.org/schemas/api_source_description-1.schema.json',
142
            'source_name': 'hello',
143
            'source_copyright': [{
144
                'file': 'report.spdx',
145
                'sha256': '!!!!value to fill during test!!!!'
146
            }, {
147
                'file': 'LICENSES/CC0-1.0.txt',
148
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
149
            }],
150
            'source_archives': {
151
                'zip': {
152
                    'sha256': '!!!!value to fill during test!!!!',
153
                }
154
            },
155
            'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
156
            'definitions': [{
157
                'type': 'resource',
158
                'identifier': 'helloapple',
159
                'long_name': 'Hello Apple',
160
                'version': [2021, 11, 10],
161
            }, {
162
                'type':       'resource',
163
                'identifier': 'hello-message',
164
                'long_name': 'Hello Message',
165
                'version':     [2021, 11, 10],
166
            }, {
167
                'type': 'mapping',
168
                'identifier': 'helloapple',
169
	        'long_name': 'Hello Apple',
170
                'version': [2021, 11, 10],
171
            }],
172
            'generated_by': expected_generated_by
173
        }
174

    
175
    def expected(self) -> list[dict]:
176
        """
177
        Convenience method to get a list of expected jsons of 2 resources,
178
        1 mapping and 1 source description we have.
179
        """
180
        return [
181
            *self.expected_resources,
182
            self.expected_mapping,
183
            self.expected_source_description
184
        ]
185

    
186
ModifyCb = Callable[[CaseSettings, dict], Optional[str]]
187

    
188
def prepare_modified(tmpdir: Path, modify_cb: ModifyCb) -> CaseSettings:
189
    """
190
    Use sample source package directory with an alternative, modified
191
    index.json.
192
    """
193
    settings = CaseSettings()
194

    
195
    for fn in settings.src_filenames:
196
        copy_path = tmpdir / 'srcdir_copy' / fn
197
        copy_path.parent.mkdir(parents=True, exist_ok=True)
198
        shutil.copy(settings.srcdir / fn, copy_path)
199

    
200
    settings.srcdir = tmpdir / 'srcdir_copy'
201

    
202
    with open(settings.srcdir / 'index.json', 'rt') as file_handle:
203
        obj = json.loads(hydrilla_util.strip_json_comments(file_handle.read()))
204

    
205
    contents = modify_cb(settings, obj)
206

    
207
    # Replace the other index.json with new one
208
    settings.index_json_path = tmpdir / 'replacement.json'
209

    
210
    if contents is None:
211
        contents = json.dumps(obj)
212

    
213
    contents = contents.encode()
214

    
215
    settings.contents['index.json'] = contents
216

    
217
    settings.sha256_hashes['index.json'] = sha256(contents).digest().hex()
218
    settings.sha1_hashes['index.json']   = sha1(contents).digest().hex()
219

    
220
    with open(settings.index_json_path, 'wb') as file_handle:
221
        file_handle.write(contents)
222

    
223
    return settings
224

    
225
@pytest.fixture()
226
def tmpdir() -> Iterable[str]:
227
    with TemporaryDirectory() as tmpdir:
228
        yield tmpdir
229

    
230
def prepare_default(tmpdir: Path) -> CaseSettings:
231
    """Use sample source package directory as exists in VCS."""
232
    return CaseSettings()
233

    
234
def modify_index_good(settings: CaseSettings, obj: dict) -> None:
235
    """
236
    Modify index.json object to make a slightly different but *also correct* one
237
    that can be used to test some different cases.
238
    """
239
    # Add comments that should be preserved.
240
    for dictionary in (obj, settings.expected_source_description):
241
        dictionary['comment'] = 'index_json comment'
242

    
243
    for i, dicts in enumerate(zip(obj['definitions'], settings.expected())):
244
        for dictionary in dicts:
245
            dictionary['comment'] = f'item {i}'
246

    
247
    # Remove spdx report generation
248
    del obj['reuse_generate_spdx_report']
249
    obj['copyright'].remove({'file': 'report.spdx'})
250

    
251
    settings.report_spdx_included = False
252

    
253
    for json_description in settings.expected():
254
        json_description['source_copyright'] = \
255
            [fr for fr in json_description['source_copyright']
256
             if fr['file'] != 'report.spdx']
257

    
258
    # Use default value ([]) for 'additionall_files' property
259
    del obj['additional_files']
260

    
261
    settings.src_filenames = [*settings.dist_filenames, 'index.json']
262

    
263
    # Use default value ([]) for 'scripts' property in one of the resources
264
    del obj['definitions'][1]['scripts']
265

    
266
    settings.expected_resources[1]['scripts'] = []
267

    
268
    for prefix in ('js', 'dist', 'src'):
269
        getattr(settings, f'{prefix}_filenames').remove('message.js')
270

    
271
    # Use default value ({}) for 'pyloads' property in mapping
272
    del obj['definitions'][2]['payloads']
273

    
274
    settings.expected_mapping['payloads'] = {}
275

    
276
    # Add some unrecognized properties that should be stripped
277
    to_process = [obj]
278
    while to_process:
279
        processed = to_process.pop()
280

    
281
        if type(processed) is list:
282
            to_process.extend(processed)
283
        elif type(processed) is dict and 'spurious_property' not in processed:
284
            to_process.extend(processed.values())
285
            processed['spurious_property'] = 'some value'
286

    
287
@pytest.mark.parametrize('prepare_source_example', [
288
    prepare_default,
289
    lambda tmpdir: prepare_modified(tmpdir, modify_index_good)
290
])
291
def test_build(tmpdir, prepare_source_example):
292
    """Build the sample source package and verify the produced files."""
293
    # First, build the package
294
    dstdir = Path(tmpdir) / 'dstdir'
295
    tmpdir = Path(tmpdir) / 'example'
296

    
297
    dstdir.mkdir(exist_ok=True)
298
    tmpdir.mkdir(exist_ok=True)
299

    
300
    settings = prepare_source_example(tmpdir)
301

    
302
    build.Build(settings.srcdir, settings.index_json_path)\
303
        .write_package_files(dstdir)
304

    
305
    # Verify directories under destination directory
306
    assert {'file', 'resource', 'mapping', 'source'} == \
307
        set([path.name for path in dstdir.iterdir()])
308

    
309
    # Verify files under 'file/'
310
    file_dir = dstdir / 'file' / 'sha256'
311

    
312
    for fn in settings.dist_filenames:
313
        dist_file_path = file_dir / settings.sha256_hashes[fn]
314
        assert dist_file_path.is_file()
315

    
316
        assert dist_file_path.read_bytes() == settings.contents[fn]
317

    
318
    sha256_hashes_set = set([settings.sha256_hashes[fn]
319
                             for fn in settings.dist_filenames])
320

    
321
    spdx_report_sha256 = None
322

    
323
    for path in file_dir.iterdir():
324
        if path.name in sha256_hashes_set:
325
            continue
326

    
327
        assert spdx_report_sha256 is None and settings.report_spdx_included
328

    
329
        with open(path, 'rt') as file_handle:
330
            spdx_contents = file_handle.read()
331

    
332
        spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex()
333
        assert spdx_report_sha256 == path.name
334

    
335
        for fn in settings.src_filenames:
336
            if not any([n in fn.lower() for n in ('license', 'reuse')]):
337
                assert settings.sha1_hashes[fn]
338

    
339
    if settings.report_spdx_included:
340
        assert spdx_report_sha256
341
        for obj in settings.expected():
342
            for file_ref in obj['source_copyright']:
343
                if file_ref['file'] == 'report.spdx':
344
                    file_ref['sha256'] = spdx_report_sha256
345

    
346
    # Verify files under 'resource/'
347
    resource_dir = dstdir / 'resource'
348

    
349
    assert set([rj['identifier'] for rj in settings.expected_resources]) == \
350
        set([path.name for path in resource_dir.iterdir()])
351

    
352
    for resource_json in settings.expected_resources:
353
        subdir = resource_dir / resource_json['identifier']
354
        assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
355

    
356
        with open(subdir / '2021.11.10', 'rt') as file_handle:
357
            assert json.load(file_handle) == resource_json
358

    
359
        hydrilla_util.validator_for('api_resource_description-1.schema.json')\
360
                     .validate(resource_json)
361

    
362
    # Verify files under 'mapping/'
363
    mapping_dir = dstdir / 'mapping'
364
    assert ['helloapple'] == [path.name for path in mapping_dir.iterdir()]
365

    
366
    subdir = mapping_dir / 'helloapple'
367
    assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
368

    
369
    with open(subdir / '2021.11.10', 'rt') as file_handle:
370
        assert json.load(file_handle) == settings.expected_mapping
371

    
372
    hydrilla_util.validator_for('api_mapping_description-1.schema.json')\
373
                 .validate(settings.expected_mapping)
374

    
375
    # Verify files under 'source/'
376
    source_dir = dstdir / 'source'
377
    assert {'hello.json', 'hello.zip'} == \
378
        set([path.name for path in source_dir.iterdir()])
379

    
380
    zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames]
381

    
382
    with ZipFile(source_dir / 'hello.zip', 'r') as archive:
383
        assert set([f.filename for f in archive.filelist]) == set(zip_filenames)
384

    
385
        for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames):
386
            with archive.open(zip_fn, 'r') as zip_file_handle:
387
                assert zip_file_handle.read() == settings.contents[src_fn]
388

    
389
    zip_ref = settings.expected_source_description['source_archives']['zip']
390
    with open(source_dir / 'hello.zip', 'rb') as file_handle:
391
        zip_ref['sha256'] = sha256(file_handle.read()).digest().hex()
392

    
393
    with open(source_dir / 'hello.json', 'rt') as file_handle:
394
        assert json.load(file_handle) == settings.expected_source_description
395

    
396
    hydrilla_util.validator_for('api_source_description-1.schema.json')\
397
                 .validate(settings.expected_source_description)
398

    
399
def modify_index_missing_file(dummy: CaseSettings, obj: dict) -> None:
400
    """
401
    Modify index.json to expect missing report.spdx file and cause an error.
402
    """
403
    del obj['reuse_generate_spdx_report']
404

    
405
def modify_index_schema_error(dummy: CaseSettings, obj: dict) -> None:
406
    """Modify index.json to be incompliant with the schema."""
407
    del obj['definitions']
408

    
409
def modify_index_bad_comment(dummy: CaseSettings, obj: dict) -> str:
410
    """Modify index.json to have an invalid '/' in it."""
411
    return json.dumps(obj) + '/something\n'
412

    
413
def modify_index_bad_json(dummy: CaseSettings, obj: dict) -> str:
414
    """Modify index.json to not be valid json even after comment stripping."""
415
    return json.dumps(obj) + '???/\n'
416

    
417
def modify_index_missing_license(settings: CaseSettings, obj: dict) -> None:
418
    """Remove a file to make package REUSE-incompliant."""
419
    (settings.srcdir / 'README.txt.license').unlink()
420

    
421
def modify_index_file_outside(dummy: CaseSettings, obj: dict) -> None:
422
    """Make index.json illegally reference a file outside srcdir."""
423
    obj['copyright'].append({'file': '../abc'})
424

    
425
def modify_index_reference_itself(dummy: CaseSettings, obj: dict) -> None:
426
    """Make index.json illegally reference index.json."""
427
    obj['copyright'].append({'file': 'index.json'})
428

    
429
def modify_index_report_excluded(dummy: CaseSettings, obj: dict) -> None:
430
    """
431
    Make index.json require generation of index.json but not include it among
432
    copyright files.
433
    """
434
    obj['copyright'] = [fr for fr in obj['copyright']
435
                        if fr['file'] != 'report.spdx']
436

    
437
@pytest.mark.parametrize('break_index_json', [
438
    (modify_index_missing_file,     FileNotFoundError),
439
    (modify_index_schema_error,     ValidationError),
440
    (modify_index_bad_comment,      json.JSONDecodeError),
441
    (modify_index_bad_json,         json.JSONDecodeError),
442
    (modify_index_missing_license,  build.ReuseError),
443
    (modify_index_file_outside,     build.FileReferenceError),
444
    (modify_index_reference_itself, build.FileReferenceError),
445
    (modify_index_report_excluded,  build.FileReferenceError)
446
])
447
def test_build_error(tmpdir: str, break_index_json: tuple[ModifyCb, type]):
448
    """Build the sample source package and verify the produced files."""
449
    dstdir = Path(tmpdir) / 'dstdir'
450
    tmpdir = Path(tmpdir) / 'example'
451

    
452
    dstdir.mkdir(exist_ok=True)
453
    tmpdir.mkdir(exist_ok=True)
454

    
455
    modify_cb, error_type = break_index_json
456

    
457
    settings = prepare_modified(tmpdir, modify_cb)
458

    
459
    with pytest.raises(error_type):
460
        build.Build(settings.srcdir, settings.index_json_path)\
461
            .write_package_files(dstdir)
(2-2/2)