Project

General

Profile

« Previous | Next » 

Revision 8a036bc7

Added by koszko over 1 year ago

add test case with modified index.json from the example

View differences:

src/hydrilla_builder/build.py
223 223

  
224 224
        index_obj = json.loads(strip_json_comments(index_json_text))
225 225

  
226
        self.files_by_path[self.index_json_path] = \
227
            FileRef(self.index_json_path, index_json_text.encode())
226
        self.files_by_path[self.srcdir / 'index.json'] = \
227
            FileRef(self.srcdir / 'index.json', index_json_text.encode())
228 228

  
229 229
        self._process_index_json(index_obj)
230 230

  
......
302 302
        """
303 303
        copy_props = ['type', 'identifier', 'long_name', 'uuid', 'description']
304 304
        if 'comment' in item_def:
305
            copy_props.append(copy_props)
305
            copy_props.append('comment')
306 306

  
307 307
        if item_def['type'] == 'resource':
308 308
            item_list = self.resource_list
......
319 319
        else:
320 320
            item_list = self.mapping_list
321 321

  
322
            payloads = dict([(pat, {'identifier': res_ref['identifier']})
323
                             for pat, res_ref in item_def['payloads'].items()])
322
            payloads = {}
323
            for pat, res_ref in item_def.get('payloads', {}).items():
324
                payloads[pat] = {'identifier': res_ref['identifier']}
324 325

  
325 326
            new_item_obj = {
326 327
                'payloads': payloads
......
386 387
            'source_archives':    source_archives_obj
387 388
        }
388 389

  
390
        if 'comment' in index_obj:
391
            self.source_description['comment'] = index_obj['comment']
392

  
389 393
    def write_source_package_zip(self, dstpath: Path):
390 394
        """
391 395
        Create a .zip archive containing files needed to build this source
src/hydrilla_builder/schemas
1
Subproject commit 7ce7ee1a4821b23909c07491798083c4b3639f98
1
Subproject commit ca1de2ed4a69a71f2f75552ade693d04ea1baa85
src/test/test_hydrilla_builder.py
14 14

  
15 15
here = Path(__file__).resolve().parent
16 16

  
17
default_srcdir = here / 'source-package-example'
18

  
19
default_js_filenames = ['bye.js', 'hello.js', 'message.js']
20
default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt']
21
default_src_filenames = [
22
    *default_dist_filenames,
23
    'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
24
]
25

  
26
default_sha1_hashes   = {}
27
default_sha256_hashes = {}
28
default_contents      = {}
29

  
30
for fn in default_src_filenames:
31
    with open(default_srcdir / fn, 'rb') as file_handle:
32
        default_contents[fn]      = file_handle.read()
33
        default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex()
34
        default_sha1_hashes[fn]   = sha1(default_contents[fn]).digest().hex()
35

  
36
class CaseSettings:
37
    """Gather parametrized values in a class."""
38
    def __init__(self):
39
        """Init CaseSettings with default values."""
40
        self.srcdir = default_srcdir
41
        self.index_json_path = Path('index.json')
42
        self.report_spdx_included = True
43

  
44
        self.js_filenames   = default_js_filenames.copy()
45
        self.dist_filenames = default_dist_filenames.copy()
46
        self.src_filenames  = default_src_filenames.copy()
47

  
48
        self.sha1_hashes   = default_sha1_hashes.copy()
49
        self.sha256_hashes = default_sha256_hashes.copy()
50
        self.contents      = default_contents.copy()
51

  
52
        self.expected_resources = [{
53
            'api_schema_version': [1, 0, 1],
54
            'source_name': 'hello',
55
            'source_copyright': [{
56
                'file': 'report.spdx',
57
                'sha256': '!!!!value to fill during test!!!!'
58
            }, {
59
                'file': 'LICENSES/CC0-1.0.txt',
60
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
61
            }],
62
            'type': 'resource',
63
            'identifier': 'helloapple',
64
            'long_name': 'Hello Apple',
65
            'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
66
            'version': [2021, 11, 10],
67
            'revision': 1,
68
            'description': 'greets an apple',
69
            'dependencies': ['hello-message'],
70
            'scripts': [{
71
                'file': 'hello.js',
72
                'sha256': self.sha256_hashes['hello.js']
73
            }, {
74
                'file': 'bye.js',
75
                'sha256': self.sha256_hashes['bye.js']
76
            }]
77
        }, {
78
            'api_schema_version': [1, 0, 1],
79
            'source_name': 'hello',
80
            'source_copyright': [{
81
                'file': 'report.spdx',
82
                'sha256': '!!!!value to fill during test!!!!'
83
            }, {
84
                'file': 'LICENSES/CC0-1.0.txt',
85
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
86
            }],
87
            'type': 'resource',
88
            'identifier': 'hello-message',
89
            'long_name': 'Hello Message',
90
            'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
91
            'version': [2021, 11, 10],
92
            'revision': 2,
93
            'description': 'define messages for saying hello and bye',
94
            'dependencies': [],
95
            'scripts': [{
96
                'file': 'message.js',
97
                'sha256': self.sha256_hashes['message.js']
98
            }]
99
        }]
100
        self.expected_mapping = {
101
            'api_schema_version': [1, 0, 1],
102
            'source_name': 'hello',
103
            'source_copyright': [{
104
                'file': 'report.spdx',
105
                'sha256': '!!!!value to fill during test!!!!'
106
            }, {
107
                'file': 'LICENSES/CC0-1.0.txt',
108
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
109
            }],
110
            'type': 'mapping',
111
	    'identifier': 'helloapple',
112
	    'long_name': 'Hello Apple',
113
	    'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
114
	    'version': [2021, 11, 10],
115
	    'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
116
	    'payloads': {
117
	        'https://hydrillabugs.koszko.org/***': {
118
		    'identifier': 'helloapple'
119
	        },
120
	        'https://hachettebugs.koszko.org/***': {
121
		    'identifier': 'helloapple'
122
                }
123
            }
124
        }
125
        self.expected_source_description = {
126
            'api_schema_version': [1, 0, 1],
127
            'source_name': 'hello',
128
            'source_copyright': [{
129
                'file': 'report.spdx',
130
                'sha256': '!!!!value to fill during test!!!!'
131
            }, {
132
                'file': 'LICENSES/CC0-1.0.txt',
133
                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
134
            }],
135
            'source_archives': {
136
                'zip': {
137
                    'sha256': '!!!!value to fill during test!!!!',
138
                }
139
            },
140
            'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
141
            'definitions': [{
142
                'type': 'resource',
143
                'identifier': 'helloapple',
144
                'version': [2021, 11, 10],
145
            }, {
146
                'type':       'resource',
147
                'identifier': 'hello-message',
148
                'version':     [2021, 11, 10],
149
            }, {
150
                'type': 'mapping',
151
                'identifier': 'helloapple',
152
                'version': [2021, 11, 10],
153
            }]
154
        }
155

  
156
    def expected(self) -> list[dict]:
157
        """
158
        Convenience method to get a list of expected jsons of 2 resources,
159
        1 mapping and 1 source description we have.
160
        """
161
        return [
162
            *self.expected_resources,
163
            self.expected_mapping,
164
            self.expected_source_description
165
        ]
166

  
17 167
@pytest.fixture()
18
def tmpdir():
168
def tmpdir() -> str:
19 169
    with TemporaryDirectory() as tmpdir:
20 170
        yield tmpdir
21 171

  
22
def test_build(tmpdir):
172
def prepare_default(tmpdir: Path) -> CaseSettings:
173
    """Use sample source package directory as exists in VCS."""
174
    return CaseSettings()
175

  
176
def prepare_external_index_json(tmpdir: Path) -> dict:
177
    """
178
    Use sample source package directory with an alternative, modified
179
    index.json.
180
    """
181
    settings = CaseSettings()
182

  
183
    from hydrilla_builder.build import strip_json_comments
184

  
185
    with open(settings.srcdir / 'index.json', 'rt') as file_handle:
186
        obj = json.loads(strip_json_comments(file_handle.read()))
187

  
188
    # Add comments that should be preserved.
189
    for dictionary in (obj, settings.expected_source_description):
190
        dictionary['comment'] = 'index_json comment'
191

  
192
    for i, dicts in enumerate(zip(obj['definitions'], settings.expected())):
193
        for dictionary in dicts:
194
            dictionary['comment'] = f'item {i}'
195

  
196
    # Remove spdx report generation
197
    del obj['reuse_generate_spdx_report']
198
    obj['copyright'].remove({'file': 'report.spdx'})
199

  
200
    settings.report_spdx_included = False
201

  
202
    for json_description in settings.expected():
203
        json_description['source_copyright'] = \
204
            [fr for fr in json_description['source_copyright']
205
             if fr['file'] != 'report.spdx']
206

  
207
    # Use default value ([]) for 'additionall_files' property
208
    del obj['additional_files']
209

  
210
    settings.src_filenames = [*settings.dist_filenames, 'index.json']
211

  
212
    # Use default value ([]) for 'scripts' property in one of the resources
213
    del obj['definitions'][1]['scripts']
214

  
215
    settings.expected_resources[1]['scripts'] = []
216

  
217
    for prefix in ('js', 'dist', 'src'):
218
        getattr(settings, f'{prefix}_filenames').remove('message.js')
219

  
220
    # Use default value ({}) for 'pyloads' property in mapping
221
    del obj['definitions'][2]['payloads']
222

  
223
    settings.expected_mapping['payloads'] = {}
224

  
225
    # Add some unrecognized properties that should be stripped
226
    to_process = [obj]
227
    while to_process:
228
        processed = to_process.pop()
229

  
230
        if type(processed) is list:
231
            to_process.extend(processed)
232
        elif type(processed) is dict and 'spurious_property' not in processed:
233
            to_process.extend(processed.values())
234
            processed['spurious_property'] = 'some value'
235

  
236
    # Replace the other index.json with new one
237
    settings.index_json_path = tmpdir / 'replacement.json'
238

  
239
    contents = json.dumps(obj).encode()
240

  
241
    settings.contents['index.json'] = contents
242

  
243
    settings.sha256_hashes['index.json'] = sha256(contents).digest().hex()
244
    settings.sha1_hashes['index.json']   = sha1(contents).digest().hex()
245

  
246
    with open(settings.index_json_path, 'wb') as file_handle:
247
        file_handle.write(contents)
248

  
249
    return settings
250

  
251
@pytest.mark.parametrize('prepare_source_example', [
252
    prepare_default, prepare_external_index_json
253
])
254
def test_build(tmpdir, prepare_source_example):
23 255
    """Build the sample source package and verify the produced files."""
24 256
    from hydrilla_builder.build import Build
25 257

  
26 258
    # First, build the package
27
    srcdir = here / 'source-package-example'
28
    dstdir = Path(tmpdir)
259
    dstdir = Path(tmpdir) / 'dstdir'
260
    tmpdir = Path(tmpdir) / 'example'
261

  
262
    dstdir.mkdir(exist_ok=True)
263
    tmpdir.mkdir(exist_ok=True)
29 264

  
30
    build = Build(srcdir, Path('index.json'))
265
    settings = prepare_source_example(tmpdir)
266

  
267
    build = Build(settings.srcdir, settings.index_json_path)
31 268
    build.write_package_files(dstdir)
32 269

  
33 270
    # Verify directories under destination directory
......
36 273

  
37 274
    # Verify files under 'file/'
38 275
    file_dir = dstdir / 'file'
39
    js_filenames = ('bye.js', 'hello.js', 'message.js')
40
    dist_filenames = (*js_filenames, str(Path('LICENSES') / 'CC0-1.0.txt'))
41
    file_contents = {}
42
    file_sha256_hashes = {}
43
    file_sha1_hashes = {}
44

  
45
    for fn in dist_filenames:
46
        with open(srcdir / fn, 'rb') as file_handle:
47
            file_contents = file_handle.read()
48 276

  
49
        file_sha256_hashes[fn] = sha256(file_contents).digest().hex()
50
        file_sha1_hashes[fn]   = sha1(file_contents).digest().hex()
51

  
52
        dist_file_path = file_dir / f'sha256-{file_sha256_hashes[fn]}'
277
    for fn in settings.dist_filenames:
278
        dist_file_path = file_dir / f'sha256-{settings.sha256_hashes[fn]}'
53 279
        assert dist_file_path.is_file()
54 280

  
55 281
        with open(dist_file_path, 'rb') as file_handle:
56
            assert file_handle.read() == file_contents
282
            assert file_handle.read() == settings.contents[fn]
283

  
284
    sha256_hashes_set = set([settings.sha256_hashes[fn]
285
                             for fn in settings.dist_filenames])
57 286

  
58
    sha256_hashes = set(file_sha256_hashes.values())
59 287
    spdx_report_sha256 = None
60
    spdx_report_checked = False
61 288

  
62 289
    for path in file_dir.iterdir():
63 290
        assert path.name.startswith('sha256-')
64
        if path.name[7:] in sha256_hashes:
291
        if path.name[7:] in sha256_hashes_set:
65 292
            continue
66 293

  
67
        assert spdx_report_sha256 is None
294
        assert spdx_report_sha256 is None and settings.report_spdx_included
68 295

  
69 296
        with open(path, 'rt') as file_handle:
70 297
            spdx_contents = file_handle.read()
......
72 299
        spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex()
73 300
        assert spdx_report_sha256 == path.name[7:]
74 301

  
75
        for fn in js_filenames:
76
            assert file_sha1_hashes[fn] in spdx_contents
302
        for fn in settings.src_filenames:
303
            if not any([n in fn.lower() for n in ('license', 'reuse')]):
304
                assert settings.sha1_hashes[fn]
305

  
306
    if settings.report_spdx_included:
307
        assert spdx_report_sha256
308
        for obj in settings.expected():
309
            for file_ref in obj['source_copyright']:
310
                if file_ref['file'] == 'report.spdx':
311
                    file_ref['sha256'] = spdx_report_sha256
77 312

  
78 313
    # Verify files under 'resource/'
79 314
    resource_dir = dstdir / 'resource'
80 315

  
81
    expected_resource_jsons = [{
82
        'api_schema_version': [1, 0, 1],
83
        'source_name': 'hello',
84
        'source_copyright': [{
85
            'file': 'report.spdx',
86
            'sha256': spdx_report_sha256
87
        }, {
88
            'file': 'LICENSES/CC0-1.0.txt',
89
            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
90
        }],
91
        'type': 'resource',
92
        'identifier': 'helloapple',
93
        'long_name': 'Hello Apple',
94
        'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
95
        'version': [2021, 11, 10],
96
        'revision': 1,
97
        'description': 'greets an apple',
98
        'dependencies': ['hello-message'],
99
        'scripts': [{
100
            'file': 'hello.js',
101
            'sha256': file_sha256_hashes['hello.js']
102
        }, {
103
            'file': 'bye.js',
104
            'sha256': file_sha256_hashes['bye.js']
105
        }]
106
    }, {
107
        'api_schema_version': [1, 0, 1],
108
        'source_name': 'hello',
109
        'source_copyright': [{
110
            'file': 'report.spdx',
111
            'sha256': spdx_report_sha256
112
        }, {
113
            'file': 'LICENSES/CC0-1.0.txt',
114
            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
115
        }],
116
        'type': 'resource',
117
        'identifier': 'hello-message',
118
        'long_name': 'Hello Message',
119
        'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
120
        'version': [2021, 11, 10],
121
        'revision': 2,
122
        'description': 'define messages for saying hello and bye',
123
        'dependencies': [],
124
        'scripts': [{
125
            'file': 'message.js',
126
            'sha256': file_sha256_hashes['message.js']
127
        }]
128
    }]
129

  
130
    assert set([rj['identifier'] for rj in expected_resource_jsons]) == \
316
    assert set([rj['identifier'] for rj in settings.expected_resources]) == \
131 317
        set([path.name for path in resource_dir.iterdir()])
132 318

  
133
    for resource_json in expected_resource_jsons:
319
    for resource_json in settings.expected_resources:
134 320
        subdir = resource_dir / resource_json['identifier']
135 321
        assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
136 322

  
......
144 330
    subdir = mapping_dir / 'helloapple'
145 331
    assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
146 332

  
147
    expected_mapping_json = {
148
        'api_schema_version': [1, 0, 1],
149
        'source_name': 'hello',
150
        'source_copyright': [{
151
            'file': 'report.spdx',
152
            'sha256': spdx_report_sha256
153
        }, {
154
            'file': 'LICENSES/CC0-1.0.txt',
155
            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
156
        }],
157
        'type': 'mapping',
158
	'identifier': 'helloapple',
159
	'long_name': 'Hello Apple',
160
	'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
161
	'version': [2021, 11, 10],
162
	'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
163
	'payloads': {
164
	    'https://hydrillabugs.koszko.org/***': {
165
		'identifier': 'helloapple'
166
	    },
167
	    'https://hachettebugs.koszko.org/***': {
168
		'identifier': 'helloapple'
169
            }
170
        }
171
    }
172

  
173 333
    with open(subdir / '2021.11.10', 'rt') as file_handle:
174
        assert json.load(file_handle) == expected_mapping_json
334
        assert json.load(file_handle) == settings.expected_mapping
175 335

  
176 336
    # Verify files under 'source/'
177 337
    source_dir = dstdir / 'source'
178 338
    assert {'hello.json', 'hello.zip'} == \
179 339
        set([path.name for path in source_dir.iterdir()])
180 340

  
181
    src_filenames = (
182
        *dist_filenames,
183
        'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
184
    )
185
    zip_filenames = [f'hello/{fn}' for fn in src_filenames]
341
    zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames]
186 342

  
187 343
    with ZipFile(source_dir / 'hello.zip', 'r') as archive:
188 344
        assert set([f.filename for f in archive.filelist]) == set(zip_filenames)
189 345

  
190
        for zip_fn, src_fn in zip(zip_filenames, src_filenames):
191
            src_path = srcdir
192
            for segment in src_fn.split('/'):
193
                src_path = src_path / segment
194

  
346
        for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames):
195 347
            with archive.open(zip_fn, 'r') as zip_file_handle:
196
                with open(src_path, 'rb') as disk_file_handle:
197
                    assert zip_file_handle.read() == disk_file_handle.read()
348
                assert zip_file_handle.read() == settings.contents[src_fn]
198 349

  
350
    zip_ref = settings.expected_source_description['source_archives']['zip']
199 351
    with open(source_dir / 'hello.zip', 'rb') as file_handle:
200
        zipfile_sha256_hash = sha256(file_handle.read()).digest().hex()
201

  
202
    expected_source_description = {
203
        'api_schema_version': [1, 0, 1],
204
        'source_name': 'hello',
205
        'source_copyright': [{
206
            'file': 'report.spdx',
207
            'sha256': spdx_report_sha256
208
        }, {
209
            'file': 'LICENSES/CC0-1.0.txt',
210
            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
211
        }],
212
        'source_archives': {
213
            'zip': {
214
                'sha256': zipfile_sha256_hash,
215
            }
216
        },
217
        'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
218
        'definitions': [{
219
            'type': 'resource',
220
            'identifier': 'helloapple',
221
            'version': [2021, 11, 10],
222
        }, {
223
            'type':       'resource',
224
            'identifier': 'hello-message',
225
            'version':     [2021, 11, 10],
226
        }, {
227
            'type': 'mapping',
228
            'identifier': 'helloapple',
229
            'version': [2021, 11, 10],
230
        }]
231
    }
352
        zip_ref['sha256'] = sha256(file_handle.read()).digest().hex()
232 353

  
233 354
    with open(source_dir / 'hello.json', 'rt') as file_handle:
234
        assert json.load(file_handle) == expected_source_description
355
        assert json.load(file_handle) == settings.expected_source_description
235 356

  
236
# TODO: also verify on slightly different examples and check error handling
357
# TODO: also check error handling

Also available in: Unified diff