Revision 8a036bc7
Added by koszko over 1 year ago
src/test/test_hydrilla_builder.py | ||
---|---|---|
14 | 14 |
|
15 | 15 |
here = Path(__file__).resolve().parent |
16 | 16 |
|
17 |
default_srcdir = here / 'source-package-example' |
|
18 |
|
|
19 |
default_js_filenames = ['bye.js', 'hello.js', 'message.js'] |
|
20 |
default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt'] |
|
21 |
default_src_filenames = [ |
|
22 |
*default_dist_filenames, |
|
23 |
'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json' |
|
24 |
] |
|
25 |
|
|
26 |
default_sha1_hashes = {} |
|
27 |
default_sha256_hashes = {} |
|
28 |
default_contents = {} |
|
29 |
|
|
30 |
for fn in default_src_filenames: |
|
31 |
with open(default_srcdir / fn, 'rb') as file_handle: |
|
32 |
default_contents[fn] = file_handle.read() |
|
33 |
default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex() |
|
34 |
default_sha1_hashes[fn] = sha1(default_contents[fn]).digest().hex() |
|
35 |
|
|
36 |
class CaseSettings: |
|
37 |
"""Gather parametrized values in a class.""" |
|
38 |
def __init__(self): |
|
39 |
"""Init CaseSettings with default values.""" |
|
40 |
self.srcdir = default_srcdir |
|
41 |
self.index_json_path = Path('index.json') |
|
42 |
self.report_spdx_included = True |
|
43 |
|
|
44 |
self.js_filenames = default_js_filenames.copy() |
|
45 |
self.dist_filenames = default_dist_filenames.copy() |
|
46 |
self.src_filenames = default_src_filenames.copy() |
|
47 |
|
|
48 |
self.sha1_hashes = default_sha1_hashes.copy() |
|
49 |
self.sha256_hashes = default_sha256_hashes.copy() |
|
50 |
self.contents = default_contents.copy() |
|
51 |
|
|
52 |
self.expected_resources = [{ |
|
53 |
'api_schema_version': [1, 0, 1], |
|
54 |
'source_name': 'hello', |
|
55 |
'source_copyright': [{ |
|
56 |
'file': 'report.spdx', |
|
57 |
'sha256': '!!!!value to fill during test!!!!' |
|
58 |
}, { |
|
59 |
'file': 'LICENSES/CC0-1.0.txt', |
|
60 |
'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
61 |
}], |
|
62 |
'type': 'resource', |
|
63 |
'identifier': 'helloapple', |
|
64 |
'long_name': 'Hello Apple', |
|
65 |
'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', |
|
66 |
'version': [2021, 11, 10], |
|
67 |
'revision': 1, |
|
68 |
'description': 'greets an apple', |
|
69 |
'dependencies': ['hello-message'], |
|
70 |
'scripts': [{ |
|
71 |
'file': 'hello.js', |
|
72 |
'sha256': self.sha256_hashes['hello.js'] |
|
73 |
}, { |
|
74 |
'file': 'bye.js', |
|
75 |
'sha256': self.sha256_hashes['bye.js'] |
|
76 |
}] |
|
77 |
}, { |
|
78 |
'api_schema_version': [1, 0, 1], |
|
79 |
'source_name': 'hello', |
|
80 |
'source_copyright': [{ |
|
81 |
'file': 'report.spdx', |
|
82 |
'sha256': '!!!!value to fill during test!!!!' |
|
83 |
}, { |
|
84 |
'file': 'LICENSES/CC0-1.0.txt', |
|
85 |
'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
86 |
}], |
|
87 |
'type': 'resource', |
|
88 |
'identifier': 'hello-message', |
|
89 |
'long_name': 'Hello Message', |
|
90 |
'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e', |
|
91 |
'version': [2021, 11, 10], |
|
92 |
'revision': 2, |
|
93 |
'description': 'define messages for saying hello and bye', |
|
94 |
'dependencies': [], |
|
95 |
'scripts': [{ |
|
96 |
'file': 'message.js', |
|
97 |
'sha256': self.sha256_hashes['message.js'] |
|
98 |
}] |
|
99 |
}] |
|
100 |
self.expected_mapping = { |
|
101 |
'api_schema_version': [1, 0, 1], |
|
102 |
'source_name': 'hello', |
|
103 |
'source_copyright': [{ |
|
104 |
'file': 'report.spdx', |
|
105 |
'sha256': '!!!!value to fill during test!!!!' |
|
106 |
}, { |
|
107 |
'file': 'LICENSES/CC0-1.0.txt', |
|
108 |
'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
109 |
}], |
|
110 |
'type': 'mapping', |
|
111 |
'identifier': 'helloapple', |
|
112 |
'long_name': 'Hello Apple', |
|
113 |
'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', |
|
114 |
'version': [2021, 11, 10], |
|
115 |
'description': 'causes apple to get greeted on Hydrillabugs issue tracker', |
|
116 |
'payloads': { |
|
117 |
'https://hydrillabugs.koszko.org/***': { |
|
118 |
'identifier': 'helloapple' |
|
119 |
}, |
|
120 |
'https://hachettebugs.koszko.org/***': { |
|
121 |
'identifier': 'helloapple' |
|
122 |
} |
|
123 |
} |
|
124 |
} |
|
125 |
self.expected_source_description = { |
|
126 |
'api_schema_version': [1, 0, 1], |
|
127 |
'source_name': 'hello', |
|
128 |
'source_copyright': [{ |
|
129 |
'file': 'report.spdx', |
|
130 |
'sha256': '!!!!value to fill during test!!!!' |
|
131 |
}, { |
|
132 |
'file': 'LICENSES/CC0-1.0.txt', |
|
133 |
'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
134 |
}], |
|
135 |
'source_archives': { |
|
136 |
'zip': { |
|
137 |
'sha256': '!!!!value to fill during test!!!!', |
|
138 |
} |
|
139 |
}, |
|
140 |
'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example', |
|
141 |
'definitions': [{ |
|
142 |
'type': 'resource', |
|
143 |
'identifier': 'helloapple', |
|
144 |
'version': [2021, 11, 10], |
|
145 |
}, { |
|
146 |
'type': 'resource', |
|
147 |
'identifier': 'hello-message', |
|
148 |
'version': [2021, 11, 10], |
|
149 |
}, { |
|
150 |
'type': 'mapping', |
|
151 |
'identifier': 'helloapple', |
|
152 |
'version': [2021, 11, 10], |
|
153 |
}] |
|
154 |
} |
|
155 |
|
|
156 |
def expected(self) -> list[dict]: |
|
157 |
""" |
|
158 |
Convenience method to get a list of expected jsons of 2 resources, |
|
159 |
1 mapping and 1 source description we have. |
|
160 |
""" |
|
161 |
return [ |
|
162 |
*self.expected_resources, |
|
163 |
self.expected_mapping, |
|
164 |
self.expected_source_description |
|
165 |
] |
|
166 |
|
|
17 | 167 |
@pytest.fixture() |
18 |
def tmpdir(): |
|
168 |
def tmpdir() -> str:
|
|
19 | 169 |
with TemporaryDirectory() as tmpdir: |
20 | 170 |
yield tmpdir |
21 | 171 |
|
22 |
def test_build(tmpdir): |
|
172 |
def prepare_default(tmpdir: Path) -> CaseSettings: |
|
173 |
"""Use sample source package directory as exists in VCS.""" |
|
174 |
return CaseSettings() |
|
175 |
|
|
176 |
def prepare_external_index_json(tmpdir: Path) -> dict: |
|
177 |
""" |
|
178 |
Use sample source package directory with an alternative, modified |
|
179 |
index.json. |
|
180 |
""" |
|
181 |
settings = CaseSettings() |
|
182 |
|
|
183 |
from hydrilla_builder.build import strip_json_comments |
|
184 |
|
|
185 |
with open(settings.srcdir / 'index.json', 'rt') as file_handle: |
|
186 |
obj = json.loads(strip_json_comments(file_handle.read())) |
|
187 |
|
|
188 |
# Add comments that should be preserved. |
|
189 |
for dictionary in (obj, settings.expected_source_description): |
|
190 |
dictionary['comment'] = 'index_json comment' |
|
191 |
|
|
192 |
for i, dicts in enumerate(zip(obj['definitions'], settings.expected())): |
|
193 |
for dictionary in dicts: |
|
194 |
dictionary['comment'] = f'item {i}' |
|
195 |
|
|
196 |
# Remove spdx report generation |
|
197 |
del obj['reuse_generate_spdx_report'] |
|
198 |
obj['copyright'].remove({'file': 'report.spdx'}) |
|
199 |
|
|
200 |
settings.report_spdx_included = False |
|
201 |
|
|
202 |
for json_description in settings.expected(): |
|
203 |
json_description['source_copyright'] = \ |
|
204 |
[fr for fr in json_description['source_copyright'] |
|
205 |
if fr['file'] != 'report.spdx'] |
|
206 |
|
|
207 |
# Use default value ([]) for 'additionall_files' property |
|
208 |
del obj['additional_files'] |
|
209 |
|
|
210 |
settings.src_filenames = [*settings.dist_filenames, 'index.json'] |
|
211 |
|
|
212 |
# Use default value ([]) for 'scripts' property in one of the resources |
|
213 |
del obj['definitions'][1]['scripts'] |
|
214 |
|
|
215 |
settings.expected_resources[1]['scripts'] = [] |
|
216 |
|
|
217 |
for prefix in ('js', 'dist', 'src'): |
|
218 |
getattr(settings, f'{prefix}_filenames').remove('message.js') |
|
219 |
|
|
220 |
# Use default value ({}) for 'pyloads' property in mapping |
|
221 |
del obj['definitions'][2]['payloads'] |
|
222 |
|
|
223 |
settings.expected_mapping['payloads'] = {} |
|
224 |
|
|
225 |
# Add some unrecognized properties that should be stripped |
|
226 |
to_process = [obj] |
|
227 |
while to_process: |
|
228 |
processed = to_process.pop() |
|
229 |
|
|
230 |
if type(processed) is list: |
|
231 |
to_process.extend(processed) |
|
232 |
elif type(processed) is dict and 'spurious_property' not in processed: |
|
233 |
to_process.extend(processed.values()) |
|
234 |
processed['spurious_property'] = 'some value' |
|
235 |
|
|
236 |
# Replace the other index.json with new one |
|
237 |
settings.index_json_path = tmpdir / 'replacement.json' |
|
238 |
|
|
239 |
contents = json.dumps(obj).encode() |
|
240 |
|
|
241 |
settings.contents['index.json'] = contents |
|
242 |
|
|
243 |
settings.sha256_hashes['index.json'] = sha256(contents).digest().hex() |
|
244 |
settings.sha1_hashes['index.json'] = sha1(contents).digest().hex() |
|
245 |
|
|
246 |
with open(settings.index_json_path, 'wb') as file_handle: |
|
247 |
file_handle.write(contents) |
|
248 |
|
|
249 |
return settings |
|
250 |
|
|
251 |
@pytest.mark.parametrize('prepare_source_example', [ |
|
252 |
prepare_default, prepare_external_index_json |
|
253 |
]) |
|
254 |
def test_build(tmpdir, prepare_source_example): |
|
23 | 255 |
"""Build the sample source package and verify the produced files.""" |
24 | 256 |
from hydrilla_builder.build import Build |
25 | 257 |
|
26 | 258 |
# First, build the package |
27 |
srcdir = here / 'source-package-example' |
|
28 |
dstdir = Path(tmpdir) |
|
259 |
dstdir = Path(tmpdir) / 'dstdir' |
|
260 |
tmpdir = Path(tmpdir) / 'example' |
|
261 |
|
|
262 |
dstdir.mkdir(exist_ok=True) |
|
263 |
tmpdir.mkdir(exist_ok=True) |
|
29 | 264 |
|
30 |
build = Build(srcdir, Path('index.json')) |
|
265 |
settings = prepare_source_example(tmpdir) |
|
266 |
|
|
267 |
build = Build(settings.srcdir, settings.index_json_path) |
|
31 | 268 |
build.write_package_files(dstdir) |
32 | 269 |
|
33 | 270 |
# Verify directories under destination directory |
... | ... | |
36 | 273 |
|
37 | 274 |
# Verify files under 'file/' |
38 | 275 |
file_dir = dstdir / 'file' |
39 |
js_filenames = ('bye.js', 'hello.js', 'message.js') |
|
40 |
dist_filenames = (*js_filenames, str(Path('LICENSES') / 'CC0-1.0.txt')) |
|
41 |
file_contents = {} |
|
42 |
file_sha256_hashes = {} |
|
43 |
file_sha1_hashes = {} |
|
44 |
|
|
45 |
for fn in dist_filenames: |
|
46 |
with open(srcdir / fn, 'rb') as file_handle: |
|
47 |
file_contents = file_handle.read() |
|
48 | 276 |
|
49 |
file_sha256_hashes[fn] = sha256(file_contents).digest().hex() |
|
50 |
file_sha1_hashes[fn] = sha1(file_contents).digest().hex() |
|
51 |
|
|
52 |
dist_file_path = file_dir / f'sha256-{file_sha256_hashes[fn]}' |
|
277 |
for fn in settings.dist_filenames: |
|
278 |
dist_file_path = file_dir / f'sha256-{settings.sha256_hashes[fn]}' |
|
53 | 279 |
assert dist_file_path.is_file() |
54 | 280 |
|
55 | 281 |
with open(dist_file_path, 'rb') as file_handle: |
56 |
assert file_handle.read() == file_contents |
|
282 |
assert file_handle.read() == settings.contents[fn] |
|
283 |
|
|
284 |
sha256_hashes_set = set([settings.sha256_hashes[fn] |
|
285 |
for fn in settings.dist_filenames]) |
|
57 | 286 |
|
58 |
sha256_hashes = set(file_sha256_hashes.values()) |
|
59 | 287 |
spdx_report_sha256 = None |
60 |
spdx_report_checked = False |
|
61 | 288 |
|
62 | 289 |
for path in file_dir.iterdir(): |
63 | 290 |
assert path.name.startswith('sha256-') |
64 |
if path.name[7:] in sha256_hashes: |
|
291 |
if path.name[7:] in sha256_hashes_set:
|
|
65 | 292 |
continue |
66 | 293 |
|
67 |
assert spdx_report_sha256 is None |
|
294 |
assert spdx_report_sha256 is None and settings.report_spdx_included
|
|
68 | 295 |
|
69 | 296 |
with open(path, 'rt') as file_handle: |
70 | 297 |
spdx_contents = file_handle.read() |
... | ... | |
72 | 299 |
spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex() |
73 | 300 |
assert spdx_report_sha256 == path.name[7:] |
74 | 301 |
|
75 |
for fn in js_filenames: |
|
76 |
assert file_sha1_hashes[fn] in spdx_contents |
|
302 |
for fn in settings.src_filenames: |
|
303 |
if not any([n in fn.lower() for n in ('license', 'reuse')]): |
|
304 |
assert settings.sha1_hashes[fn] |
|
305 |
|
|
306 |
if settings.report_spdx_included: |
|
307 |
assert spdx_report_sha256 |
|
308 |
for obj in settings.expected(): |
|
309 |
for file_ref in obj['source_copyright']: |
|
310 |
if file_ref['file'] == 'report.spdx': |
|
311 |
file_ref['sha256'] = spdx_report_sha256 |
|
77 | 312 |
|
78 | 313 |
# Verify files under 'resource/' |
79 | 314 |
resource_dir = dstdir / 'resource' |
80 | 315 |
|
81 |
expected_resource_jsons = [{ |
|
82 |
'api_schema_version': [1, 0, 1], |
|
83 |
'source_name': 'hello', |
|
84 |
'source_copyright': [{ |
|
85 |
'file': 'report.spdx', |
|
86 |
'sha256': spdx_report_sha256 |
|
87 |
}, { |
|
88 |
'file': 'LICENSES/CC0-1.0.txt', |
|
89 |
'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
90 |
}], |
|
91 |
'type': 'resource', |
|
92 |
'identifier': 'helloapple', |
|
93 |
'long_name': 'Hello Apple', |
|
94 |
'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', |
|
95 |
'version': [2021, 11, 10], |
|
96 |
'revision': 1, |
|
97 |
'description': 'greets an apple', |
|
98 |
'dependencies': ['hello-message'], |
|
99 |
'scripts': [{ |
|
100 |
'file': 'hello.js', |
|
101 |
'sha256': file_sha256_hashes['hello.js'] |
|
102 |
}, { |
|
103 |
'file': 'bye.js', |
|
104 |
'sha256': file_sha256_hashes['bye.js'] |
|
105 |
}] |
|
106 |
}, { |
|
107 |
'api_schema_version': [1, 0, 1], |
|
108 |
'source_name': 'hello', |
|
109 |
'source_copyright': [{ |
|
110 |
'file': 'report.spdx', |
|
111 |
'sha256': spdx_report_sha256 |
|
112 |
}, { |
|
113 |
'file': 'LICENSES/CC0-1.0.txt', |
|
114 |
'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
115 |
}], |
|
116 |
'type': 'resource', |
|
117 |
'identifier': 'hello-message', |
|
118 |
'long_name': 'Hello Message', |
|
119 |
'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e', |
|
120 |
'version': [2021, 11, 10], |
|
121 |
'revision': 2, |
|
122 |
'description': 'define messages for saying hello and bye', |
|
123 |
'dependencies': [], |
|
124 |
'scripts': [{ |
|
125 |
'file': 'message.js', |
|
126 |
'sha256': file_sha256_hashes['message.js'] |
|
127 |
}] |
|
128 |
}] |
|
129 |
|
|
130 |
assert set([rj['identifier'] for rj in expected_resource_jsons]) == \ |
|
316 |
assert set([rj['identifier'] for rj in settings.expected_resources]) == \ |
|
131 | 317 |
set([path.name for path in resource_dir.iterdir()]) |
132 | 318 |
|
133 |
for resource_json in expected_resource_jsons:
|
|
319 |
for resource_json in settings.expected_resources:
|
|
134 | 320 |
subdir = resource_dir / resource_json['identifier'] |
135 | 321 |
assert ['2021.11.10'] == [path.name for path in subdir.iterdir()] |
136 | 322 |
|
... | ... | |
144 | 330 |
subdir = mapping_dir / 'helloapple' |
145 | 331 |
assert ['2021.11.10'] == [path.name for path in subdir.iterdir()] |
146 | 332 |
|
147 |
expected_mapping_json = { |
|
148 |
'api_schema_version': [1, 0, 1], |
|
149 |
'source_name': 'hello', |
|
150 |
'source_copyright': [{ |
|
151 |
'file': 'report.spdx', |
|
152 |
'sha256': spdx_report_sha256 |
|
153 |
}, { |
|
154 |
'file': 'LICENSES/CC0-1.0.txt', |
|
155 |
'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
156 |
}], |
|
157 |
'type': 'mapping', |
|
158 |
'identifier': 'helloapple', |
|
159 |
'long_name': 'Hello Apple', |
|
160 |
'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', |
|
161 |
'version': [2021, 11, 10], |
|
162 |
'description': 'causes apple to get greeted on Hydrillabugs issue tracker', |
|
163 |
'payloads': { |
|
164 |
'https://hydrillabugs.koszko.org/***': { |
|
165 |
'identifier': 'helloapple' |
|
166 |
}, |
|
167 |
'https://hachettebugs.koszko.org/***': { |
|
168 |
'identifier': 'helloapple' |
|
169 |
} |
|
170 |
} |
|
171 |
} |
|
172 |
|
|
173 | 333 |
with open(subdir / '2021.11.10', 'rt') as file_handle: |
174 |
assert json.load(file_handle) == expected_mapping_json
|
|
334 |
assert json.load(file_handle) == settings.expected_mapping
|
|
175 | 335 |
|
176 | 336 |
# Verify files under 'source/' |
177 | 337 |
source_dir = dstdir / 'source' |
178 | 338 |
assert {'hello.json', 'hello.zip'} == \ |
179 | 339 |
set([path.name for path in source_dir.iterdir()]) |
180 | 340 |
|
181 |
src_filenames = ( |
|
182 |
*dist_filenames, |
|
183 |
'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json' |
|
184 |
) |
|
185 |
zip_filenames = [f'hello/{fn}' for fn in src_filenames] |
|
341 |
zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames] |
|
186 | 342 |
|
187 | 343 |
with ZipFile(source_dir / 'hello.zip', 'r') as archive: |
188 | 344 |
assert set([f.filename for f in archive.filelist]) == set(zip_filenames) |
189 | 345 |
|
190 |
for zip_fn, src_fn in zip(zip_filenames, src_filenames): |
|
191 |
src_path = srcdir |
|
192 |
for segment in src_fn.split('/'): |
|
193 |
src_path = src_path / segment |
|
194 |
|
|
346 |
for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames): |
|
195 | 347 |
with archive.open(zip_fn, 'r') as zip_file_handle: |
196 |
with open(src_path, 'rb') as disk_file_handle: |
|
197 |
assert zip_file_handle.read() == disk_file_handle.read() |
|
348 |
assert zip_file_handle.read() == settings.contents[src_fn] |
|
198 | 349 |
|
350 |
zip_ref = settings.expected_source_description['source_archives']['zip'] |
|
199 | 351 |
with open(source_dir / 'hello.zip', 'rb') as file_handle: |
200 |
zipfile_sha256_hash = sha256(file_handle.read()).digest().hex() |
|
201 |
|
|
202 |
expected_source_description = { |
|
203 |
'api_schema_version': [1, 0, 1], |
|
204 |
'source_name': 'hello', |
|
205 |
'source_copyright': [{ |
|
206 |
'file': 'report.spdx', |
|
207 |
'sha256': spdx_report_sha256 |
|
208 |
}, { |
|
209 |
'file': 'LICENSES/CC0-1.0.txt', |
|
210 |
'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] |
|
211 |
}], |
|
212 |
'source_archives': { |
|
213 |
'zip': { |
|
214 |
'sha256': zipfile_sha256_hash, |
|
215 |
} |
|
216 |
}, |
|
217 |
'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example', |
|
218 |
'definitions': [{ |
|
219 |
'type': 'resource', |
|
220 |
'identifier': 'helloapple', |
|
221 |
'version': [2021, 11, 10], |
|
222 |
}, { |
|
223 |
'type': 'resource', |
|
224 |
'identifier': 'hello-message', |
|
225 |
'version': [2021, 11, 10], |
|
226 |
}, { |
|
227 |
'type': 'mapping', |
|
228 |
'identifier': 'helloapple', |
|
229 |
'version': [2021, 11, 10], |
|
230 |
}] |
|
231 |
} |
|
352 |
zip_ref['sha256'] = sha256(file_handle.read()).digest().hex() |
|
232 | 353 |
|
233 | 354 |
with open(source_dir / 'hello.json', 'rt') as file_handle: |
234 |
assert json.load(file_handle) == expected_source_description |
|
355 |
assert json.load(file_handle) == settings.expected_source_description
|
|
235 | 356 |
|
236 |
# TODO: also verify on slightly different examples and check error handling |
|
357 |
# TODO: also check error handling |
Also available in: Unified diff
add test case with modified index.json from the example