Revision 61f0aa75
Added by koszko over 1 year ago
src/hydrilla/builder/build.py | ||
---|---|---|
30 | 30 |
import json |
31 | 31 |
import re |
32 | 32 |
import zipfile |
33 |
from pathlib import Path |
|
33 |
import subprocess |
|
34 |
from pathlib import Path, PurePosixPath |
|
34 | 35 |
from hashlib import sha256 |
35 | 36 |
from sys import stderr |
37 |
from contextlib import contextmanager |
|
38 |
from tempfile import TemporaryDirectory, TemporaryFile |
|
39 |
from typing import Optional, Iterable, Union |
|
36 | 40 |
|
37 | 41 |
import jsonschema |
38 | 42 |
import click |
39 | 43 |
|
40 | 44 |
from .. import util |
41 | 45 |
from . import _version |
46 |
from . import local_apt |
|
47 |
from .piggybacking import Piggybacked |
|
48 |
from .common_errors import * |
|
42 | 49 |
|
43 | 50 |
here = Path(__file__).resolve().parent |
44 | 51 |
|
45 | 52 |
_ = util.translation(here / 'locales').gettext |
46 | 53 |
|
47 |
index_validator = util.validator_for('package_source-1.0.1.schema.json')
|
|
54 |
index_validator = util.validator_for('package_source-2.schema.json')
|
|
48 | 55 |
|
49 | 56 |
schemas_root = 'https://hydrilla.koszko.org/schemas' |
50 | 57 |
|
... | ... | |
53 | 60 |
'version': _version.version |
54 | 61 |
} |
55 | 62 |
|
56 |
class FileReferenceError(Exception): |
|
57 |
""" |
|
58 |
Exception used to report various problems concerning files referenced from |
|
59 |
source package's index.json. |
|
60 |
""" |
|
61 |
|
|
62 |
class ReuseError(Exception): |
|
63 |
class ReuseError(SubprocessError): |
|
63 | 64 |
""" |
64 | 65 |
Exception used to report various problems when calling the REUSE tool. |
65 | 66 |
""" |
66 | 67 |
|
67 |
class FileBuffer: |
|
68 |
""" |
|
69 |
Implement a file-like object that buffers data written to it. |
|
70 |
""" |
|
71 |
def __init__(self): |
|
72 |
""" |
|
73 |
Initialize FileBuffer. |
|
74 |
""" |
|
75 |
self.chunks = [] |
|
76 |
|
|
77 |
def write(self, b): |
|
78 |
""" |
|
79 |
Buffer 'b', return number of bytes buffered. |
|
80 |
|
|
81 |
'b' is expected to be an instance of 'bytes' or 'str', in which case it |
|
82 |
gets encoded as UTF-8. |
|
83 |
""" |
|
84 |
if type(b) is str: |
|
85 |
b = b.encode() |
|
86 |
self.chunks.append(b) |
|
87 |
return len(b) |
|
88 |
|
|
89 |
def flush(self): |
|
90 |
""" |
|
91 |
A no-op mock of file-like object's flush() method. |
|
92 |
""" |
|
93 |
pass |
|
94 |
|
|
95 |
def get_bytes(self): |
|
96 |
""" |
|
97 |
Return all data written so far concatenated into a single 'bytes' |
|
98 |
object. |
|
99 |
""" |
|
100 |
return b''.join(self.chunks) |
|
101 |
|
|
102 |
def generate_spdx_report(root): |
|
68 |
def generate_spdx_report(root: Path) -> bytes: |
|
103 | 69 |
""" |
104 | 70 |
Use REUSE tool to generate an SPDX report for sources under 'root' and |
105 | 71 |
return the report's contents as 'bytes'. |
106 | 72 |
|
107 |
'root' shall be an instance of pathlib.Path. |
|
108 |
|
|
109 | 73 |
In case the directory tree under 'root' does not constitute a |
110 |
REUSE-compliant package, linting report is printed to standard output and
|
|
111 |
an exception is raised.
|
|
74 |
REUSE-compliant package, as exception is raised with linting report
|
|
75 |
included in it.
|
|
112 | 76 |
|
113 |
In case the reuse package is not installed, an exception is also raised.
|
|
77 |
In case the reuse tool is not installed, an exception is also raised.
|
|
114 | 78 |
""" |
115 |
try: |
|
116 |
from reuse._main import main as reuse_main |
|
117 |
except ModuleNotFoundError: |
|
118 |
raise ReuseError(_('couldnt_import_reuse_is_it_installed')) |
|
79 |
for command in [ |
|
80 |
['reuse', '--root', str(root), 'lint'], |
|
81 |
['reuse', '--root', str(root), 'spdx'] |
|
82 |
]: |
|
83 |
try: |
|
84 |
cp = subprocess.run(command, capture_output=True, text=True) |
|
85 |
except FileNotFoundError: |
|
86 |
raise ReuseError(_('couldnt_execute_reuse_is_it_installed')) |
|
119 | 87 |
|
120 |
mocked_output = FileBuffer() |
|
121 |
if reuse_main(args=['--root', str(root), 'lint'], out=mocked_output) != 0: |
|
122 |
stderr.write(mocked_output.get_bytes().decode()) |
|
123 |
raise ReuseError(_('spdx_report_from_reuse_incompliant')) |
|
88 |
if cp.returncode != 0: |
|
89 |
msg = _('reuse_command_{}_failed').format(' '.join(command)) |
|
90 |
raise ReuseError(msg, cp) |
|
124 | 91 |
|
125 |
mocked_output = FileBuffer() |
|
126 |
if reuse_main(args=['--root', str(root), 'spdx'], out=mocked_output) != 0: |
|
127 |
stderr.write(mocked_output.get_bytes().decode()) |
|
128 |
raise ReuseError("Couldn't generate an SPDX report for package.") |
|
129 |
|
|
130 |
return mocked_output.get_bytes() |
|
92 |
return cp.stdout.encode() |
|
131 | 93 |
|
132 | 94 |
class FileRef: |
133 | 95 |
"""Represent reference to a file in the package.""" |
134 |
def __init__(self, path: Path, contents: bytes):
|
|
96 |
def __init__(self, path: PurePosixPath, contents: bytes) -> None:
|
|
135 | 97 |
"""Initialize FileRef.""" |
136 |
self.include_in_distribution = False |
|
137 |
self.include_in_zipfile = True
|
|
138 |
self.path = path |
|
139 |
self.contents = contents |
|
98 |
self.include_in_distribution = False
|
|
99 |
self.include_in_source_archive = True
|
|
100 |
self.path = path
|
|
101 |
self.contents = contents
|
|
140 | 102 |
|
141 | 103 |
self.contents_hash = sha256(contents).digest().hex() |
142 | 104 |
|
143 |
def make_ref_dict(self, filename: str):
|
|
105 |
def make_ref_dict(self) -> dict[str, str]:
|
|
144 | 106 |
""" |
145 | 107 |
Represent the file reference through a dict that can be included in JSON |
146 | 108 |
defintions. |
147 | 109 |
""" |
148 | 110 |
return { |
149 |
'file': filename,
|
|
111 |
'file': str(self.path),
|
|
150 | 112 |
'sha256': self.contents_hash |
151 | 113 |
} |
152 | 114 |
|
115 |
@contextmanager |
|
116 |
def piggybacked_system(piggyback_def: Optional[dict], |
|
117 |
piggyback_files: Optional[Path]) \ |
|
118 |
-> Iterable[Piggybacked]: |
|
119 |
""" |
|
120 |
Resolve resources from a foreign software packaging system. Optionally, use |
|
121 |
package files (.deb's, etc.) from a specified directory instead of resolving |
|
122 |
and downloading them. |
|
123 |
""" |
|
124 |
if piggyback_def is None: |
|
125 |
yield Piggybacked() |
|
126 |
else: |
|
127 |
# apt is the only supported system right now |
|
128 |
assert piggyback_def['system'] == 'apt' |
|
129 |
|
|
130 |
with local_apt.piggybacked_system(piggyback_def, piggyback_files) \ |
|
131 |
as piggybacked: |
|
132 |
yield piggybacked |
|
133 |
|
|
153 | 134 |
class Build: |
154 | 135 |
""" |
155 | 136 |
Build a Hydrilla package. |
156 | 137 |
""" |
157 |
def __init__(self, srcdir, index_json_path): |
|
138 |
def __init__(self, srcdir: Path, index_json_path: Path, |
|
139 |
piggyback_files: Optional[Path]=None): |
|
158 | 140 |
""" |
159 | 141 |
Initialize a build. All files to be included in a distribution package |
160 | 142 |
are loaded into memory, all data gets validated and all necessary |
161 | 143 |
computations (e.g. preparing of hashes) are performed. |
162 |
|
|
163 |
'srcdir' and 'index_json' are expected to be pathlib.Path objects. |
|
164 | 144 |
""" |
165 | 145 |
self.srcdir = srcdir.resolve() |
166 |
self.index_json_path = index_json_path |
|
146 |
self.piggyback_files = piggyback_files |
|
147 |
# TODO: the piggyback files we set are ignored for now; use them |
|
148 |
if piggyback_files is None: |
|
149 |
piggyback_default_path = \ |
|
150 |
srcdir.parent / f'{srcdir.name}.foreign-packages' |
|
151 |
if piggyback_default_path.exists(): |
|
152 |
self.piggyback_files = piggyback_default_path |
|
167 | 153 |
self.files_by_path = {} |
168 | 154 |
self.resource_list = [] |
169 | 155 |
self.mapping_list = [] |
170 | 156 |
|
171 | 157 |
if not index_json_path.is_absolute(): |
172 |
self.index_json_path = (self.srcdir / self.index_json_path) |
|
173 |
|
|
174 |
self.index_json_path = self.index_json_path.resolve() |
|
158 |
index_json_path = (self.srcdir / index_json_path) |
|
175 | 159 |
|
176 |
with open(self.index_json_path, 'rt') as index_file:
|
|
160 |
with open(index_json_path, 'rt') as index_file: |
|
177 | 161 |
index_json_text = index_file.read() |
178 | 162 |
|
179 | 163 |
index_obj = json.loads(util.strip_json_comments(index_json_text)) |
180 | 164 |
|
181 |
self.files_by_path[self.srcdir / 'index.json'] = \ |
|
182 |
FileRef(self.srcdir / 'index.json', index_json_text.encode()) |
|
165 |
index_desired_path = PurePosixPath('index.json') |
|
166 |
self.files_by_path[index_desired_path] = \ |
|
167 |
FileRef(index_desired_path, index_json_text.encode()) |
|
183 | 168 |
|
184 | 169 |
self._process_index_json(index_obj) |
185 | 170 |
|
186 |
def _process_file(self, filename: str, include_in_distribution: bool=True): |
|
171 |
def _process_file(self, filename: Union[str, PurePosixPath], |
|
172 |
piggybacked: Piggybacked, |
|
173 |
include_in_distribution: bool=True): |
|
187 | 174 |
""" |
188 | 175 |
Resolve 'filename' relative to srcdir, load it to memory (if not loaded |
189 | 176 |
before), compute its hash and store its information in |
190 | 177 |
'self.files_by_path'. |
191 | 178 |
|
192 |
'filename' shall represent a relative path using '/' as a separator.
|
|
179 |
'filename' shall represent a relative path withing package directory.
|
|
193 | 180 |
|
194 | 181 |
if 'include_in_distribution' is True it shall cause the file to not only |
195 | 182 |
be included in the source package's zipfile, but also written as one of |
196 | 183 |
built package's files. |
197 | 184 |
|
185 |
For each file an attempt is made to resolve it using 'piggybacked' |
|
186 |
object. If a file is found and pulled from foreign software packaging |
|
187 |
system this way, it gets automatically excluded from inclusion in |
|
188 |
Hydrilla source package's zipfile. |
|
189 |
|
|
198 | 190 |
Return file's reference object that can be included in JSON defintions |
199 | 191 |
of various kinds. |
200 | 192 |
""" |
201 |
path = self.srcdir |
|
202 |
for segment in filename.split('/'): |
|
203 |
path /= segment |
|
204 |
|
|
205 |
path = path.resolve() |
|
206 |
if not path.is_relative_to(self.srcdir): |
|
207 |
raise FileReferenceError(_('loading_{}_outside_package_dir') |
|
208 |
.format(filename)) |
|
209 |
|
|
210 |
if str(path.relative_to(self.srcdir)) == 'index.json': |
|
211 |
raise FileReferenceError(_('loading_reserved_index_json')) |
|
193 |
include_in_source_archive = True |
|
194 |
|
|
195 |
desired_path = PurePosixPath(filename) |
|
196 |
if '..' in desired_path.parts: |
|
197 |
msg = _('path_contains_double_dot_{}').format(filename) |
|
198 |
raise FileReferenceError(msg) |
|
199 |
|
|
200 |
path = piggybacked.resolve_file(desired_path) |
|
201 |
if path is None: |
|
202 |
path = (self.srcdir / desired_path).resolve() |
|
203 |
if not path.is_relative_to(self.srcdir): |
|
204 |
raise FileReferenceError(_('loading_{}_outside_package_dir') |
|
205 |
.format(filename)) |
|
206 |
|
|
207 |
if str(path.relative_to(self.srcdir)) == 'index.json': |
|
208 |
raise FileReferenceError(_('loading_reserved_index_json')) |
|
209 |
else: |
|
210 |
include_in_source_archive = False |
|
212 | 211 |
|
213 |
file_ref = self.files_by_path.get(path) |
|
212 |
file_ref = self.files_by_path.get(desired_path)
|
|
214 | 213 |
if file_ref is None: |
215 | 214 |
with open(path, 'rb') as file_handle: |
216 | 215 |
contents = file_handle.read() |
217 | 216 |
|
218 |
file_ref = FileRef(path, contents) |
|
219 |
self.files_by_path[path] = file_ref |
|
217 |
file_ref = FileRef(desired_path, contents)
|
|
218 |
self.files_by_path[desired_path] = file_ref
|
|
220 | 219 |
|
221 | 220 |
if include_in_distribution: |
222 | 221 |
file_ref.include_in_distribution = True |
223 | 222 |
|
224 |
return file_ref.make_ref_dict(filename) |
|
223 |
if not include_in_source_archive: |
|
224 |
file_ref.include_in_source_archive = False |
|
225 |
|
|
226 |
return file_ref.make_ref_dict() |
|
225 | 227 |
|
226 |
def _prepare_source_package_zip(self, root_dir_name: str): |
|
228 |
def _prepare_source_package_zip(self, source_name: str, |
|
229 |
piggybacked: Piggybacked) -> str: |
|
227 | 230 |
""" |
228 | 231 |
Create and store in memory a .zip archive containing files needed to |
229 | 232 |
build this source package. |
230 | 233 |
|
231 |
'root_dir_name' shall not contain any slashes ('/').
|
|
234 |
'src_dir_name' shall not contain any slashes ('/').
|
|
232 | 235 |
|
233 | 236 |
Return zipfile's sha256 sum's hexstring. |
234 | 237 |
""" |
235 |
fb = FileBuffer() |
|
236 |
root_dir_path = Path(root_dir_name) |
|
238 |
tf = TemporaryFile() |
|
239 |
source_dir_path = PurePosixPath(source_name) |
|
240 |
piggybacked_dir_path = PurePosixPath(f'{source_name}.foreign-packages') |
|
237 | 241 |
|
238 |
def zippath(file_path): |
|
239 |
file_path = root_dir_path / file_path.relative_to(self.srcdir) |
|
240 |
return file_path.as_posix() |
|
241 |
|
|
242 |
with zipfile.ZipFile(fb, 'w') as xpi: |
|
242 |
with zipfile.ZipFile(tf, 'w') as zf: |
|
243 | 243 |
for file_ref in self.files_by_path.values(): |
244 |
if file_ref.include_in_zipfile: |
|
245 |
xpi.writestr(zippath(file_ref.path), file_ref.contents) |
|
244 |
if file_ref.include_in_source_archive: |
|
245 |
zf.writestr(str(source_dir_path / file_ref.path), |
|
246 |
file_ref.contents) |
|
247 |
|
|
248 |
for desired_path, real_path in piggybacked.archive_files(): |
|
249 |
zf.writestr(str(piggybacked_dir_path / desired_path), |
|
250 |
real_path.read_bytes()) |
|
246 | 251 |
|
247 |
self.source_zip_contents = fb.get_bytes() |
|
252 |
tf.seek(0) |
|
253 |
self.source_zip_contents = tf.read() |
|
248 | 254 |
|
249 | 255 |
return sha256(self.source_zip_contents).digest().hex() |
250 | 256 |
|
251 |
def _process_item(self, item_def: dict): |
|
257 |
def _process_item(self, item_def: dict, piggybacked: Piggybacked):
|
|
252 | 258 |
""" |
253 | 259 |
Process 'item_def' as definition of a resource/mapping and store in |
254 | 260 |
memory its processed form and files used by it. |
... | ... | |
266 | 272 |
|
267 | 273 |
copy_props.append('revision') |
268 | 274 |
|
269 |
script_file_refs = [self._process_file(f['file']) |
|
275 |
script_file_refs = [self._process_file(f['file'], piggybacked)
|
|
270 | 276 |
for f in item_def.get('scripts', [])] |
271 | 277 |
|
272 | 278 |
deps = [{'identifier': res_ref['identifier']} |
273 | 279 |
for res_ref in item_def.get('dependencies', [])] |
274 | 280 |
|
275 | 281 |
new_item_obj = { |
276 |
'dependencies': deps,
|
|
282 |
'dependencies': [*piggybacked.package_must_depend, *deps],
|
|
277 | 283 |
'scripts': script_file_refs |
278 | 284 |
} |
279 | 285 |
else: |
... | ... | |
308 | 314 |
in it. |
309 | 315 |
""" |
310 | 316 |
index_validator.validate(index_obj) |
317 |
match = re.match(r'.*-((([1-9][0-9]*|0)\.)+)schema\.json$', |
|
318 |
index_obj['$schema']) |
|
319 |
self.source_schema_ver = \ |
|
320 |
[int(n) for n in filter(None, match.group(1).split('.'))] |
|
311 | 321 |
|
312 |
schema = f'{schemas_root}/api_source_description-1.schema.json' |
|
322 |
out_schema = f'{schemas_root}/api_source_description-1.schema.json'
|
|
313 | 323 |
|
314 | 324 |
self.source_name = index_obj['source_name'] |
315 | 325 |
|
316 | 326 |
generate_spdx = index_obj.get('reuse_generate_spdx_report', False) |
317 | 327 |
if generate_spdx: |
318 | 328 |
contents = generate_spdx_report(self.srcdir) |
319 |
spdx_path = (self.srcdir / 'report.spdx').resolve()
|
|
329 |
spdx_path = PurePosixPath('report.spdx')
|
|
320 | 330 |
spdx_ref = FileRef(spdx_path, contents) |
321 | 331 |
|
322 |
spdx_ref.include_in_zipfile = False
|
|
332 |
spdx_ref.include_in_source_archive = False
|
|
323 | 333 |
self.files_by_path[spdx_path] = spdx_ref |
324 | 334 |
|
325 |
self.copyright_file_refs = \ |
|
326 |
[self._process_file(f['file']) for f in index_obj['copyright']] |
|
335 |
piggyback_def = None |
|
336 |
if self.source_schema_ver >= [1, 1] and 'piggyback_on' in index_obj: |
|
337 |
piggyback_def = index_obj['piggyback_on'] |
|
327 | 338 |
|
328 |
if generate_spdx and not spdx_ref.include_in_distribution: |
|
329 |
raise FileReferenceError(_('report_spdx_not_in_copyright_list')) |
|
339 |
with piggybacked_system(piggyback_def, self.piggyback_files) \ |
|
340 |
as piggybacked: |
|
341 |
copyright_to_process = [ |
|
342 |
*(file_ref['file'] for file_ref in index_obj['copyright']), |
|
343 |
*piggybacked.package_license_files |
|
344 |
] |
|
345 |
self.copyright_file_refs = [self._process_file(f, piggybacked) |
|
346 |
for f in copyright_to_process] |
|
330 | 347 |
|
331 |
item_refs = [self._process_item(d) for d in index_obj['definitions']] |
|
348 |
if generate_spdx and not spdx_ref.include_in_distribution: |
|
349 |
raise FileReferenceError(_('report_spdx_not_in_copyright_list')) |
|
332 | 350 |
|
333 |
for file_ref in index_obj.get('additional_files', []):
|
|
334 |
self._process_file(file_ref['file'], include_in_distribution=False)
|
|
351 |
item_refs = [self._process_item(d, piggybacked)
|
|
352 |
for d in index_obj['definitions']]
|
|
335 | 353 |
|
336 |
root_dir_path = Path(self.source_name) |
|
354 |
for file_ref in index_obj.get('additional_files', []): |
|
355 |
self._process_file(file_ref['file'], piggybacked, |
|
356 |
include_in_distribution=False) |
|
337 | 357 |
|
338 |
source_archives_obj = { |
|
339 |
'zip' : { |
|
340 |
'sha256': self._prepare_source_package_zip(root_dir_path) |
|
341 |
} |
|
342 |
} |
|
358 |
zipfile_sha256 = self._prepare_source_package_zip\ |
|
359 |
(self.source_name, piggybacked) |
|
360 |
|
|
361 |
source_archives_obj = {'zip' : {'sha256': zipfile_sha256}} |
|
343 | 362 |
|
344 | 363 |
self.source_description = { |
345 |
'$schema': schema, |
|
364 |
'$schema': out_schema,
|
|
346 | 365 |
'source_name': self.source_name, |
347 | 366 |
'source_copyright': self.copyright_file_refs, |
348 | 367 |
'upstream_url': index_obj['upstream_url'], |
... | ... | |
398 | 417 |
|
399 | 418 |
dir_type = click.Path(exists=True, file_okay=False, resolve_path=True) |
400 | 419 |
|
420 |
@click.command(help=_('build_package_from_srcdir_to_dstdir')) |
|
401 | 421 |
@click.option('-s', '--srcdir', default='./', type=dir_type, show_default=True, |
402 | 422 |
help=_('source_directory_to_build_from')) |
403 | 423 |
@click.option('-i', '--index-json', default='index.json', type=click.Path(), |
404 | 424 |
help=_('path_instead_of_index_json')) |
425 |
@click.option('-p', '--piggyback-files', type=click.Path(), |
|
426 |
help=_('path_instead_for_piggyback_files')) |
|
405 | 427 |
@click.option('-d', '--dstdir', type=dir_type, required=True, |
406 | 428 |
help=_('built_package_files_destination')) |
407 | 429 |
@click.version_option(version=_version.version, prog_name='Hydrilla builder', |
408 | 430 |
message=_('%(prog)s_%(version)s_license'), |
409 | 431 |
help=_('version_printing')) |
410 |
def perform(srcdir, index_json, dstdir): |
|
411 |
"""<this will be replaced by a localized docstring for Click to pick up>""" |
|
412 |
build = Build(Path(srcdir), Path(index_json)) |
|
413 |
build.write_package_files(Path(dstdir)) |
|
414 |
|
|
415 |
perform.__doc__ = _('build_package_from_srcdir_to_dstdir') |
|
432 |
def perform(srcdir, index_json, piggyback_files, dstdir): |
|
433 |
""" |
|
434 |
Execute Hydrilla builder to turn source package into a distributable one. |
|
416 | 435 |
|
417 |
perform = click.command()(perform) |
|
436 |
This command is meant to be the entry point of hydrilla-builder command |
|
437 |
exported by this package. |
|
438 |
""" |
|
439 |
build = Build(Path(srcdir), Path(index_json), |
|
440 |
piggyback_files and Path(piggyback_files)) |
|
441 |
build.write_package_files(Path(dstdir)) |
Also available in: Unified diff
support piggybacking on APT packages