Project

General

Profile

« Previous | Next » 

Revision f42f5c19

Added by koszko over 1 year ago

incorporate version 2 of Hydrilla JSON schemas

View differences:

.gitmodules
4 4
#
5 5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6 6

  
7
[submodule "hydrilla-json-schemas"]
8
	path = src/hydrilla/schemas
7
[submodule "hydrilla-json-schemas-1.x"]
8
	path = src/hydrilla/schemas/1.x
9 9
	url = ../hydrilla-json-schemas
10
[submodule "hydrilla-json-schemas-2.x"]
11
	path = src/hydrilla/schemas/2.x
12
	url = ../hydrilla-json-schemas/
13
	branch = koszko
10 14
[submodule "hydrilla-source-package-example"]
11 15
	path = tests/source-package-example
12 16
	url = ../hydrilla-source-package-example
MANIFEST.in
4 4
#
5 5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6 6

  
7
include src/hydrilla/schemas/*.schema.json*
7
include src/hydrilla/schemas/*/*.schema.json*
8 8
include src/hydrilla/builder/locales/*/LC_MESSAGES/hydrilla-messages.po
9 9
include tests/source-package-example/*
10 10
include tests/source-package-example/LICENSES/*
src/hydrilla/builder/build.py
51 51

  
52 52
_ = util.translation(here / 'locales').gettext
53 53

  
54
index_validator = util.validator_for('package_source-2.schema.json')
54
def index_validator(major_schema_version):
55
    """
56
    Create an index.json schema validator specific to the requested schema
57
    version series.
58
    """
59
    exact_version = {1: '1.0.1', 2: '2'}[major_schema_version]
60

  
61
    return util.validator_for(f'package_source-{exact_version}.schema.json')
55 62

  
56 63
schemas_root = 'https://hydrilla.koszko.org/schemas'
57 64

  
......
158 165
        if not index_json_path.is_absolute():
159 166
            index_json_path = (self.srcdir / index_json_path)
160 167

  
161
        with open(index_json_path, 'rt') as index_file:
162
            index_json_text = index_file.read()
168
        index_obj, major = util.load_instance_from_file(index_json_path)
163 169

  
164
        index_obj = json.loads(util.strip_json_comments(index_json_text))
170
        if major not in (1, 2):
171
            msg = _('unknown_schema_package_source_{}')\
172
                .format(index_json_path)
173
            raise util.UnknownSchemaError(msg)
165 174

  
166 175
        index_desired_path = PurePosixPath('index.json')
167 176
        self.files_by_path[index_desired_path] = \
168
            FileRef(index_desired_path, index_json_text.encode())
177
            FileRef(index_desired_path, index_json_path.read_bytes())
169 178

  
170
        self._process_index_json(index_obj)
179
        self._process_index_json(index_obj, major)
171 180

  
172 181
    def _process_file(self, filename: Union[str, PurePosixPath],
173 182
                      piggybacked: Piggybacked,
......
308 317
        props_in_ref = ('type', 'identifier', 'version', 'long_name')
309 318
        return dict([(prop, new_item_obj[prop]) for prop in props_in_ref])
310 319

  
311
    def _process_index_json(self, index_obj: dict):
320
    def _process_index_json(self, index_obj: dict,
321
                            major_schema_version: int) -> None:
312 322
        """
313 323
        Process 'index_obj' as contents of source package's index.json and store
314 324
        in memory this source package's zipfile as well as package's individual
315 325
        files and computed definitions of the source package and items defined
316 326
        in it.
317 327
        """
318
        index_validator.validate(index_obj)
328
        index_validator(major_schema_version).validate(index_obj)
329

  
319 330
        match = re.match(r'.*-((([1-9][0-9]*|0)\.)+)schema\.json$',
320 331
                         index_obj['$schema'])
321 332
        self.source_schema_ver = \
src/hydrilla/builder/locales/en_US/LC_MESSAGES/hydrilla-messages.po
7 7
msgstr ""
8 8
"Project-Id-Version: hydrilla.builder 0.1.dev16+g4e46d7f.d20220211\n"
9 9
"Report-Msgid-Bugs-To: koszko@koszko.org\n"
10
"POT-Creation-Date: 2022-05-11 11:10+0200\n"
10
"POT-Creation-Date: 2022-05-27 18:49+0200\n"
11 11
"PO-Revision-Date: 2022-02-12 00:00+0000\n"
12 12
"Last-Translator: Wojtek Kosior <koszko@koszko.org>\n"
13 13
"Language: en_US\n"
......
18 18
"Content-Transfer-Encoding: 8bit\n"
19 19
"Generated-By: Babel 2.8.0\n"
20 20

  
21
#: src/hydrilla/builder/build.py:86 src/hydrilla/builder/local_apt.py:118
22
#: src/hydrilla/builder/local_apt.py:407
21
#: src/hydrilla/builder/build.py:93 src/hydrilla/builder/local_apt.py:118
22
#: src/hydrilla/builder/local_apt.py:410
23 23
msgid "couldnt_execute_{}_is_it_installed"
24 24
msgstr "Could not execute '{}'. Is the tool installed and reachable via PATH?"
25 25

  
26
#: src/hydrilla/builder/build.py:90 src/hydrilla/builder/local_apt.py:122
27
#: src/hydrilla/builder/local_apt.py:411
26
#: src/hydrilla/builder/build.py:97 src/hydrilla/builder/local_apt.py:122
27
#: src/hydrilla/builder/local_apt.py:414
28 28
msgid "command_{}_failed"
29 29
msgstr "The following command finished execution with a non-zero exit status: {}"
30 30

  
31
#: src/hydrilla/builder/build.py:198
31
#: src/hydrilla/builder/build.py:171
32
msgid "unknown_schema_package_source_{}"
33
msgstr ""
34
"The provided JSON at '{}' does not use any of the known package source "
35
"JSON schemas."
36

  
37
#: src/hydrilla/builder/build.py:207
32 38
msgid "path_contains_double_dot_{}"
33 39
msgstr ""
34 40
"Attempt to load '{}' which includes a forbidden parent reference ('..') "
35 41
"in the path."
36 42

  
37
#: src/hydrilla/builder/build.py:205
43
#: src/hydrilla/builder/build.py:214
38 44
msgid "loading_{}_outside_package_dir"
39 45
msgstr "Attempt to load '{}' which lies outside package source directory."
40 46

  
41
#: src/hydrilla/builder/build.py:209
47
#: src/hydrilla/builder/build.py:218
42 48
msgid "loading_reserved_index_json"
43 49
msgstr "Attempt to load 'index.json' which is a reserved filename."
44 50

  
45
#: src/hydrilla/builder/build.py:216
51
#: src/hydrilla/builder/build.py:225
46 52
msgid "referenced_file_{}_missing"
47 53
msgstr "Referenced file '{}' is missing."
48 54

  
49
#: src/hydrilla/builder/build.py:351
55
#: src/hydrilla/builder/build.py:362
50 56
msgid "report_spdx_not_in_copyright_list"
51 57
msgstr ""
52 58
"Told to generate 'report.spdx' but 'report.spdx' is not listed among "
53 59
"copyright files. Refusing to proceed."
54 60

  
55
#: src/hydrilla/builder/build.py:422
61
#: src/hydrilla/builder/build.py:433
56 62
msgid "build_package_from_srcdir_to_dstdir"
57 63
msgstr ""
58 64
"Build Hydrilla package from `scrdir` and write the resulting files under "
59 65
"`dstdir`."
60 66

  
61
#: src/hydrilla/builder/build.py:424
67
#: src/hydrilla/builder/build.py:435
62 68
msgid "source_directory_to_build_from"
63 69
msgstr "Source directory to build from."
64 70

  
65
#: src/hydrilla/builder/build.py:426
71
#: src/hydrilla/builder/build.py:437
66 72
msgid "path_instead_of_index_json"
67 73
msgstr ""
68 74
"Path to file to be processed instead of index.json (if not absolute, "
69 75
"resolved relative to srcdir)."
70 76

  
71
#: src/hydrilla/builder/build.py:428
77
#: src/hydrilla/builder/build.py:439
72 78
msgid "path_instead_for_piggyback_files"
73 79
msgstr ""
74 80
"Path to a non-standard directory with foreign packages' archive files to "
75 81
"use."
76 82

  
77
#: src/hydrilla/builder/build.py:430
83
#: src/hydrilla/builder/build.py:441
78 84
msgid "built_package_files_destination"
79 85
msgstr "Destination directory to write built package files to."
80 86

  
81
#: src/hydrilla/builder/build.py:432
87
#: src/hydrilla/builder/build.py:443
82 88
#, python-format
83 89
msgid "%(prog)s_%(version)s_license"
84 90
msgstr ""
......
89 95
"This is free software: you are free to change and redistribute it.\n"
90 96
"There is NO WARRANTY, to the extent permitted by law."
91 97

  
92
#: src/hydrilla/builder/build.py:433
98
#: src/hydrilla/builder/build.py:444
93 99
msgid "version_printing"
94 100
msgstr "Print version information and exit."
95 101

  
......
105 111
msgid "distro_{}_unknown"
106 112
msgstr "Attempt to use an unknown software distribution '{}'."
107 113

  
108
#: src/hydrilla/builder/local_apt.py:190
114
#: src/hydrilla/builder/local_apt.py:191
109 115
msgid "couldnt_import_{}_is_it_installed"
110 116
msgstr ""
111 117
"Could not import '{}'. Is the module installed and visible to this Python"
112 118
" instance?"
113 119

  
114
#: src/hydrilla/builder/local_apt.py:198
120
#: src/hydrilla/builder/local_apt.py:199
115 121
msgid "gpg_couldnt_recv_key_{}"
116 122
msgstr "Could not import PGP key '{}'."
117 123

  
118
#: src/hydrilla/builder/local_apt.py:312
124
#: src/hydrilla/builder/local_apt.py:313
119 125
msgid "apt_install_output_not_understood"
120 126
msgstr "The output of an 'apt-get install' command was not understood."
121 127

  
122
#: src/hydrilla/builder/local_apt.py:338
128
#: src/hydrilla/builder/local_apt.py:339
123 129
msgid "apt_download_gave_bad_filename_{}"
124 130
msgstr "The 'apt-get download' command produced a file with unexpected name '{}'."
125 131

  
......
129 135
"Attempt to load '{}' which lies outside piggybacked packages files root "
130 136
"directory."
131 137

  
132
#: src/hydrilla/util/_util.py:79
138
#: src/hydrilla/util/_util.py:86
133 139
msgid "bad_comment"
134 140
msgstr "bad comment"
135 141

  
src/hydrilla/schemas
1
Subproject commit 4b4da5a02bc311603469eea7b3dfd4f1bbb911fd
src/hydrilla/schemas/1.x
1
Subproject commit 09634f3446866f712a022327683b1149d8f46bf0
src/hydrilla/schemas/2.x
1
Subproject commit 6b6ae219929dc1d47e1dff2a780784b78bd825b8
src/hydrilla/util/__init__.py
4 4
#
5 5
# Available under the terms of Creative Commons Zero v1.0 Universal.
6 6

  
7
from ._util import strip_json_comments, normalize_version, parse_version, \
8
    version_string, validator_for, translation
7
from ._util import normalize_version, parse_version, version_string, \
8
    validator_for, load_instance_from_file, translation, UnknownSchemaError
src/hydrilla/util/_util.py
39 39

  
40 40
here = Path(__file__).resolve().parent
41 41

  
42
class UnknownSchemaError(Exception):
43
    """
44
    Exception used to record problems with JSON documents for which not even
45
    the appropriate validation schema could be determined.
46
    """
47
    pass
48

  
42 49
_strip_comment_re = re.compile(r'''
43 50
^ # match from the beginning of each line
44 51
( # catch the part before '//' comment
......
111 118
    return '.'.join([str(n) for n in ver]) + ('' if rev is None else f'-{rev}')
112 119

  
113 120
schemas = {}
114
for path in (here.parent / 'schemas').glob('*-1.0.1.schema.json'):
115
    schema = json.loads(path.read_text())
116
    schemas[schema['$id']] = schema
121
for series_dir in (here.parent / 'schemas').glob('*.x'):
122
    for path in series_dir.glob("*.schema.json"):
123
        schema = json.loads(path.read_text())
124
        schemas[schema['$id']] = schema
117 125

  
118 126
common_schema_filename = 'common_definitions-1.schema.json'
119 127
common_schema_path = here.parent / "schemas" / common_schema_filename
120 128

  
121
resolver = RefResolver(
122
    base_uri=f'file://{str(common_schema_path)}',
123
    referrer=f'https://hydrilla.koszko.org/{common_schema_filename}',
124
    store=schemas
125
)
129
def validator_for(schema: Union[str, dict]) -> Draft7Validator:
130
    """
131
    Prepare a validator for the provided schema.
126 132

  
127
def validator_for(schema_filename: str) -> Draft7Validator:
133
    Other schemas under '../schemas' can be referenced.
128 134
    """
129
    Prepare a validator for one of the schemas in '../schemas'.
135
    if isinstance(schema, str):
136
        schema = schemas[f'https://hydrilla.koszko.org/schemas/{schema}']
137

  
138
    resolver = RefResolver(
139
        base_uri=schema['$id'],
140
        referrer=schema,
141
        handlers={'https': lambda uri: schemas[uri]}
142
    )
143

  
144
    return Draft7Validator(schema, resolver=resolver)
145

  
146
_major_version_re = re.compile(r'''
147
-
148
(?P<major>[1-9][0-9]*)
149
(?: # this repeated group matches the remaining version numbers
150
  \.
151
  (?:[1-9][0-9]*|0)
152
)*
153
\.schema\.json
154
$
155
''', re.VERBOSE)
156

  
157
def load_instance_from_file(path: Path) -> tuple[dict, Optional[int]]:
158
    """
159
    Open a file and load its contents as a JSON document (with additional
160
    '//' comments support). Then parse its "$schema" property (if present)
161
    and return a tuple of the document instance and the major number of
162
    schema version.
130 163

  
131
    This function is not thread-safe.
164
    If no schema version number can be extracted, None is used instead.
132 165
    """
133
    return Draft7Validator(resolver.resolve(schema_filename)[1],
134
                           resolver=resolver)
166
    instance = json.loads(strip_json_comments(path.read_text()))
167
    major = None
168

  
169
    if type(instance) is dict and type(instance.get('$schema')) is str:
170
        match = _major_version_re.search(instance.get('$schema'))
171
        major = match and int(match.group('major'))
172

  
173
    return instance, major
135 174

  
136 175
def translation(localedir: Union[Path, str], lang: Optional[str]=None) \
137 176
    -> gettext.GNUTranslations:
tests/test_build.py
34 34

  
35 35
orig_srcdir = here / 'source-package-example'
36 36

  
37
index_text = (orig_srcdir / 'index.json').read_text()
38
index_obj = json.loads(hydrilla_util.strip_json_comments(index_text))
37
index_obj, _ = hydrilla_util.load_instance_from_file(orig_srcdir / 'index.json')
39 38

  
40 39
def read_files(*file_list):
41 40
    """

Also available in: Unified diff