Revision f8dedf60
Added by koszko about 1 year ago
| common/entities.js | ||
|---|---|---|
| 116 | 116 |
} |
| 117 | 117 |
#EXPORT get_used_files AS get_files |
| 118 | 118 |
|
| 119 |
/* |
|
| 120 |
* Regex to parse URIs like: |
|
| 121 |
* https://hydrilla.koszko.org/schemas/api_mapping_description-2.schema.json |
|
| 122 |
*/ |
|
| 123 |
const name_base_re = "(?<name_base>[^/]*)"; |
|
| 124 |
const major_number_re = "(?<major>[1-9][0-9]*)"; |
|
| 125 |
const minor_number_re = "(?:[1-9][0-9]*|0)"; |
|
| 126 |
const numbers_rest_re = `(?:\\.${minor_number_re})*`;
|
|
| 127 |
const version_re = `(?<ver>${major_number_re}${numbers_rest_re})`;
|
|
| 128 |
const schema_name_re = `${name_base_re}-${version_re}\\.schema\\.json`;
|
|
| 129 |
|
|
| 130 |
const haketilo_schema_name_regex = new RegExp(schema_name_re); |
|
| 131 |
#EXPORT haketilo_schema_name_regex |
|
| 132 |
|
|
| 133 |
/* Extract the number that indicates entity's compatibility mode. */ |
|
| 134 |
function get_schema_major_version(instance) {
|
|
| 135 |
const match = haketilo_schema_name_regex.exec(instance.$schema); |
|
| 136 |
|
|
| 137 |
return parseInt(match.groups.major); |
|
| 138 |
} |
|
| 139 |
#EXPORT get_schema_major_version |
|
| 140 |
|
|
| 119 | 141 |
#IF NEVER |
| 120 | 142 |
|
| 121 | 143 |
/* |
| common/jsonschema.js | ||
|---|---|---|
| 57 | 57 |
|
| 58 | 58 |
#FROM common/jsonschema/scan.js IMPORT SchemaScanResult, scan |
| 59 | 59 |
|
| 60 |
#FROM common/entities.js IMPORT haketilo_schema_name_regex |
|
| 61 |
|
|
| 60 | 62 |
#EXPORT scan |
| 61 | 63 |
#EXPORT SchemaScanResult |
| 62 | 64 |
|
| ... | ... | |
| 86 | 88 |
#INCLUDE schemas/2.x/common_definitions-2.schema.json |
| 87 | 89 |
].reduce((ac, s) => Object.assign(ac, {[s.$id]: s}), {});
|
| 88 | 90 |
|
| 89 |
const name_base_re = "(?<name_base>[^/]*)"; |
|
| 90 |
const major_number_re = "(?<major>[1-9][0-9]*)"; |
|
| 91 |
const minor_number_re = "(?:[1-9][0-9]*|0)"; |
|
| 92 |
const numbers_rest_re = `(?:\\.${minor_number_re})*`;
|
|
| 93 |
const version_re = `(?<ver>${major_number_re}${numbers_rest_re})`;
|
|
| 94 |
const schema_name_re = `${name_base_re}-${version_re}\\.schema\\.json`;
|
|
| 95 |
|
|
| 96 |
const haketilo_schema_name_regex = new RegExp(schema_name_re); |
|
| 97 |
|
|
| 98 | 91 |
for (const [$id, schema] of [...Object.entries(haketilo_schemas)]) {
|
| 99 | 92 |
const match = haketilo_schema_name_regex.exec($id); |
| 100 | 93 |
const schema_name = |
| ... | ... | |
| 103 | 96 |
} |
| 104 | 97 |
|
| 105 | 98 |
#EXPORT haketilo_schemas |
| 106 |
#EXPORT haketilo_schema_name_regex |
|
| 107 | 99 |
|
| 108 | 100 |
const haketilo_validator = new Validator(); |
| 109 | 101 |
Object.values(haketilo_schemas) |
| common/policy.js | ||
|---|---|---|
| 49 | 49 |
* CSP rule that either blocks all scripts or only allows scripts with specified |
| 50 | 50 |
* nonce attached. |
| 51 | 51 |
*/ |
| 52 |
function make_csp(nonce) |
|
| 53 |
{
|
|
| 54 |
const rule = nonce ? `nonce-${nonce}` : "none";
|
|
| 52 |
function make_csp(nonce) {
|
|
| 53 |
const rule = nonce ? `'nonce-${nonce}'` : "'none'";
|
|
| 55 | 54 |
const csp_list = [ |
| 56 |
["prefetch-src", "none"],
|
|
| 57 |
["script-src-attr", "none"],
|
|
| 58 |
["script-src", rule], |
|
| 55 |
["prefetch-src", "'none'"],
|
|
| 56 |
["script-src-attr", "'none'"],
|
|
| 57 |
["script-src", rule, "'unsafe-eval'"],
|
|
| 59 | 58 |
["script-src-elem", rule] |
| 60 | 59 |
]; |
| 61 |
return csp_list.map(([a, b]) => `${a} '${b}';`).join(" ");
|
|
| 60 |
return csp_list.map(words => `${words.join(" ")};`).join(" ");
|
|
| 62 | 61 |
} |
| 63 | 62 |
|
| 64 | 63 |
function decide_policy(patterns_tree, url, default_allow, secret) |
| ... | ... | |
| 113 | 112 |
#EXPORT decide_policy |
| 114 | 113 |
|
| 115 | 114 |
#EXPORT () => ({allow: false, csp: make_csp()}) AS fallback_policy
|
| 115 |
|
|
| 116 |
#IF NEVER |
|
| 117 |
|
|
| 118 |
/* |
|
| 119 |
* Note: the functions below were overeagerly written and are not used now but |
|
| 120 |
* might prove useful to once we add more functionalities and are hence kept... |
|
| 121 |
*/ |
|
| 122 |
|
|
| 123 |
function relaxed_csp_eval(csp) {
|
|
| 124 |
const new_csp_list = []; |
|
| 125 |
|
|
| 126 |
for (const directive of csp.split(";")) {
|
|
| 127 |
const directive_words = directive.trim().split(" ");
|
|
| 128 |
if (directive_words[0] === "script-src") |
|
| 129 |
directive_words.push("'unsafe-eval'");
|
|
| 130 |
|
|
| 131 |
new_csp_list.push(directive_words); |
|
| 132 |
} |
|
| 133 |
|
|
| 134 |
new_policy.csp = new_csp_list.map(d => `${d.join(" ")}';`).join(" ");
|
|
| 135 |
} |
|
| 136 |
|
|
| 137 |
function relax_policy_eval(policy) {
|
|
| 138 |
const new_policy = Object.assign({}, policy);
|
|
| 139 |
|
|
| 140 |
return Object.assign(new_policy, {csp: relaxed_csp_eval(policy.csp)});
|
|
| 141 |
} |
|
| 142 |
#EXPORT relax_policy_eval |
|
| 143 |
|
|
| 144 |
#ENDIF |
|
| html/install.js | ||
|---|---|---|
| 49 | 49 |
#FROM html/DOM_helpers.js IMPORT clone_template, Showable |
| 50 | 50 |
#FROM common/entities.js IMPORT item_id_string, version_string, get_files |
| 51 | 51 |
#FROM common/misc.js IMPORT sha256_async AS compute_sha256 |
| 52 |
#FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas, \
|
|
| 53 |
haketilo_schema_name_regex
|
|
| 52 |
#FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas |
|
| 53 |
#FROM common/entities.js IMPORT haketilo_schema_name_regex
|
|
| 54 | 54 |
|
| 55 | 55 |
#FROM html/repo_query_cacher_client.js IMPORT indirect_fetch |
| 56 | 56 |
|
| test/haketilo_test/unit/test_policy_deciding.py | ||
|---|---|---|
| 23 | 23 |
|
| 24 | 24 |
from ..script_loader import load_script |
| 25 | 25 |
|
| 26 |
csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$') |
|
| 27 |
rule_re = re.compile(r'^\s*(?P<src_kind>\S+)\s+(?P<allowed_origins>\S+)$') |
|
| 26 |
csp_re = re.compile(r''' |
|
| 27 |
^ |
|
| 28 |
\S+(?:\s+\S+)+; # first directive |
|
| 29 |
(?: |
|
| 30 |
\s+\S+(?:\s+\S+)+; # subsequent directive |
|
| 31 |
)* |
|
| 32 |
$ |
|
| 33 |
''', |
|
| 34 |
re.VERBOSE) |
|
| 35 |
|
|
| 36 |
rule_re = re.compile(r''' |
|
| 37 |
^ |
|
| 38 |
\s* |
|
| 39 |
(?P<src_kind>\S+) |
|
| 40 |
\s+ |
|
| 41 |
(?P<allowed_origins> |
|
| 42 |
\S+(?:\s+\S+)* |
|
| 43 |
) |
|
| 44 |
$ |
|
| 45 |
''', re.VERBOSE) |
|
| 46 |
|
|
| 28 | 47 |
def parse_csp(csp): |
| 29 |
''' |
|
| 30 |
Parsing of CSP string into a dict. A simplified format of CSP is assumed. |
|
| 31 |
''' |
|
| 48 |
'''Parsing of CSP string into a dict.''' |
|
| 32 | 49 |
assert csp_re.match(csp) |
| 33 | 50 |
|
| 34 | 51 |
result = {}
|
| 35 | 52 |
|
| 36 | 53 |
for rule in csp.split(';')[:-1]:
|
| 37 | 54 |
match = rule_re.match(rule) |
| 38 |
result[match.group('src_kind')] = match.group('allowed_origins')
|
|
| 55 |
result[match.group('src_kind')] = match.group('allowed_origins').split()
|
|
| 39 | 56 |
|
| 40 | 57 |
return result |
| 41 | 58 |
|
| ... | ... | |
| 78 | 95 |
for prop in ('mapping', 'payload', 'nonce', 'error'):
|
| 79 | 96 |
assert prop not in policy |
| 80 | 97 |
assert parse_csp(policy['csp']) == {
|
| 81 |
'prefetch-src': "'none'",
|
|
| 82 |
'script-src-attr': "'none'",
|
|
| 83 |
'script-src': "'none'",
|
|
| 84 |
'script-src-elem': "'none'"
|
|
| 98 |
'prefetch-src': ["'none'"],
|
|
| 99 |
'script-src-attr': ["'none'"],
|
|
| 100 |
'script-src': ["'none'", "'unsafe-eval'"],
|
|
| 101 |
'script-src-elem': ["'none'"]
|
|
| 85 | 102 |
} |
| 86 | 103 |
|
| 87 | 104 |
policy = execute_in_page( |
| ... | ... | |
| 95 | 112 |
for prop in ('payload', 'nonce', 'error'):
|
| 96 | 113 |
assert prop not in policy |
| 97 | 114 |
assert parse_csp(policy['csp']) == {
|
| 98 |
'prefetch-src': "'none'",
|
|
| 99 |
'script-src-attr': "'none'",
|
|
| 100 |
'script-src': "'none'",
|
|
| 101 |
'script-src-elem': "'none'"
|
|
| 115 |
'prefetch-src': ["'none'"],
|
|
| 116 |
'script-src-attr': ["'none'"],
|
|
| 117 |
'script-src': ["'none'", "'unsafe-eval'"],
|
|
| 118 |
'script-src-elem': ["'none'"]
|
|
| 102 | 119 |
} |
| 103 | 120 |
|
| 104 | 121 |
policy = execute_in_page( |
| ... | ... | |
| 114 | 131 |
assert policy['nonce'] == \ |
| 115 | 132 |
sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex()
|
| 116 | 133 |
assert parse_csp(policy['csp']) == {
|
| 117 |
'prefetch-src': f"'none'",
|
|
| 118 |
'script-src-attr': f"'none'",
|
|
| 119 |
'script-src': f"'nonce-{policy['nonce']}'",
|
|
| 120 |
'script-src-elem': f"'nonce-{policy['nonce']}'"
|
|
| 134 |
'prefetch-src': ["'none'"],
|
|
| 135 |
'script-src-attr': ["'none'"],
|
|
| 136 |
'script-src': [f"'nonce-{policy['nonce']}'", "'unsafe-eval'"],
|
|
| 137 |
'script-src-elem': [f"'nonce-{policy['nonce']}'"]
|
|
| 121 | 138 |
} |
| 122 | 139 |
|
| 123 | 140 |
policy = execute_in_page( |
| ... | ... | |
| 128 | 145 |
for prop in ('mapping', 'payload', 'nonce'):
|
| 129 | 146 |
assert prop not in policy |
| 130 | 147 |
assert parse_csp(policy['csp']) == {
|
| 131 |
'prefetch-src': "'none'",
|
|
| 132 |
'script-src-attr': "'none'",
|
|
| 133 |
'script-src': "'none'",
|
|
| 134 |
'script-src-elem': "'none'"
|
|
| 148 |
'prefetch-src': ["'none'"],
|
|
| 149 |
'script-src-attr': ["'none'"],
|
|
| 150 |
'script-src': ["'none'", "'unsafe-eval'"],
|
|
| 151 |
'script-src-elem': ["'none'"]
|
|
| 135 | 152 |
} |
| test/haketilo_test/unit/test_policy_enforcing.py | ||
|---|---|---|
| 31 | 31 |
allow_policy = {'allow': True}
|
| 32 | 32 |
block_policy = {
|
| 33 | 33 |
'allow': False, |
| 34 |
'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none'; script-src-elem 'none'; frame-src http://* https://*;" |
|
| 34 |
'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none' 'unsafe-eval'; script-src-elem 'none'; frame-src http://* https://*;"
|
|
| 35 | 35 |
} |
| 36 | 36 |
payload_policy = {
|
| 37 | 37 |
'mapping': 'somemapping', |
| 38 | 38 |
'payload': {'identifier': 'someresource'},
|
| 39 |
'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}'; script-src-elem 'nonce-{nonce}';"
|
|
| 39 |
'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}' 'unsafe-eval'; script-src-elem 'nonce-{nonce}';"
|
|
| 40 | 40 |
} |
| 41 | 41 |
|
| 42 | 42 |
def content_script(): |
| test/haketilo_test/unit/test_webrequest.py | ||
|---|---|---|
| 85 | 85 |
payload_csp_header = {
|
| 86 | 86 |
'name': f'Content-Security-Policy', |
| 87 | 87 |
'value': ("prefetch-src 'none'; script-src-attr 'none'; "
|
| 88 |
f"script-src '{nonce}'; script-src-elem '{nonce}';")
|
|
| 88 |
f"script-src '{nonce}' 'unsafe-eval'; script-src-elem '{nonce}';")
|
|
| 89 | 89 |
} |
| 90 | 90 |
|
| 91 | 91 |
sample_payload_headers = [ |
| ... | ... | |
| 107 | 107 |
sample_blocked_headers.append({
|
| 108 | 108 |
'name': f'Content-Security-Policy', |
| 109 | 109 |
'value': ("prefetch-src 'none'; script-src-attr 'none'; "
|
| 110 |
f"script-src 'none'; script-src-elem 'none';")
|
|
| 110 |
"script-src 'none' 'unsafe-eval'; script-src-elem 'none';")
|
|
| 111 | 111 |
}) |
| 112 | 112 |
|
| 113 | 113 |
@pytest.mark.get_page('https://gotmyowndoma.in')
|
| test/haketilo_test/unit/utils.py | ||
|---|---|---|
| 228 | 228 |
return driver.execute_script( |
| 229 | 229 |
''' |
| 230 | 230 |
document.haketilo_scripts_allowed = false; |
| 231 |
document.haketilo_eval_allowed = false; |
|
| 231 | 232 |
const html_ns = "http://www.w3.org/1999/xhtml"; |
| 232 | 233 |
const script = document.createElementNS(html_ns, "script"); |
| 233 |
script.innerHTML = "document.haketilo_scripts_allowed = true;"; |
|
| 234 |
script.innerHTML = ` |
|
| 235 |
document.haketilo_scripts_allowed = true; |
|
| 236 |
eval('document.haketilo_eval_allowed = true;');
|
|
| 237 |
`; |
|
| 234 | 238 |
if (arguments[0]) |
| 235 | 239 |
script.setAttribute("nonce", arguments[0]);
|
| 236 | 240 |
(document.head || document.documentElement).append(script); |
| 241 |
|
|
| 242 |
if (document.haketilo_scripts_allowed != |
|
| 243 |
document.haketilo_eval_allowed) |
|
| 244 |
throw "scripts allowed but eval blocked"; |
|
| 245 |
|
|
| 237 | 246 |
return document.haketilo_scripts_allowed; |
| 238 | 247 |
''', |
| 239 | 248 |
nonce) |
| test/haketilo_test/world_wide_library.py | ||
|---|---|---|
| 152 | 152 |
'id_suffix': 'a-w-required-mapping-v1', |
| 153 | 153 |
'files_count': 1, |
| 154 | 154 |
'dependencies': [], |
| 155 |
'required_mappings': [{'identifier': 'mapping-a'}]
|
|
| 155 |
'required_mappings': [{'identifier': 'mapping-a'}],
|
|
| 156 |
'include_in_query': False |
|
| 156 | 157 |
}) |
| 157 | 158 |
|
| 158 | 159 |
sample_resource_templates.append({
|
| ... | ... | |
| 160 | 161 |
'files_count': 1, |
| 161 | 162 |
'dependencies': [], |
| 162 | 163 |
'required_mappings': [{'identifier': 'mapping-a'}],
|
| 163 |
'schema_ver': '2' |
|
| 164 |
'schema_ver': '2', |
|
| 165 |
'include_in_query': False |
|
| 164 | 166 |
}) |
| 165 | 167 |
|
| 166 | 168 |
sample_resources_catalog = {}
|
| ... | ... | |
| 193 | 195 |
|
| 194 | 196 |
sufs = [srt["id_suffix"], *[l for l in srt["id_suffix"] if l.isalpha()]] |
| 195 | 197 |
patterns = [f'https://example_{suf}.com/*' for suf in set(sufs)]
|
| 196 |
payloads = {}
|
|
| 198 |
mapping['payloads'] = {}
|
|
| 197 | 199 |
|
| 198 | 200 |
for pat in patterns: |
| 199 |
payloads[pat] = {'identifier': resource['identifier']}
|
|
| 201 |
mapping['payloads'][pat] = {'identifier': resource['identifier']}
|
|
| 200 | 202 |
|
| 201 |
queryable_url = pat.replace('*', 'something')
|
|
| 202 |
if queryable_url not in sample_queries: |
|
| 203 |
sample_queries[queryable_url] = [] |
|
| 203 |
if not srt.get('include_in_query', True):
|
|
| 204 |
continue |
|
| 204 | 205 |
|
| 205 |
sample_queries[queryable_url].append({
|
|
| 206 |
sample_queries.setdefault(pat.replace('*', 'something'), []).append({
|
|
| 206 | 207 |
'identifier': mapping['identifier'], |
| 207 | 208 |
'long_name': mapping['long_name'], |
| 208 | 209 |
'version': mapping_versions[1] |
| 209 | 210 |
}) |
| 210 | 211 |
|
| 211 |
mapping['payloads'] = payloads |
|
| 212 |
|
|
| 213 | 212 |
for item in resource, mapping: |
| 214 | 213 |
if 'required_mappings' in srt: |
| 215 | 214 |
item['required_mappings'] = srt['required_mappings'] |
Also available in: Unified diff
allow eval() in injected scripts