Revision 1c65dd5c
Added by koszko over 1 year ago
background/indexeddb_files_server.js | ||
---|---|---|
59 | 59 |
getting.defs_by_res_id.set(id, definition); |
60 | 60 |
|
61 | 61 |
const file_proms = (definition.scripts || []) |
62 |
.map(s => haketilodb.idb_get(getting.tx, "files", s.hash_key));
|
|
62 |
.map(s => haketilodb.idb_get(getting.tx, "files", s.sha256));
|
|
63 | 63 |
|
64 | 64 |
const deps_proms = (definition.dependencies || []) |
65 | 65 |
.map(dep_id => get_resource_files(getting, dep_id)); |
common/indexeddb.js | ||
---|---|---|
3 | 3 |
* |
4 | 4 |
* Function: Facilitate use of IndexedDB within Haketilo. |
5 | 5 |
* |
6 |
* Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> |
|
6 |
* Copyright (C) 2021, 2022 Wojtek Kosior <koszko@koszko.org>
|
|
7 | 7 |
* |
8 | 8 |
* This program is free software: you can redistribute it and/or modify |
9 | 9 |
* it under the terms of the GNU General Public License as published by |
... | ... | |
59 | 59 |
const version_nr = ver => ver.slice(0, 3).reduce(nr_reductor, [2, 0])[1]; |
60 | 60 |
|
61 | 61 |
const stores = [ |
62 |
["files", {keyPath: "hash_key"}],
|
|
63 |
["file_uses", {keyPath: "hash_key"}],
|
|
62 |
["files", {keyPath: "sha256"}],
|
|
63 |
["file_uses", {keyPath: "sha256"}],
|
|
64 | 64 |
["resource", {keyPath: "identifier"}], |
65 | 65 |
["mapping", {keyPath: "identifier"}], |
66 | 66 |
["settings", {keyPath: "name"}], |
... | ... | |
110 | 110 |
for (const [store_name, key_mode] of stores) |
111 | 111 |
store = opened_db.createObjectStore(store_name, key_mode); |
112 | 112 |
|
113 |
const ctx = make_context(store.transaction, initial_data.files);
|
|
113 |
const ctx = make_context(store.transaction, initial_data.file); |
|
114 | 114 |
await _save_items(initial_data.resources, initial_data.mappings, ctx); |
115 | 115 |
|
116 | 116 |
return opened_db; |
... | ... | |
175 | 175 |
|
176 | 176 |
/* |
177 | 177 |
* item_store_names should be an array with either string "mapping", string |
178 |
* "resource" or both. files should be an object with values being contents of |
|
179 |
* files that are to be possibly saved in this transaction and keys of the form |
|
180 |
* `sha256-<file's-sha256-sum>`. |
|
178 |
* "resource" or both. files should be an object with an "sha256" property whose |
|
179 |
* values will be yet another object with values being contents of files that |
|
180 |
* are to be possibly saved in this transaction and keys being hexadecimal |
|
181 |
* representations of files' SHA256 sums. |
|
181 | 182 |
* |
182 | 183 |
* Returned is a context object wrapping the transaction and handling the |
183 | 184 |
* counting of file references in IndexedDB. |
... | ... | |
192 | 193 |
|
193 | 194 |
async function incr_file_uses(context, file_ref, by=1) |
194 | 195 |
{ |
195 |
const hash_key = file_ref.hash_key;
|
|
196 |
let uses = context.file_uses[hash_key];
|
|
196 |
const sha256 = file_ref.sha256;
|
|
197 |
let uses = context.file_uses[sha256];
|
|
197 | 198 |
if (uses === undefined) { |
198 |
uses = await idb_get(context.transaction, "file_uses", hash_key);
|
|
199 |
uses = await idb_get(context.transaction, "file_uses", sha256);
|
|
199 | 200 |
if (uses) |
200 | 201 |
[uses.new, uses.initial] = [false, uses.uses]; |
201 | 202 |
else |
202 |
uses = {hash_key, uses: 0, new: true, initial: 0};
|
|
203 |
uses = {sha256, uses: 0, new: true, initial: 0};
|
|
203 | 204 |
|
204 |
context.file_uses[hash_key] = uses;
|
|
205 |
context.file_uses[sha256] = uses;
|
|
205 | 206 |
} |
206 | 207 |
|
207 | 208 |
uses.uses = uses.uses + by; |
... | ... | |
213 | 214 |
{ |
214 | 215 |
for (const uses of Object.values(context.file_uses)) { |
215 | 216 |
if (uses.uses < 0) |
216 |
console.error("internal error: uses < 0 for file " + uses.hash_key);
|
|
217 |
console.error("internal error: uses < 0 for file " + uses.sha256);
|
|
217 | 218 |
|
218 | 219 |
const is_new = uses.new; |
219 | 220 |
const initial_uses = uses.initial; |
220 |
const hash_key = uses.hash_key;
|
|
221 |
const sha256 = uses.sha256;
|
|
221 | 222 |
|
222 | 223 |
delete uses.new; |
223 | 224 |
delete uses.initial; |
224 | 225 |
|
225 | 226 |
if (uses.uses < 1) { |
226 | 227 |
if (!is_new) { |
227 |
idb_del(context.transaction, "file_uses", hash_key);
|
|
228 |
idb_del(context.transaction, "files", hash_key);
|
|
228 |
idb_del(context.transaction, "file_uses", sha256);
|
|
229 |
idb_del(context.transaction, "files", sha256);
|
|
229 | 230 |
} |
230 | 231 |
|
231 | 232 |
continue; |
... | ... | |
239 | 240 |
if (initial_uses > 0) |
240 | 241 |
continue; |
241 | 242 |
|
242 |
const file = context.files[hash_key];
|
|
243 |
const file = context.files.sha256[sha256];
|
|
243 | 244 |
if (file === undefined) { |
244 | 245 |
context.transaction.abort(); |
245 |
throw "file not present: " + hash_key;
|
|
246 |
throw "file not present: " + sha256;
|
|
246 | 247 |
} |
247 | 248 |
|
248 |
idb_put(context.transaction, "files", {hash_key, contents: file});
|
|
249 |
idb_put(context.transaction, "files", {sha256, contents: file});
|
|
249 | 250 |
} |
250 | 251 |
|
251 | 252 |
return context.result; |
... | ... | |
283 | 284 |
* } |
284 | 285 |
* }, |
285 | 286 |
* }, |
286 |
* files: { |
|
287 |
* "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n", |
|
288 |
* "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");" |
|
287 |
* file: { |
|
288 |
* sha256: { |
|
289 |
* "f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n", |
|
290 |
* "b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");" |
|
291 |
* } |
|
289 | 292 |
* } |
290 | 293 |
* } |
291 | 294 |
*/ |
292 | 295 |
async function save_items(data) |
293 | 296 |
{ |
294 | 297 |
const item_store_names = ["resource", "mapping"]; |
295 |
const context = await start_items_transaction(item_store_names, data.files);
|
|
298 |
const context = await start_items_transaction(item_store_names, data.file); |
|
296 | 299 |
|
297 | 300 |
return _save_items(data.resources, data.mappings, context); |
298 | 301 |
} |
default_settings.json | ||
---|---|---|
50 | 50 |
"source_name": "haketilo-default-settings", |
51 | 51 |
"source_copyright": [{ |
52 | 52 |
"file": "CC0-1.0.txt", |
53 |
"hash_key": "sha256-a2010f343487d3f7618affe54f789f5487602331c0a8d03f49e9a7c547cf0499"
|
|
53 |
"sha256": "a2010f343487d3f7618affe54f789f5487602331c0a8d03f49e9a7c547cf0499"
|
|
54 | 54 |
}], |
55 | 55 |
"type": "mapping", |
56 | 56 |
"identifier": "haketilo-demo-message", |
... | ... | |
67 | 67 |
} |
68 | 68 |
}, |
69 | 69 |
"files": { |
70 |
"sha256-a2010f343487d3f7618affe54f789f5487602331c0a8d03f49e9a7c547cf0499": "Creative Commons Legal Code\n\nCC0 1.0 Universal\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN\n ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS\n INFORMATION ON AN \"AS-IS\" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES\n REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS\n PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM\n THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED\n HEREUNDER.\n\nStatement of Purpose\n\nThe laws of most jurisdictions throughout the world automatically confer\nexclusive Copyright and Related Rights (defined below) upon the creator\nand subsequent owner(s) (each and all, an \"owner\") of an original work of\nauthorship and/or a database (each, a \"Work\").\n\nCertain owners wish to permanently relinquish those rights to a Work for\nthe purpose of contributing to a commons of creative, cultural and\nscientific works (\"Commons\") that the public can reliably and without fear\nof later claims of infringement build upon, modify, incorporate in other\nworks, reuse and redistribute as freely as possible in any form whatsoever\nand for any purposes, including without limitation commercial purposes.\nThese owners may contribute to the Commons to promote the ideal of a free\nculture and the further production of creative, cultural and scientific\nworks, or to gain reputation or greater distribution for their Work in\npart through the use and efforts of others.\n\nFor these and/or other purposes and motivations, and without any\nexpectation of additional consideration or compensation, the person\nassociating CC0 with a Work (the \"Affirmer\"), to the extent that he or she\nis an owner of Copyright and Related Rights in the Work, voluntarily\nelects to apply CC0 to the Work and publicly distribute the Work under its\nterms, with knowledge of his or her Copyright and Related Rights in the\nWork and the meaning and intended legal effect of CC0 on those rights.\n\n1. Copyright and Related Rights. A Work made available under CC0 may be\nprotected by copyright and related or neighboring rights (\"Copyright and\nRelated Rights\"). Copyright and Related Rights include, but are not\nlimited to, the following:\n\n i. the right to reproduce, adapt, distribute, perform, display,\n communicate, and translate a Work;\n ii. moral rights retained by the original author(s) and/or performer(s);\niii. publicity and privacy rights pertaining to a person's image or\n likeness depicted in a Work;\n iv. rights protecting against unfair competition in regards to a Work,\n subject to the limitations in paragraph 4(a), below;\n v. rights protecting the extraction, dissemination, use and reuse of data\n in a Work;\n vi. database rights (such as those arising under Directive 96/9/EC of the\n European Parliament and of the Council of 11 March 1996 on the legal\n protection of databases, and under any national implementation\n thereof, including any amended or successor version of such\n directive); and\nvii. other similar, equivalent or corresponding rights throughout the\n world based on applicable law or treaty, and any national\n implementations thereof.\n\n2. Waiver. To the greatest extent permitted by, but not in contravention\nof, applicable law, Affirmer hereby overtly, fully, permanently,\nirrevocably and unconditionally waives, abandons, and surrenders all of\nAffirmer's Copyright and Related Rights and associated claims and causes\nof action, whether now known or unknown (including existing as well as\nfuture claims and causes of action), in the Work (i) in all territories\nworldwide, (ii) for the maximum duration provided by applicable law or\ntreaty (including future time extensions), (iii) in any current or future\nmedium and for any number of copies, and (iv) for any purpose whatsoever,\nincluding without limitation commercial, advertising or promotional\npurposes (the \"Waiver\"). Affirmer makes the Waiver for the benefit of each\nmember of the public at large and to the detriment of Affirmer's heirs and\nsuccessors, fully intending that such Waiver shall not be subject to\nrevocation, rescission, cancellation, termination, or any other legal or\nequitable action to disrupt the quiet enjoyment of the Work by the public\nas contemplated by Affirmer's express Statement of Purpose.\n\n3. Public License Fallback. Should any part of the Waiver for any reason\nbe judged legally invalid or ineffective under applicable law, then the\nWaiver shall be preserved to the maximum extent permitted taking into\naccount Affirmer's express Statement of Purpose. In addition, to the\nextent the Waiver is so judged Affirmer hereby grants to each affected\nperson a royalty-free, non transferable, non sublicensable, non exclusive,\nirrevocable and unconditional license to exercise Affirmer's Copyright and\nRelated Rights in the Work (i) in all territories worldwide, (ii) for the\nmaximum duration provided by applicable law or treaty (including future\ntime extensions), (iii) in any current or future medium and for any number\nof copies, and (iv) for any purpose whatsoever, including without\nlimitation commercial, advertising or promotional purposes (the\n\"License\"). The License shall be deemed effective as of the date CC0 was\napplied by Affirmer to the Work. Should any part of the License for any\nreason be judged legally invalid or ineffective under applicable law, such\npartial invalidity or ineffectiveness shall not invalidate the remainder\nof the License, and in such case Affirmer hereby affirms that he or she\nwill not (i) exercise any of his or her remaining Copyright and Related\nRights in the Work or (ii) assert any associated claims and causes of\naction with respect to the Work, in either case contrary to Affirmer's\nexpress Statement of Purpose.\n\n4. Limitations and Disclaimers.\n\n a. No trademark or patent rights held by Affirmer are waived, abandoned,\n surrendered, licensed or otherwise affected by this document.\n b. Affirmer offers the Work as-is and makes no representations or\n warranties of any kind concerning the Work, express, implied,\n statutory or otherwise, including without limitation warranties of\n title, merchantability, fitness for a particular purpose, non\n infringement, or the absence of latent or other defects, accuracy, or\n the present or absence of errors, whether or not discoverable, all to\n the greatest extent permissible under applicable law.\n c. Affirmer disclaims responsibility for clearing rights of other persons\n that may apply to the Work or any use thereof, including without\n limitation any person's Copyright and Related Rights in the Work.\n Further, Affirmer disclaims responsibility for obtaining any necessary\n consents, permissions or other rights required for any use of the\n Work.\n d. Affirmer understands and acknowledges that Creative Commons is not a\n party to this document and has no duty or obligation with respect to\n this CC0 or use of the Work.\n", |
|
71 |
"sha256-f1d9dc9f2f4edaaebd9167eb8b79a6da6434313df47921aa4ea8ae278f7739e3": "/**\n * Haketilo demo script.\n *\n * Copyright (C) 2021 Wojtek Kosior\n * Available under the terms of Creative Commons Zero\n * <https://creativecommons.org/publicdomain/zero/1.0/legalcode>\n */\n\n{\n const banner = document.createElement(\"h2\");\n \n banner.textContent = \"Hoooray! Haketilo works :D\";\n \n banner.setAttribute(\"style\", `\\\n margin: 1em; \\\n border-radius: 1em 0px; \\\n background-color: #474; \\\n padding: 10px 20px; \\\n color: #eee; \\\n box-shadow: 0 6px 8px 0 rgba(0,0,0,0.24), 0 17px 50px 0 rgba(0,0,0,0.19); \\\n display: inline-block;\\\n `);\n \n document.body.prepend(banner);\n}\n" |
|
70 |
"sha256": { |
|
71 |
"a2010f343487d3f7618affe54f789f5487602331c0a8d03f49e9a7c547cf0499": "Creative Commons Legal Code\n\nCC0 1.0 Universal\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN\n ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS\n INFORMATION ON AN \"AS-IS\" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES\n REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS\n PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM\n THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED\n HEREUNDER.\n\nStatement of Purpose\n\nThe laws of most jurisdictions throughout the world automatically confer\nexclusive Copyright and Related Rights (defined below) upon the creator\nand subsequent owner(s) (each and all, an \"owner\") of an original work of\nauthorship and/or a database (each, a \"Work\").\n\nCertain owners wish to permanently relinquish those rights to a Work for\nthe purpose of contributing to a commons of creative, cultural and\nscientific works (\"Commons\") that the public can reliably and without fear\nof later claims of infringement build upon, modify, incorporate in other\nworks, reuse and redistribute as freely as possible in any form whatsoever\nand for any purposes, including without limitation commercial purposes.\nThese owners may contribute to the Commons to promote the ideal of a free\nculture and the further production of creative, cultural and scientific\nworks, or to gain reputation or greater distribution for their Work in\npart through the use and efforts of others.\n\nFor these and/or other purposes and motivations, and without any\nexpectation of additional consideration or compensation, the person\nassociating CC0 with a Work (the \"Affirmer\"), to the extent that he or she\nis an owner of Copyright and Related Rights in the Work, voluntarily\nelects to apply CC0 to the Work and publicly distribute the Work under its\nterms, with knowledge of his or her Copyright and Related Rights in the\nWork and the meaning and intended legal effect of CC0 on those rights.\n\n1. Copyright and Related Rights. A Work made available under CC0 may be\nprotected by copyright and related or neighboring rights (\"Copyright and\nRelated Rights\"). Copyright and Related Rights include, but are not\nlimited to, the following:\n\n i. the right to reproduce, adapt, distribute, perform, display,\n communicate, and translate a Work;\n ii. moral rights retained by the original author(s) and/or performer(s);\niii. publicity and privacy rights pertaining to a person's image or\n likeness depicted in a Work;\n iv. rights protecting against unfair competition in regards to a Work,\n subject to the limitations in paragraph 4(a), below;\n v. rights protecting the extraction, dissemination, use and reuse of data\n in a Work;\n vi. database rights (such as those arising under Directive 96/9/EC of the\n European Parliament and of the Council of 11 March 1996 on the legal\n protection of databases, and under any national implementation\n thereof, including any amended or successor version of such\n directive); and\nvii. other similar, equivalent or corresponding rights throughout the\n world based on applicable law or treaty, and any national\n implementations thereof.\n\n2. Waiver. To the greatest extent permitted by, but not in contravention\nof, applicable law, Affirmer hereby overtly, fully, permanently,\nirrevocably and unconditionally waives, abandons, and surrenders all of\nAffirmer's Copyright and Related Rights and associated claims and causes\nof action, whether now known or unknown (including existing as well as\nfuture claims and causes of action), in the Work (i) in all territories\nworldwide, (ii) for the maximum duration provided by applicable law or\ntreaty (including future time extensions), (iii) in any current or future\nmedium and for any number of copies, and (iv) for any purpose whatsoever,\nincluding without limitation commercial, advertising or promotional\npurposes (the \"Waiver\"). Affirmer makes the Waiver for the benefit of each\nmember of the public at large and to the detriment of Affirmer's heirs and\nsuccessors, fully intending that such Waiver shall not be subject to\nrevocation, rescission, cancellation, termination, or any other legal or\nequitable action to disrupt the quiet enjoyment of the Work by the public\nas contemplated by Affirmer's express Statement of Purpose.\n\n3. Public License Fallback. Should any part of the Waiver for any reason\nbe judged legally invalid or ineffective under applicable law, then the\nWaiver shall be preserved to the maximum extent permitted taking into\naccount Affirmer's express Statement of Purpose. In addition, to the\nextent the Waiver is so judged Affirmer hereby grants to each affected\nperson a royalty-free, non transferable, non sublicensable, non exclusive,\nirrevocable and unconditional license to exercise Affirmer's Copyright and\nRelated Rights in the Work (i) in all territories worldwide, (ii) for the\nmaximum duration provided by applicable law or treaty (including future\ntime extensions), (iii) in any current or future medium and for any number\nof copies, and (iv) for any purpose whatsoever, including without\nlimitation commercial, advertising or promotional purposes (the\n\"License\"). The License shall be deemed effective as of the date CC0 was\napplied by Affirmer to the Work. Should any part of the License for any\nreason be judged legally invalid or ineffective under applicable law, such\npartial invalidity or ineffectiveness shall not invalidate the remainder\nof the License, and in such case Affirmer hereby affirms that he or she\nwill not (i) exercise any of his or her remaining Copyright and Related\nRights in the Work or (ii) assert any associated claims and causes of\naction with respect to the Work, in either case contrary to Affirmer's\nexpress Statement of Purpose.\n\n4. Limitations and Disclaimers.\n\n a. No trademark or patent rights held by Affirmer are waived, abandoned,\n surrendered, licensed or otherwise affected by this document.\n b. Affirmer offers the Work as-is and makes no representations or\n warranties of any kind concerning the Work, express, implied,\n statutory or otherwise, including without limitation warranties of\n title, merchantability, fitness for a particular purpose, non\n infringement, or the absence of latent or other defects, accuracy, or\n the present or absence of errors, whether or not discoverable, all to\n the greatest extent permissible under applicable law.\n c. Affirmer disclaims responsibility for clearing rights of other persons\n that may apply to the Work or any use thereof, including without\n limitation any person's Copyright and Related Rights in the Work.\n Further, Affirmer disclaims responsibility for obtaining any necessary\n consents, permissions or other rights required for any use of the\n Work.\n d. Affirmer understands and acknowledges that Creative Commons is not a\n party to this document and has no duty or obligation with respect to\n this CC0 or use of the Work.\n", |
|
72 |
"sha256-f1d9dc9f2f4edaaebd9167eb8b79a6da6434313df47921aa4ea8ae278f7739e3": "/**\n * Haketilo demo script.\n *\n * Copyright (C) 2021 Wojtek Kosior\n * Available under the terms of Creative Commons Zero\n * <https://creativecommons.org/publicdomain/zero/1.0/legalcode>\n */\n\n{\n const banner = document.createElement(\"h2\");\n \n banner.textContent = \"Hoooray! Haketilo works :D\";\n \n banner.setAttribute(\"style\", `\\\n margin: 1em; \\\n border-radius: 1em 0px; \\\n background-color: #474; \\\n padding: 10px 20px; \\\n color: #eee; \\\n box-shadow: 0 6px 8px 0 rgba(0,0,0,0.24), 0 17px 50px 0 rgba(0,0,0,0.19); \\\n display: inline-block;\\\n `);\n \n document.body.prepend(banner);\n}\n" |
|
73 |
} |
|
72 | 74 |
} |
73 | 75 |
} |
html/install.js | ||
---|---|---|
49 | 49 |
#FROM html/DOM_helpers.js IMPORT clone_template, Showable |
50 | 50 |
#FROM common/entities.js IMPORT item_id_string, version_string, get_files, \ |
51 | 51 |
is_valid_version |
52 |
#FROM common/misc.js IMPORT sha256_async AS sha256 |
|
52 |
#FROM common/misc.js IMPORT sha256_async AS compute_sha256
|
|
53 | 53 |
|
54 | 54 |
const coll = new Intl.Collator(); |
55 | 55 |
|
... | ... | |
134 | 134 |
/* Make a link to view a file from the repository. */ |
135 | 135 |
const make_file_link = (preview_ctx, file_ref) => { |
136 | 136 |
const a = document.createElement("a"); |
137 |
a.href = `${this.repo_url}file/${file_ref.hash_key}`;
|
|
137 |
a.href = `${this.repo_url}file/sha256/${file_ref.sha256}`;
|
|
138 | 138 |
a.innerText = file_ref.file; |
139 | 139 |
|
140 | 140 |
return a; |
... | ... | |
213 | 213 |
|
214 | 214 |
const files = response.json.source_copyright |
215 | 215 |
.concat(item_type === "resource" ? response.json.scripts : []); |
216 |
for (const file of files) { |
|
217 |
file.hash_key = `sha256-${file.sha256}`; |
|
218 |
delete file.sha256; |
|
219 |
} |
|
220 | 216 |
|
221 | 217 |
if (item_type === "mapping") { |
222 | 218 |
for (const res_ref of Object.values(response.json.payloads)) |
... | ... | |
294 | 290 |
dialog.close(this.dialog_ctx); |
295 | 291 |
} |
296 | 292 |
|
297 |
const process_file = async (work, hash_key) => {
|
|
293 |
const process_file = async (work, sha256) => {
|
|
298 | 294 |
if (!work.is_ok) |
299 | 295 |
return; |
300 | 296 |
|
... | ... | |
302 | 298 |
|
303 | 299 |
try { |
304 | 300 |
var file_uses = await haketilodb.idb_get(work.file_uses_transaction, |
305 |
"file_uses", hash_key);
|
|
301 |
"file_uses", sha256);
|
|
306 | 302 |
if (!work.is_ok) |
307 | 303 |
return; |
308 | 304 |
} catch(e) { |
... | ... | |
311 | 307 |
} |
312 | 308 |
|
313 | 309 |
if (!file_uses) { |
314 |
const url = `${this.repo_url}file/${hash_key}`;
|
|
310 |
const url = `${this.repo_url}file/sha256/${sha256}`;
|
|
315 | 311 |
|
316 | 312 |
try { |
317 | 313 |
var response = await fetch(url); |
... | ... | |
336 | 332 |
return work.err(e, msg); |
337 | 333 |
} |
338 | 334 |
|
339 |
const digest = await sha256(text); |
|
335 |
const digest = await compute_sha256(text);
|
|
340 | 336 |
if (!work.is_ok) |
341 | 337 |
return; |
342 |
if (`sha256-${digest}` !== hash_key) {
|
|
338 |
if (digest !== sha256) {
|
|
343 | 339 |
const msg = `${url} served a file with different SHA256 cryptographic sum :(`; |
344 | 340 |
return work.err(null, msg); |
345 | 341 |
} |
346 | 342 |
|
347 |
work.result.push([hash_key, text]);
|
|
343 |
work.result.push([sha256, text]);
|
|
348 | 344 |
} |
349 | 345 |
|
350 | 346 |
if (--work.waiting === 0) |
... | ... | |
359 | 355 |
|
360 | 356 |
for (const item_def of item_defs) { |
361 | 357 |
for (const file of get_files(item_def)) { |
362 |
if (!processed_files.has(file.hash_key)) {
|
|
363 |
processed_files.add(file.hash_key);
|
|
364 |
process_file(work, file.hash_key);
|
|
358 |
if (!processed_files.has(file.sha256)) {
|
|
359 |
processed_files.add(file.sha256);
|
|
360 |
process_file(work, file.sha256);
|
|
365 | 361 |
} |
366 | 362 |
} |
367 | 363 |
} |
... | ... | |
379 | 375 |
|
380 | 376 |
try { |
381 | 377 |
var files = (await get_missing_files(item_defs)) |
382 |
.reduce((ac, [hk, txt]) => Object.assign(ac, {[hk]: txt}), {});
|
|
378 |
.reduce((ac, [h, txt]) => Object.assign(ac, {[h]: txt}), {});
|
|
383 | 379 |
} catch(e) { |
384 | 380 |
var dialog_prom = dialog.error(this.dialog_ctx, e); |
385 | 381 |
} |
386 | 382 |
|
387 | 383 |
if (files !== undefined) { |
388 |
const data = {files};
|
|
384 |
const data = {file: {sha256: files}};
|
|
389 | 385 |
const names = [["mappings", "mapping"], ["resources", "resource"]]; |
390 | 386 |
|
391 | 387 |
for (const [set_name, type] of names) { |
html/item_preview.js | ||
---|---|---|
63 | 63 |
*/ |
64 | 64 |
function make_file_link(preview_object, file_ref) { |
65 | 65 |
const a = document.createElement("a"); |
66 |
a.href = `${file_preview_link}#${file_ref.hash_key}`;
|
|
66 |
a.href = `${file_preview_link}#${file_ref.sha256}`;
|
|
67 | 67 |
a.innerText = file_ref.file; |
68 | 68 |
a.target = "_blank"; |
69 | 69 |
return a; |
html/payload_create.js | ||
---|---|---|
45 | 45 |
#IMPORT common/indexeddb.js AS haketilodb |
46 | 46 |
|
47 | 47 |
#FROM html/DOM_helpers.js IMPORT clone_template |
48 |
#FROM common/sha256.js IMPORT sha256 |
|
48 |
#FROM common/sha256.js IMPORT sha256 AS compute_sha256
|
|
49 | 49 |
#FROM common/patterns.js IMPORT validate_normalize_url_pattern, \ |
50 | 50 |
patterns_doc_url |
51 | 51 |
|
... | ... | |
94 | 94 |
const script = form_ctx.script.value; |
95 | 95 |
if (!script) |
96 | 96 |
throw "The 'script' field is required!"; |
97 |
const hash_key = `sha256-${sha256(script)}`;
|
|
97 |
const sha256 = compute_sha256(script);
|
|
98 | 98 |
|
99 | 99 |
const resource = { |
100 | 100 |
source_name: identifier, |
... | ... | |
106 | 106 |
version: [1], |
107 | 107 |
description, |
108 | 108 |
dependencies: [], |
109 |
scripts: [{file: "payload.js", hash_key}]
|
|
109 |
scripts: [{file: "payload.js", sha256}]
|
|
110 | 110 |
}; |
111 | 111 |
|
112 | 112 |
const mapping = { |
... | ... | |
121 | 121 |
payloads |
122 | 122 |
}; |
123 | 123 |
|
124 |
return {identifier, resource, mapping, files: {[hash_key]: script}};
|
|
124 |
return {identifier, resource, mapping, files_by_sha256: {[sha256]: script}};
|
|
125 | 125 |
} |
126 | 126 |
|
127 | 127 |
function clear_form(form_ctx) |
... | ... | |
137 | 137 |
{ |
138 | 138 |
const db = await haketilodb.get(); |
139 | 139 |
const tx_starter = haketilodb.start_items_transaction; |
140 |
const tx_ctx = await tx_starter(["resource", "mapping"], saving.files); |
|
140 |
const files = {sha256: saving.files_by_sha256}; |
|
141 |
const tx_ctx = await tx_starter(["resource", "mapping"], files); |
|
141 | 142 |
|
142 | 143 |
for (const type of ["resource", "mapping"]) { |
143 | 144 |
if (!saving[`override_${type}`] && |
test/unit/test_indexeddb.py | ||
---|---|---|
6 | 6 |
|
7 | 7 |
# This file is part of Haketilo |
8 | 8 |
# |
9 |
# Copyright (C) 2021,2022 Wojtek Kosior <koszko@koszko.org> |
|
9 |
# Copyright (C) 2021, 2022 Wojtek Kosior <koszko@koszko.org>
|
|
10 | 10 |
# |
11 | 11 |
# This program is free software: you can redistribute it and/or modify |
12 | 12 |
# it under the terms of the CC0 1.0 Universal License as published by |
... | ... | |
73 | 73 |
.then(finalize_transaction); |
74 | 74 |
returnval(promise); |
75 | 75 |
}''', |
76 |
sample_item, sample_files_by_hash)
|
|
76 |
sample_item, {'sha256': sample_files_by_sha256})
|
|
77 | 77 |
|
78 | 78 |
database_contents = get_db_contents(execute_in_page) |
79 | 79 |
|
80 | 80 |
assert len(database_contents['files']) == 4 |
81 |
assert all([sample_files_by_hash[file['hash_key']] == file['contents']
|
|
81 |
assert all([sample_files_by_sha256[file['sha256']] == file['contents']
|
|
82 | 82 |
for file in database_contents['files']]) |
83 | 83 |
assert all([len(file) == 2 for file in database_contents['files']]) |
84 | 84 |
|
85 | 85 |
assert len(database_contents['file_uses']) == 4 |
86 | 86 |
assert all([uses['uses'] == 1 for uses in database_contents['file_uses']]) |
87 |
assert set([uses['hash_key'] for uses in database_contents['file_uses']]) \
|
|
88 |
== set([file['hash_key'] for file in database_contents['files']])
|
|
87 |
assert set([uses['sha256'] for uses in database_contents['file_uses']]) \
|
|
88 |
== set([file['sha256'] for file in database_contents['files']])
|
|
89 | 89 |
|
90 | 90 |
assert database_contents['mapping'] == [] |
91 | 91 |
assert database_contents['resource'] == [sample_item] |
... | ... | |
93 | 93 |
# See if trying to add an item without providing all its files ends in an |
94 | 94 |
# exception and aborts the transaction as it should. |
95 | 95 |
sample_item['scripts'].append(sample_file_ref('combined.js')) |
96 |
incomplete_files = {**sample_files_by_hash}
|
|
97 |
incomplete_files.pop(sample_files['combined.js']['hash_key'])
|
|
96 |
incomplete_files = {**sample_files_by_sha256}
|
|
97 |
incomplete_files.pop(sample_files['combined.js']['sha256'])
|
|
98 | 98 |
exception = execute_in_page( |
99 | 99 |
'''{ |
100 | 100 |
const args = arguments; |
... | ... | |
112 | 112 |
} |
113 | 113 |
returnval(try_add_item()); |
114 | 114 |
}''', |
115 |
sample_item, incomplete_files)
|
|
115 |
sample_item, {'sha256': incomplete_files})
|
|
116 | 116 |
|
117 | 117 |
previous_database_contents = database_contents |
118 | 118 |
database_contents = get_db_contents(execute_in_page) |
119 | 119 |
|
120 | 120 |
assert 'file not present' in exception |
121 | 121 |
for key, val in database_contents.items(): |
122 |
keyfun = lambda item: item.get('hash_key') or item['identifier']
|
|
122 |
keyfun = lambda item: item.get('sha256') or item['identifier']
|
|
123 | 123 |
assert sorted(previous_database_contents[key], key=keyfun) \ |
124 | 124 |
== sorted(val, key=keyfun) |
125 | 125 |
|
... | ... | |
132 | 132 |
.then(finalize_transaction); |
133 | 133 |
returnval(promise); |
134 | 134 |
}''', |
135 |
sample_item, sample_files_by_hash)
|
|
135 |
sample_item, {'sha256': sample_files_by_sha256})
|
|
136 | 136 |
|
137 | 137 |
database_contents = get_db_contents(execute_in_page) |
138 | 138 |
|
... | ... | |
141 | 141 |
sample_files_list = [sample_files[name] for name in names] |
142 | 142 |
uses_list = [1, 2, 1, 1, 1] |
143 | 143 |
|
144 |
uses = dict([(uses['hash_key'], uses['uses'])
|
|
144 |
uses = dict([(uses['sha256'], uses['uses'])
|
|
145 | 145 |
for uses in database_contents['file_uses']]) |
146 |
assert uses == dict([(file['hash_key'], nr)
|
|
146 |
assert uses == dict([(file['sha256'], nr)
|
|
147 | 147 |
for file, nr in zip(sample_files_list, uses_list)]) |
148 | 148 |
|
149 |
files = dict([(file['hash_key'], file['contents'])
|
|
149 |
files = dict([(file['sha256'], file['contents'])
|
|
150 | 150 |
for file in database_contents['files']]) |
151 |
assert files == dict([(file['hash_key'], file['contents'])
|
|
151 |
assert files == dict([(file['sha256'], file['contents'])
|
|
152 | 152 |
for file in sample_files_list]) |
153 | 153 |
|
154 | 154 |
del database_contents['resource'][0]['source_copyright'][0]['extra_prop'] |
... | ... | |
174 | 174 |
sample_files_list = [sample_files[name] for name in names] |
175 | 175 |
uses_list = [1, 1] |
176 | 176 |
|
177 |
uses = dict([(uses['hash_key'], uses['uses'])
|
|
177 |
uses = dict([(uses['sha256'], uses['uses'])
|
|
178 | 178 |
for uses in results[0]['file_uses']]) |
179 |
assert uses == dict([(file['hash_key'], 1) for file in sample_files_list])
|
|
179 |
assert uses == dict([(file['sha256'], 1) for file in sample_files_list])
|
|
180 | 180 |
|
181 |
files = dict([(file['hash_key'], file['contents'])
|
|
181 |
files = dict([(file['sha256'], file['contents'])
|
|
182 | 182 |
for file in results[0]['files']]) |
183 |
assert files == dict([(file['hash_key'], file['contents'])
|
|
183 |
assert files == dict([(file['sha256'], file['contents'])
|
|
184 | 184 |
for file in sample_files_list]) |
185 | 185 |
|
186 | 186 |
assert results[0]['resource'] == [] |
... | ... | |
206 | 206 |
'0.1.1': sample_mapping |
207 | 207 |
} |
208 | 208 |
}, |
209 |
'files': sample_files_by_hash |
|
209 |
'file': { |
|
210 |
'sha256': sample_files_by_sha256 |
|
211 |
} |
|
210 | 212 |
} |
211 | 213 |
|
212 | 214 |
clear_indexeddb(execute_in_page) |
... | ... | |
350 | 352 |
'0.1.1': sample_mapping |
351 | 353 |
} |
352 | 354 |
}, |
353 |
'files': sample_files_by_hash |
|
355 |
'file': { |
|
356 |
'sha256': sample_files_by_sha256 |
|
357 |
} |
|
354 | 358 |
} |
355 | 359 |
driver.switch_to.window(windows[1]) |
356 | 360 |
execute_in_page('initial_data = arguments[0];', initial_data) |
... | ... | |
423 | 427 |
'0.1.1': sample_mapping2 |
424 | 428 |
} |
425 | 429 |
}, |
426 |
'files': sample_files_by_hash |
|
430 |
'file': { |
|
431 |
'sha256': sample_files_by_sha256 |
|
432 |
} |
|
427 | 433 |
} |
428 | 434 |
execute_in_page('returnval(save_items(arguments[0]));', sample_data) |
429 | 435 |
execute_in_page('returnval(set_setting("option22", "abc"));') |
test/unit/test_indexeddb_files_server.py | ||
---|---|---|
35 | 35 |
|
36 | 36 |
sample_files = dict(sample_files_list) |
37 | 37 |
|
38 |
sample_files, sample_files_by_hash = make_sample_files(sample_files)
|
|
38 |
sample_files, sample_files_by_sha256 = make_sample_files(sample_files)
|
|
39 | 39 |
|
40 | 40 |
def make_sample_resource_with_deps(n): |
41 | 41 |
resource = make_sample_resource(with_files=False) |
... | ... | |
53 | 53 |
sample_data = { |
54 | 54 |
'resources': sample_data_dict(resources), |
55 | 55 |
'mapping': {}, |
56 |
'files': sample_files_by_hash |
|
56 |
'file': { |
|
57 |
'sha256': sample_files_by_sha256 |
|
58 |
} |
|
57 | 59 |
} |
58 | 60 |
|
59 | 61 |
def prepare_test_page(initial_indexeddb_data, execute_in_page): |
test/unit/test_item_list.py | ||
---|---|---|
106 | 106 |
sample_data = { |
107 | 107 |
'resources': {}, |
108 | 108 |
'mappings': {}, |
109 |
'files': sample_files_by_hash |
|
109 |
'file': { |
|
110 |
'sha256': sample_files_by_sha256 |
|
111 |
} |
|
110 | 112 |
} |
111 | 113 |
|
112 | 114 |
indexes_added = set() |
... | ... | |
189 | 191 |
sample_data = { |
190 | 192 |
'resources': {}, |
191 | 193 |
'mappings': {}, |
192 |
'files': sample_files_by_hash |
|
194 |
'file': { |
|
195 |
'sha256': sample_files_by_sha256 |
|
196 |
} |
|
193 | 197 |
} |
194 | 198 |
sample_data[item_type + 's'] = sample_data_dict(items) |
195 | 199 |
|
test/unit/test_item_preview.py | ||
---|---|---|
160 | 160 |
|
161 | 161 |
# Cause the "link" to `bye.js` to be invalid. |
162 | 162 |
sample_resource = make_sample_resource() |
163 |
sample_resource['scripts'][1]['hash_key'] = 'dummy nonexistent key'
|
|
163 |
sample_resource['scripts'][1]['sha256'] = 'dummy nonexistent hash'
|
|
164 | 164 |
|
165 | 165 |
execute_in_page( |
166 | 166 |
''' |
test/unit/test_payload_create.py | ||
---|---|---|
19 | 19 |
|
20 | 20 |
import pytest |
21 | 21 |
import re |
22 |
from hashlib import sha256 |
|
23 |
|
|
22 | 24 |
from selenium.webdriver.support.ui import WebDriverWait |
23 | 25 |
|
24 | 26 |
from ..extension_crafting import ExtraHTML |
... | ... | |
137 | 139 |
'long_name': long_name, |
138 | 140 |
'scripts': [{ |
139 | 141 |
'file': 'payload.js', |
140 |
'hash_key': make_hash_key(form_data['script'])
|
|
142 |
'sha256': sha256(form_data['script'].encode()).digest().hex()
|
|
141 | 143 |
}] |
142 | 144 |
}] |
143 | 145 |
|
test/unit/utils.py | ||
---|---|---|
33 | 33 |
patterns_doc_url = \ |
34 | 34 |
'https://hydrillabugs.koszko.org/projects/haketilo/wiki/URL_patterns' |
35 | 35 |
|
36 |
def make_hash_key(file_contents): |
|
37 |
return f'sha256-{sha256(file_contents.encode()).digest().hex()}' |
|
38 |
|
|
39 | 36 |
def sample_file(contents): |
40 | 37 |
return { |
41 |
'hash_key': make_hash_key(contents),
|
|
38 |
'sha256': sha256(contents.encode()).digest().hex(),
|
|
42 | 39 |
'contents': contents |
43 | 40 |
} |
44 | 41 |
|
... | ... | |
51 | 48 |
sample_files = dict([(name, sample_file(contents)) |
52 | 49 |
for name, contents in names_contents.items()]) |
53 | 50 |
|
54 |
sample_files_by_hash = dict([[file['hash_key'], file['contents']]
|
|
55 |
for file in sample_files.values()]) |
|
51 |
sample_files_by_sha256 = dict([[file['sha256'], file['contents']]
|
|
52 |
for file in sample_files.values()])
|
|
56 | 53 |
|
57 |
return sample_files, sample_files_by_hash
|
|
54 |
return sample_files, sample_files_by_sha256
|
|
58 | 55 |
|
59 |
sample_files, sample_files_by_hash = make_sample_files({
|
|
56 |
sample_files, sample_files_by_sha256 = make_sample_files({
|
|
60 | 57 |
'report.spdx': '<!-- dummy report -->', |
61 | 58 |
'LICENSES/somelicense.txt': 'Permission is granted...', |
62 | 59 |
'LICENSES/CC0-1.0.txt': 'Dummy Commons...', |
... | ... | |
73 | 70 |
""" |
74 | 71 |
return { |
75 | 72 |
'file': file_name, |
76 |
'hash_key': sample_files_dict[file_name]['hash_key']
|
|
73 |
'sha256': sample_files_dict[file_name]['sha256']
|
|
77 | 74 |
} |
78 | 75 |
|
79 | 76 |
def make_sample_mapping(with_files=True): |
... | ... | |
154 | 151 |
return { |
155 | 152 |
'resources': sample_data_dict([make_sample_resource()]), |
156 | 153 |
'mappings': sample_data_dict([make_sample_mapping()]), |
157 |
'files': sample_files_by_hash |
|
154 |
'file': { |
|
155 |
'sha256': sample_files_by_sha256 |
|
156 |
} |
|
158 | 157 |
} |
159 | 158 |
|
160 | 159 |
def clear_indexeddb(execute_in_page): |
test/world_wide_library.py | ||
---|---|---|
114 | 114 |
for i in range(9)] |
115 | 115 |
sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents] |
116 | 116 |
|
117 |
file_url = lambda hashed: f'https://hydril.la/file/sha256-{hashed}'
|
|
117 |
file_url = lambda hashed: f'https://hydril.la/file/sha256/{hashed}'
|
|
118 | 118 |
|
119 | 119 |
sample_files_catalog = dict([(file_url(h), make_handler(c)) |
120 | 120 |
for h, c in zip(sample_hashes, sample_contents)]) |
Also available in: Unified diff
adapt to changes in file path format
From now on we assume Hydrilla serves file contents at 'file/sha256/' instead of 'file/sha256-'.
With this commit we also stop using the "hash_key" property internally.