// Generate THIRD_PARTY_NOTICES.md from the actual dependency graph: // Rust crates compiled into the WebAssembly engine (cargo metadata for the // wasm32 target), npm packages bundled into the extension, and model files. import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { DTYPE, LOCK, MODEL, REPO } from "./lib.mjs"; const VISION_LOCK = JSON.parse(fs.readFileSync(path.join(REPO, "tools/vision_model_lock.json"), "utf8")); const ORT = JSON.parse(fs.readFileSync(path.join(REPO, "node_modules/onnxruntime-web/package.json"), "utf8")); const meta = JSON.parse( execFileSync("cargo", ["metadata", "--format-version", "1", "--locked", "--filter-platform", "wasm32-unknown-unknown"], { cwd: REPO, maxBuffer: 1 << 28 }).toString() ); // The crates actually compiled into the WebAssembly engine: `cargo tree` for // the wasm32 target with the default (shipped) features, normal dependencies // only. (`cargo metadata`'s resolve graph over-includes optional backends such // as CUDA/ROCm that are never built for wasm.) Metadata supplies licence // fields and package locations. const tree = execFileSync( "cargo", ["tree", "--locked", "--target", "wasm32-unknown-unknown", "-e", "normal", "--prefix", "none", "--format", "{p}"], { cwd: REPO, maxBuffer: 1 << 26 } ).toString(); const compiled = new Set( tree.split("\n").map((l) => l.replace(/ \(.*$/, "").trim()).filter(Boolean) ); const crates = meta.packages .filter((p) => compiled.has(`${p.name} v${p.version}`) && p.name !== "clearscroll-engine") .sort((a, b) => a.name.localeCompare(b.name)); const missing = [...compiled].filter((n) => !n.startsWith("clearscroll-engine ") && !crates.some((p) => `${p.name} v${p.version}` === n)); if (missing.length) throw new Error(`cargo tree lists crates not found in cargo metadata: ${missing.join(", ")}`); const lock = LOCK.models[MODEL]; const lines = [ "# Third-party notices", "", "Generated by `node scripts/third-party.mjs`. ClearScroll itself is licensed under the ISC License (see LICENSE).", "", "## Model", "", `The extension ships the weights and vocabulary of **${lock.repo}** (revision \`${lock.revision}\`), licensed under **${lock.license}**,`, `converted to ClearScroll's weight format. Modifications: linear layers transposed, position embeddings truncated to 128${DTYPE === "f16" ? ", values rounded to IEEE-754 half precision (binary16)" : ""}.`, "The WordPiece vocabulary originates from Google's BERT (bert-base-uncased, Apache-2.0).", "The Apache License 2.0 text is included in `licenses/Apache-2.0.txt`.", "", "## Image model", "", `The extension ships the image and text encoders of **${VISION_LOCK.upstream.repo}** (revision \`${VISION_LOCK.upstream.revision}\`), licensed under **${VISION_LOCK.upstream.license}**,`, `as int8-quantised ONNX files from **${VISION_LOCK.repo}** (revision \`${VISION_LOCK.revision}\`), together with its SentencePiece tokenizer (\`tokenizer.json\`).`, "Modifications by that conversion: export to ONNX, dynamic int8 quantisation. ClearScroll does not modify the files (sha256 pinned in `tools/vision_model_lock.json`).", "The Apache License 2.0 text is included in `licenses/Apache-2.0.txt`.", "", `## Rust crates compiled into the WebAssembly engine (${crates.length})`, "", "| Crate | Version | License |", "|---|---|---|", ...crates.map((c) => `| ${c.name} | ${c.version} | ${c.license ?? (c.license_file ? `see ${c.license_file}` : "unknown")} |`), "", "## JavaScript and WebAssembly", "", `**ONNX Runtime Web ${ORT.version}** (\`onnxruntime-web\`, ${ORT.license}, © Microsoft Corporation) is bundled into \`offscreen/vision.worker.js\` and ships its WebAssembly runtime (\`offscreen/vision/ort-wasm-simd-threaded.wasm\`).`, "Its licence is `licenses/onnxruntime-LICENSE.txt`; the notices for third-party code compiled into the runtime are reproduced unchanged from the ONNX Runtime repository (tag v" + ORT.version + ") in `licenses/onnxruntime-ThirdPartyNotices.txt`.", "No other npm package is bundled: esbuild, TypeScript, Playwright, axe-core and @types/* are build/test tools only.", "The WebAssembly glue code of the text engine is generated by wasm-bindgen (MIT OR Apache-2.0).", "", ]; // ── Full licence texts ──────────────────────────────────────────────────── // Licence/notice files shipped inside each crate package, de-duplicated by // content. Crates whose package contains none are listed explicitly rather // than given a reconstructed text. const LICENSE_FILE = /^(licen[cs]e|copying|unlicense|notice|copyright|authors)([-._].*)?$/i; const texts = new Map(); // sha256 → { text, files: Set, crates: Set } const without = []; for (const c of crates) { const dir = path.dirname(c.manifest_path); const files = fs.readdirSync(dir).filter((f) => LICENSE_FILE.test(f) && fs.statSync(path.join(dir, f)).isFile()).sort(); if (!files.length) { without.push(c); continue; } for (const f of files) { const text = fs.readFileSync(path.join(dir, f), "utf8").replace(/\r\n/g, "\n").trim(); const key = createHash("sha256").update(text).digest("hex"); if (!texts.has(key)) texts.set(key, { text, files: new Set(), crates: new Set() }); texts.get(key).files.add(f); texts.get(key).crates.add(`${c.name} ${c.version}`); } } const out = [ "THIRD-PARTY LICENCE TEXTS — ClearScroll WebAssembly engine", "Generated by `node scripts/third-party.mjs` from the licence files contained in each", "Rust crate compiled into offscreen/pkg/clearscroll_engine_bg.wasm (wasm32 target).", "Identical texts are printed once, followed by the crates that ship them.", "", ]; const sorted = [...texts.values()].sort((a, b) => b.crates.size - a.crates.size || a.text.localeCompare(b.text)); for (const t of sorted) { out.push("=".repeat(78), `Used by (${[...t.files].join(", ")}): ${[...t.crates].sort().join(", ")}`, "=".repeat(78), "", t.text, ""); } if (without.length) { out.push("=".repeat(78), "Crates whose package contains no licence file (SPDX licence and source given):", "=".repeat(78), ""); for (const c of without) out.push(`${c.name} ${c.version}: ${c.license ?? "unknown"} — ${c.repository ?? "no repository field"}`); out.push(""); } fs.mkdirSync(path.join(REPO, "licenses"), { recursive: true }); fs.writeFileSync(path.join(REPO, "licenses/THIRD_PARTY_LICENSES.txt"), out.join("\n")); lines.push( "## Full licence texts", "", `The licence and notice files of the crates above are reproduced in \`licenses/THIRD_PARTY_LICENSES.txt\` (${sorted.length} distinct texts from ${crates.length - without.length} crates).`, without.length ? `${without.length} crate packages contain no licence file; they are listed at the end of that file with their SPDX licence and repository: ${without.map((c) => c.name).join(", ")}.` + (without.every((c) => /Apache-2\.0/.test(c.license ?? "")) ? " All of them offer Apache-2.0 (most as \"MIT OR Apache-2.0\"); ClearScroll redistributes them under Apache-2.0, whose full text is `licenses/Apache-2.0.txt`." : " Not all of them offer Apache-2.0 — review before release.") : "Every crate package contains a licence file.", "" ); lines.push("\n## Source availability\n\nSource Code Form for every listed Rust crate is available at `https://crates.io/crates/NAME/VERSION` using its exact name and version in the table above. This includes the unmodified MPL-2.0 component **colored 3.1.1**: https://crates.io/crates/colored/3.1.1 . ClearScroll makes no modifications to these crate sources.\n\nThe text model's Apache-2.0 declaration and origin are available in its exact upstream model card: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/blob/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/README.md . The modifications to the redistributed model are described above.\n".replace("https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/blob/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/README.md", `https://huggingface.co/${lock.repo}/blob/${lock.revision}/README.md`)); fs.writeFileSync(path.join(REPO, "THIRD_PARTY_NOTICES.md"), lines.join("\n")); console.log(`licenses/THIRD_PARTY_LICENSES.txt: ${sorted.length} distinct texts, ${without.length} crates without licence files, ${(fs.statSync(path.join(REPO, "licenses/THIRD_PARTY_LICENSES.txt")).size / 1024).toFixed(0)} KiB`); console.log(`THIRD_PARTY_NOTICES.md: ${crates.length} crates`); const unknown = crates.filter((c) => !c.license); if (unknown.length) console.warn("crates without SPDX license field:", unknown.map((c) => c.name).join(", "));