// Reading a workspace's packages for the Zena compiler // (docs/design/package-manifest.md): `loadWorkspace` reads the // `zena-package.json` manifests and the pins in `zena-lock.json` into // `PackageConfig`s, keyed by package. The standard library's manifest is // built in (`buildStdlibConfig`). import { parseJson, JsonArray, JsonValue, JsonObject, JsonParseError } from 'zena:json'; import { PackageConfig, ModuleExport } from './module-resolver.zena'; import { CompilerHost } from './library-loader.zena'; import { dirName, indexOfByte, normalizePath } from './module-resolver.zena'; import { HashMap } from 'zena:collections'; // ============================================================================ // Pins // ============================================================================ // The version of the lockfile's format this compiler reads, which the // lockfile records as `lockfileVersion`. It is the format's own number, // unrelated to any package's version. The package manager writes and // reads lockfiles too (`LOCKFILE_VERSION` in `package-manager/resolution`), // so a change to the format raises both, and // `packages/cli/tests/lockfile-version_test.zena` checks they agree. export let LOCKFILE_VERSION = 1; // Throws unless the lockfile `json`, from `file`, is of `LOCKFILE_VERSION`: // a newer one asks for a newer `zena`, and an older one, or none, for the // lockfile to be written again. function checkLockfileVersion(json: JsonObject, file: String) { let again = "delete it and run zena pkg update to write it again"; let value = json["lockfileVersion"]; if (value == null) { throw new Error(`${file} has no "lockfileVersion": ${again}`); } if (!(value is Box) || value.value != ((value.value as i32) as f64) || value.value < 1.0) { throw new Error(`the "lockfileVersion" of ${file} is not a whole number of at least 1`); } let version = value.value as i32; if (version > LOCKFILE_VERSION) { throw new Error(`${file} is lockfile version ${version}, newer than the ${LOCKFILE_VERSION} this zena reads: upgrade zena`); } if (version < LOCKFILE_VERSION) { throw new Error(`${file} is lockfile version ${version}, older than the ${LOCKFILE_VERSION} this zena reads: ${again}`); } } // Adds the packages the `resolved` section of `json`, from `file`, pins: // each keyed by DID and version and rooted at its unpacked source // archive in `packageCache`, with the names its own libraries import. // Returns their keys. function addPins( json: JsonObject, packageCache: String, result: HashMap, file: String ) { let manifest = json; let keys = new Array(); let sources = new Map(); if (manifest.has("resolved")) { let resolved = manifest["resolved"] as JsonObject; resolved.forEach((key, value) => { if (!isPackageKey(key)) { throw new Error( `${file}: "resolved" has '${key}', which is not a DID and version such as did:plc:example@1.4.2`); } if (!(value is JsonObject)) { throw new Error(`${file}: "resolved" entry ${key} is not an object`); } let entry = value; let source = stringMember(entry, "source", key, file); if (!isCidText(source)) { throw new Error(`${file}: ${key} has source '${source}', which is not a CID`); } if (let (true, other) = sources.get(source)) { throw new Error(`${file}: ${other} and ${key} have the same source archive, ${source}`); } sources[source] = key; let config = new PackageConfig(`${packageCache}/sources/${source}`, null); config.dependencies = nameMap(entry, key, file); result[key] = config; keys.push(key); }); } for (let key in keys) { let dependencies = result[key].dependencies as HashMap; dependencies.forEach((name, target) => { if (!isResolved(result, target)) { throw new Error( `${file}: ${key} depends on ${target}, as '${name}', but "resolved" has no ${target}`); } }); } return keys; } // The `dependencies` member of `object`: local names, each mapped to a // package key. Empty when there is none. function nameMap(object: JsonObject, owner: String, file: String): HashMap { let names = new Map(); if (!object.has("dependencies")) { return names; } let dependencies = object["dependencies"]; if (!(dependencies is JsonObject)) { throw new Error(`${file}: the dependencies of ${owner} are not an object`); } dependencies.forEach((name, target) => { if (!isLocalName(name)) { throw new Error(`${file}: ${owner} has a dependency named '${name}'`); } if (!(target is String)) { throw new Error(`${file}: ${owner}'s dependency '${name}' is not a string`); } names[name] = target; }); return names; } function isResolved(packages: HashMap, key: String): boolean { if (let (true, config) = packages.get(key)) { return config.dependencies != null; } return false; } function stringMember(object: JsonObject, member: String, owner: String, file: String): String { let value = object[member]; if (!(value is String)) { throw new Error(`${file}: ${owner} has no "${member}" string`); } return value; } // Whether `key` names a package by DID and version: `did:`, more text, // `@`, a digit, and a version's characters (digits, letters, `.`, `-` // and `+`). function isPackageKey(key: String): boolean { if (!key.startsWith("did:")) { return false; } var at = -1; for (var i = 0; i < key.length; i += 1) { if (key.getByteAt(i) == 64) { // '@' at = i; } } if (at <= 4 || at == key.length - 1) { return false; } let first = key.getByteAt(at + 1); if (first < 48 || first > 57) { return false; } for (var i = at + 1; i < key.length; i += 1) { let b = key.getByteAt(i); let allowed = (b >= 48 && b <= 57) || (b >= 65 && b <= 90) || (b >= 97 && b <= 122) || b == 46 || b == 45 || b == 43; // '.', '-', '+' if (!allowed) { return false; } } return true; } // The `workspace` section of the lockfile `json`, from `file`: each // workspace package's directory, relative to the root, mapped to its // names, each mapped to a package key. function workspacePins(json: JsonObject, file: String) { let pins = new Map>(); if (!json.has("workspace")) { return pins; } let workspace = json["workspace"]; if (!(workspace is JsonObject)) { throw new Error(`${file}: "workspace" is not an object`); } workspace.forEach((dir, names) => { if (!(names is JsonObject)) { throw new Error(`${file}: "workspace" entry ${dir} is not an object`); } let holder = new JsonObject(); holder["dependencies"] = names; pins[dir] = nameMap(holder, dir, file); }); return pins; } // `dir` relative to the workspace's root `root`, as the lockfile's // `workspace` section names packages: `.` for the root, and a directory // outside it as it is. package-manager/workspace's relativeDir is the // same rule. function relativeDir(root: String, dir: String) { if (dir == root) { return "."; } if (root == ".") { return if (dir.startsWith("./")) dir.sliceBytes(2, dir.length) else dir; } let prefix = `${root}/`; return if (dir.startsWith(prefix)) dir.sliceBytes(prefix.length, dir.length) else dir; } // Whether `text` looks like a base32 CID string: `b` and then lowercase // base32. It becomes a directory name in the package cache, so nothing // else, a `/` or `..` least of all, may pass. function isCidText(text: String): boolean { if (text.length < 2 || text.getByteAt(0) != 98) { // 'b' return false; } for (var i = 1; i < text.length; i += 1) { let b = text.getByteAt(i); if (!((b >= 97 && b <= 122) || (b >= 50 && b <= 55))) { return false; } } return true; } // The packages of the workspace in `root` (docs/design/package-manifest.md): // the root manifest's, each workspace package's, and those of the packages // they depend on by path, with published packages pinned in // `zena-lock.json`. Without a `zena-package.json` in `root`, there are none, // and a file sees only the standard library and its relative imports. // // Each package is keyed by its `name`, or by its directory when it has // none, and gets its own names (`PackageConfig.dependencies`): its // dependencies' local names, and its own. Files in no package see the // packages by their keys, and the dependencies of a root manifest that is // not a package. A relative path that climbs out of the working directory // is read against `projectRoot`, its absolute path, when given. Published // packages are added only with a `packageCache`. export async function loadWorkspace( host: CompilerHost, root: String, projectRoot: String | null = null, packageCache: String | null = null ): Future> { if (await fileExists(host, joinPath(root, "zena-package.json"))) { return await new WorkspaceLoader(host, projectRoot, packageCache).load(root); } return new Map(); } final class WorkspaceLoader { #host: CompilerHost; #projectRoot: String | null; #packageCache: String | null; #result = new Map(); // Package keys by directory, so a package two others depend on is // loaded once. #keys = new Map(); // Directories by package key, to report two packages with one name. #dirs = new Map(); // The workspace's root. var #root = "."; // The lockfile's `workspace` section: the pins each workspace package's // names stand for, by the package's directory relative to the root. var #workspacePins = new Map>(); new(host: CompilerHost, projectRoot: String | null, packageCache: String | null) : #host = host, #projectRoot = projectRoot, #packageCache = packageCache; async load(root: String): Future> { let cache = this.#packageCache; let lock = joinPath(root, "zena-lock.json"); if (cache != null && await fileExists(this.#host, lock)) { let pins = await this.#readJson(lock); checkLockfileVersion(pins, "zena-lock.json"); this.#workspacePins = workspacePins(pins, "zena-lock.json"); for (let key in addPins(pins, cache, this.#result, "zena-lock.json")) { // A published package's own manifest gives its exports, once it is // in the cache. let config = this.#result[key]; let manifest = joinPath(config.root, "zena-package.json"); if (await fileExists(this.#host, manifest)) { let json = await this.#readJson(manifest); config.exports = parseExports(json); config.source = parseSource(json, manifest); config.main = parseMain(json, config.exports, manifest); } } } let manifest = await this.#readJson(joinPath(root, "zena-package.json")); this.#root = normalizePath(root); if (manifest.has("name")) { await this.#loadPackage(root, manifest); } else { await this.#loadRootNames(root, manifest); } let workspace = manifest["workspace"]; if (workspace != null) { let packages = (workspace as JsonObject)["packages"]; if (packages != null) { let list = packages as JsonArray; for (var i = 0; i < list.length; i += 1) { await this.#loadWorkspaceEntry(root, list[i] as String); } } } return this.#result; } // A `workspace.packages` entry: a directory, or `dir/*` for each // directory in `dir` that holds a manifest. async #loadWorkspaceEntry(root: String, entry: String): Future { if (entry.endsWith("/*")) { let parent = joinPath(root, entry.sliceBytes(0, entry.length - 2)); let names = await this.#host.listDir(parent); if (names == null) { throw new Error(`zena-package.json: workspace.packages names ${parent}, which is not a directory`); } for (let name in names) { let dir = joinPath(parent, name); if (await fileExists(this.#host, joinPath(dir, "zena-package.json"))) { await this.#loadPackageAt(dir); } } return; } await this.#loadPackageAt(joinPath(root, entry)); } // The key of the package in `dir`, loading it first if need be. async #loadPackageAt(dir: String): Future { let path = normalizePath(dir); if (let (true, key) = this.#keys.get(path)) { return key; } let manifest = joinPath(path, "zena-package.json"); if (!(await fileExists(this.#host, manifest))) { throw new Error(`${path} has no zena-package.json`); } return await this.#loadPackage(path, await this.#readJson(manifest)); } // Loads the package in `dir`, whose manifest is `manifest`, and returns // its key: its `name`, which gives a scope as `:`, or its // directory when it has no name (docs/design/import-names.md, "Workspace // packages"). A package in a scope also names itself by its plain // `name`, unless one of its dependencies has that name. async #loadPackage(dir: String, manifest: JsonObject): Future { let path = normalizePath(dir); let nameValue = manifest["name"]; var name: String | null = null; var plainName: String | null = null; if (nameValue != null) { if (!(nameValue is String)) { throw new Error(`${path}/zena-package.json: the name is not a package name`); } let colon = indexOfByte(nameValue, 58); // ':' let plain = if (colon < 0) nameValue else nameValue.sliceBytes(colon + 1, nameValue.length); let nameScope = if (colon < 0) null else nameValue.sliceBytes(0, colon); if (!isPackageName(plain) || (nameScope != null && !isScopeName(nameScope))) { throw new Error(`${path}/zena-package.json: the name is not a package name`); } name = nameValue; plainName = plain; } let key = name ?? path; if (let (true, other) = this.#dirs.get(key)) { throw new Error(`${other} and ${path} are both packages named '${key}'`); } if (this.#result.has(key)) { throw new Error(`${path}/zena-package.json: '${key}' is also the root's name for another package`); } let config = new PackageConfig(path, parseExports(manifest)); config.source = parseSource(manifest, `${path}/zena-package.json`); config.main = parseMain(manifest, config.exports, `${path}/zena-package.json`); let names = new Map(); config.dependencies = names; this.#result[key] = config; this.#keys[path] = key; this.#dirs[key] = path; if (name != null) { names[name] = key; } if (plainName != null) { // Set before the dependencies, so that one with the name replaces it. names[plainName] = key; } let dependencies = manifest["dependencies"]; if (dependencies == null) { return key; } let entries = dependencies as JsonObject; for (let localName in entries.keys()) { let target = await this.#dependency(path, localName, entries[localName]); if (target != null) { names[localName] = target; } } return key; } // A root manifest that is not a package: its dependencies are names for // the files in no package, such as scripts and examples. async #loadRootNames(root: String, manifest: JsonObject): Future { let dependencies = manifest["dependencies"]; if (dependencies == null) { return; } let path = normalizePath(root); let entries = dependencies as JsonObject; for (let localName in entries.keys()) { let target = await this.#dependency(path, localName, entries[localName]); if (target != null && target != localName) { if (this.#result.has(localName)) { throw new Error(`${path}/zena-package.json: '${localName}' is also a package's name`); } let alias = new PackageConfig("", null); alias.aliasOf = target; this.#result[localName] = alias; } } } // The key of the package the dependency `localName` of the manifest in // `dir` names, loading or registering it first; null for a published // package when there is no package cache. async #dependency(dir: String, localName: String, value: JsonValue): Future { if (!isLocalName(localName)) { throw new Error(`${dir}/zena-package.json: '${localName}' is not a package name`); } if (!(value is JsonObject)) { throw new Error(`${dir}/zena-package.json: dependency '${localName}' is not an object`); } let entry = value; if (entry.has("path")) { let target = resolveManifestPath(entry["path"] as String, dir, this.#projectRoot); return await this.#loadPackageAt(target); } if (entry.has("wit")) { let witRoot = resolveManifestPath(entry["wit"] as String, dir, this.#projectRoot); if (let (true, existing) = this.#result.get(localName)) { if (existing.witRoot != witRoot) { throw new Error(`${dir}/zena-package.json: '${localName}' names a WIT package another package names differently`); } } else { let wit = new PackageConfig("", null); wit.witRoot = witRoot; this.#result[localName] = wit; } return localName; } if (this.#packageCache == null) { return null; } let did = entry["did"]; let version = entry["version"]; if (!(did is String) || !(version is String)) { throw new Error(`${dir}/zena-package.json: dependency '${localName}' has no path, WIT, or DID and version`); } // zena-lock.json's `workspace` section says which pin the name stands // for in this package. var pinned: String | null = null; if (let (true, names) = this.#workspacePins.get(relativeDir(this.#root, dir))) { if (let (true, key) = names.get(localName)) { pinned = key; } } let missing = `${dir}/zena-package.json: zena-lock.json pins no version of '${localName}' (${did}). Run 'zena pkg add' or 'zena pkg update'.`; let found = pinned; if (found == null) { throw new Error(missing); } if (!isResolved(this.#result, found)) { throw new Error(missing); } return found; } async #readJson(path: String): Future { let text = await this.#host.readFile(path); var json: JsonValue = null; try { json = parseJson(text); } catch (e) { // The parser knows where in the text it stopped, not which file the // text came from. if (e is JsonParseError) { let fault = e; throw new Error(`${path}:${fault.line}:${fault.column}: ${fault.message}`); } throw e; } if (!(json is JsonObject)) { throw new Error(`${path} is not a JSON object`); } return json; } } // `name` joined onto `dir`, leaving a path relative to the working // directory as short as it was. function joinPath(dir: String, name: String): String { if (dir == "" || dir == ".") { return name; } return `${dir}/${name}`; } // Whether the file `path` exists, asked of its directory's listing. A host // that cannot list directories, such as the browser language service's, is // asked for the file, and an empty or missing one counts as absent. async function fileExists(host: CompilerHost, path: String): Future { let names = await host.listDir(dirName(path)); if (names == null) { try { return (await host.readFile(path)) != ""; } catch (e) { return false; } } var base = path; for (var i = 0; i < path.length; i += 1) { if (path.getByteAt(i) == 47) { // '/' base = path.sliceBytes(i + 1, path.length); } } for (let name in names) { if (name == base) { return true; } } return false; } // Whether `name` can be a local name in imports: a plain name, or a // package's name in a scope, `scope:name` (docs/design/import-names.md). // `zena` is the standard library's, as a name and as a scope. function isLocalName(name: String): boolean { let colon = indexOfByte(name, 58); // ':' if (colon < 0) { return isPackageName(name); } return isScopeName(name.sliceBytes(0, colon)) && isNamePart(name.sliceBytes(colon + 1, name.length)); } // Whether `name` can be a package's own name without its scope: a // manifest's `name` is this, or `scope:` and this. function isPackageName(name: String): boolean { return isNamePart(name) && name != "zena"; } // Whether `name` can be a scope's: `zena` is the standard library. function isScopeName(name: String): boolean { return isNamePart(name) && name != "zena"; } // Whether `part` can be a package's name or a scope's: not empty, and // without the `/` and `:` that imports are made of. function isNamePart(part: String): boolean { return part != "" && !part.contains("/") && !part.contains(":"); } // The directory a manifest's libraries are in, relative to the manifest: // its `source`, or `src` when it has none (package-manifest.md, "source"). // `manifestPath` names the manifest in an error. export function parseSource(obj: JsonObject, manifestPath: String): String { let value = obj["source"]; if (value == null) { return "src"; } if (!(value is String)) { throw new Error(`${manifestPath}: source is not a string`); } let text = value; // `normalizePath` makes `./src/` `src`, and `` or `./` `.`. let source = normalizePath(text); if (text.startsWith("/") || source == ".." || source.startsWith("../")) { throw new Error(`${manifestPath}: source '${text}' is not a directory inside the package`); } return source; } // A manifest's `main`, the library an import of the package's name alone // imports: `main` when it has none. One it names must be among its // `exports` when it lists them. export function parseMain(obj: JsonObject, exports: HashMap | null, manifestPath: String): String { let value = obj["main"]; if (value == null) { return "main"; } if (!(value is String)) { throw new Error(`${manifestPath}: main is not a string`); } let main = value; if (main == "" || main.startsWith("/") || main.endsWith("/") || main.endsWith(".zena")) { throw new Error(`${manifestPath}: main '${main}' is not a library's name`); } if (exports != null && !exports.has(main)) { throw new Error(`${manifestPath}: main '${main}' is not one of its exports`); } return main; } // The `exports` of a manifest, or null when it has none: an open package. export function parseExports(obj: JsonObject): HashMap | null { if (!obj.has("exports")) { return null; } let exportsMap = new Map(); let exportsObj = obj["exports"] as JsonObject; exportsObj.forEach((moduleName, moduleValue) => { let moduleExport = parseModuleExport(moduleValue); exportsMap[moduleName] = moduleExport; }); return exportsMap; } // A directory a manifest names, relative to `manifestDir` (see // `loadWorkspace`). function resolveManifestPath(value: String, manifestDir: String, projectRoot: String | null): String { if (value.startsWith("/")) { return normalizePath(value); } let resolved = normalizePath(`${manifestDir}/${value}`); if (projectRoot != null && (resolved == ".." || resolved.startsWith("../"))) { return normalizePath(`${projectRoot}/${resolved}`); } return resolved; } // Parse a module export entry: {}, { "path": "..." }, or { "virtual": { ... } }, // any of which may carry "internal": true. function parseModuleExport(value: JsonValue): ModuleExport { let obj = value as JsonObject; let internalValue = obj["internal"]; let internal = internalValue is Box && internalValue.value; if (obj.has("virtual")) { let virtualObj = obj["virtual"] as JsonObject; let virtual = new Map(); virtualObj.forEach((target, entryPath) => { virtual[target] = entryPath as String; }); return new ModuleExport(virtual, null, internal); } if (obj.has("path")) { return new ModuleExport(null, obj["path"] as String, internal); } return new ModuleExport(null, null, internal); } /** * The user's packages plus the ones the stdlib carries: `wasi`, the * WIT-backed package whose interfaces the stdlib itself imports * (`zena:fetch` on the component target speaks `wasi:http`), rooted * at the WIT shipped under the stdlib. A user manifest that declares * `wasi` keeps its own — a test's fixture package, say. The `zena` * package is not here: it is built from the manifest, not a path. */ export function withBuiltinPackages( stdlibRoot: String, packages: Map ): HashMap { let all = new Map(); for (let name in packages.keys()) { all[name] = packages[name]; } if (!all.has("wasi")) { let wasi = new PackageConfig("", null); wasi.witRoot = `${stdlibRoot}/wit`; all["wasi"] = wasi; } return all; } // The stdlib's own manifest lives in ./stdlib-manifest.zena; re-exported // here so callers keep one import for both manifests. export { buildStdlibConfig } from './stdlib-manifest.zena';