/** * Fetching pinned packages into the package cache * (docs/design/package-manager.md, "Fetching and verification"). * * For each package, `fetchPackage`: * * 1. resolves its DID to the document that names its PDS and signing key; * 2. fetches the release record with `com.atproto.sync.getRecord`, whose * proof is checked against that key, and checks that the record's CID * is the one the lockfile pins; * 3. checks the release: its version, that its dependencies are the ones * the lockfile maps the package's names to, and that its source * artifact is the pinned one; * 4. downloads the source archive, in one blob or in parts, and checks it * against its CID; * 5. unpacks it into the cache. * * Checking the release against the checksum log comes later, with the * log. */ import {ByteBuffer} from 'zena:core'; import {Cid} from 'atproto/cid'; import {BlobRef, Value} from 'atproto/data'; import {DidDocument} from 'atproto/did'; import {IdentityResolver} from 'atproto/identity'; import {getVerifiedBlob, getVerifiedRecord} from 'atproto/sync'; import {SourceFile, unpackSource} from './archive.zena'; import {checkSource} from './validate.zena'; import {PackageCache} from './cache.zena'; import {Artifact, Dependency, RELEASE_TYPE, Release, SOURCE} from './records.zena'; import {Pin} from './resolution.zena'; import {Version} from 'zena-lang:semver'; /** A pinned package that cannot be fetched as pinned. */ export class FetchError extends Error { new(message: String) : super(message); } /** * Fetches `pin` into `cache` unless it is already there. Returns whether * it fetched it. Throws `FetchError` when what the package's PDS serves * differs from what the lockfile pins, and the error of whatever * else failed, such as resolving the DID or a request. */ export async function fetchPackage(pin: Pin, cache: PackageCache, resolver: IdentityResolver): Future { if (await cache.hasSource(pin.source)) { return false; } let document = await resolver.resolveDid(pin.did); let versionText = pin.version.toString(); let proof = await getVerifiedRecord(document, RELEASE_TYPE, versionText); let record = proof.record; let recordCid = proof.cid; if (record == null || recordCid == null) { throw new FetchError(`${pin.key}: the package has no release ${versionText}`); } if (!recordCid.equals(pin.release)) { throw new FetchError( `${pin.key}: release ${versionText} is ${recordCid.toString()}, but zena-lock.json pins ${pin.release.toString()}. The release changed after it was pinned.`); } let release = readRelease(record, versionText, pin); checkDependencies(pin, release); let source = release.artifact(SOURCE); if (source == null) { throw new FetchError(`${pin.key}: release ${versionText} has no source artifact`); } if (!source.cid.equals(pin.source)) { throw new FetchError( `${pin.key}: release ${versionText} has source ${source.cid.toString()}, but zena-lock.json pins ${pin.source.toString()}`); } let bytes = await download(document, source, pin); // The package's PDS may be any PDS, where nothing checked what was // written, so the release is checked against its archive here too. let problems = checkSource(release, pin.did, bytes); if (problems.length > 0) { var message = `${pin.key}: release ${versionText} does not agree with its source archive:`; for (let problem in problems) { message = `${message}\n ${problem}`; } throw new FetchError(message); } await cache.storeSource(pin.source, unpack(bytes, pin)); return true; } function unpack(bytes: FixedArray, pin: Pin): FixedArray { try { return unpackSource(bytes); } catch (e) { throw new FetchError(`${pin.key}: its source archive cannot be unpacked: ${e.message}`); } } /** What `fetchAll` did with one pin. */ export final class FetchResult { pin: Pin; /** Whether the package was fetched, rather than found in the cache. */ fetched: boolean; /** Why the package could not be fetched, or null when it was. */ error: String | null; new(pin: Pin, fetched: boolean, error: String | null) : pin = pin, fetched = fetched, error = error; } /** * Fetches each of `pins` into `cache`, one after another, and returns what * happened to each. A pin that cannot be fetched does not stop the rest. */ export async function fetchAll(pins: GrowableArray, cache: PackageCache, resolver: IdentityResolver): Future> { let results = new GrowableArray(); for (let pin in pins) { try { let fetched = await fetchPackage(pin, cache, resolver); results.push(new FetchResult(pin, fetched, null)); } catch (e) { results.push(new FetchResult(pin, false, e.message)); } } return results; } function readRelease(record: Value, versionText: String, pin: Pin): Release { try { return Release.fromValue(record, versionText); } catch (e) { throw new FetchError(`${pin.key}: ${e.message}`); } } /** * The lockfile maps each of the package's own names to a pinned * package, and the release says which package each name is and which * versions it accepts: they must agree, name for name. */ function checkDependencies(pin: Pin, release: Release): void { for (let dependency in release.dependencies) { let wanted = `${dependency.did} ${dependency.range.text}`; if (let (true, pinned) = pin.dependencies.get(dependency.name)) { if (!pinMatches(pinned, dependency)) { throw new FetchError( `${pin.key}: its dependency '${dependency.name}' is ${wanted}, but zena-lock.json maps it to ${pinned}`); } } else { throw new FetchError( `${pin.key}: its dependency '${dependency.name}' (${wanted}) is not in zena-lock.json`); } } if (pin.dependencies.size != release.dependencies.length) { throw new FetchError( `${pin.key}: zena-lock.json gives it ${pin.dependencies.size} dependencies, but release ${pin.version.toString()} has ${release.dependencies.length}`); } } /** Whether the pin `key`, `did@version`, is of `dependency`'s package and in its range. */ function pinMatches(key: String, dependency: Dependency): boolean { var at = -1; for (var i = 0; i < key.length; i += 1) { if (key.getByteAt(i) == 64) { // '@' at = i; } } if (at < 0 || key.sliceBytes(0, at) != dependency.did) { return false; } try { return dependency.range.includes(Version.parse(key.sliceBytes(at + 1, key.length))); } catch (e) { return false; } } /** The artifact's bytes, from its one blob or its parts, checked against its CID. */ async function download(document: DidDocument, artifact: Artifact, pin: Pin): Future> { let blob = artifact.blob; if (blob != null) { return await getVerifiedBlob(document, blob.cid); } let whole = new ByteBuffer(); for (let part in artifact.parts as FixedArray) { whole.writeBytes((await getVerifiedBlob(document, part.cid)) as ByteArray); } let bytes = whole.toByteArray() as FixedArray; if (!Cid.forBlob(bytes).equals(artifact.cid)) { throw new FetchError(`${pin.key}: the parts of ${artifact.cid.toString()} do not make it up`); } return bytes; }