fix: fetch binary on demand from github releases

README.md

@@ -46,7 +46,7 @@ ipfs version v0.23.0
 
 ## Usage
 
-This module downloads Kubo (go-ipfs) binaries from https://dist.ipfs.tech into your project.
+This module downloads Kubo (go-ipfs) binaries from [GitHub releases](https://github.com/ipfs/kubo/releases) into your project.
 
 It will download the kubo version that matches the npm version of this module. So depending on `kubo@0.23.0` will install `kubo v0.23.0` for your current system architecture, in to your project at `node_modules/kubo/kubo/ipfs` and additional symlink to it at `node_modules/kubo/bin/ipfs`.
 
@@ -60,21 +60,96 @@ import { path } from 'kubo'
 console.info('kubo is installed at', path())
 ```
 
-An error will be thrown if the path to the binary cannot be resolved.
+If the binary has not been downloaded yet, `path()` fetches it on the first
+call and returns once it is ready, so it works whether or not the install
+script ran (see [Installing without lifecycle scripts](#installing-without-lifecycle-scripts)).
+Pass `path({ autoDownload: false })` to resolve only an existing binary and
+throw when it is missing.
 
 ### Caching
 
-Downloaded archives are placed in OS-specific cache directory which can be customized by setting `NPM_KUBO_CACHE` in env.
+Downloaded archives (the `.tar.gz` and its `.sha512`) are cached in an
+OS-specific directory, keyed by version, OS, and architecture, so the binary is
+fetched once and reused across every project on the machine:
+
+- Linux: `~/.cache/npm-kubo` (or `$XDG_CACHE_HOME/npm-kubo`)
+- macOS: `~/Library/Caches/npm-kubo`
+- Windows: `%LOCALAPPDATA%\npm-kubo\Cache`
+
+Set `NPM_KUBO_CACHE` to an absolute path to override it. On a cache hit nothing
+is fetched over the network: the download URL is derived from the version, OS,
+and architecture.
+
+#### Caching in GitHub Actions
+
+`actions/setup-node` with `cache: npm` only persists npm's own cache (`~/.npm`),
+not the directory above, so cache it explicitly. Point `NPM_KUBO_CACHE` at an
+absolute, stable path and restore it with `actions/cache` before installing,
+keyed on the kubo version:
+
+```yaml
+env:
+  NPM_KUBO_CACHE: ${{ github.workspace }}/.kubo-cache
+steps:
+  - uses: actions/checkout@v4
+  # read the pinned kubo version so the cache key follows it, no manual bump
+  - id: kubo
+    run: echo "version=$(jq -r '.packages["node_modules/kubo"].version' package-lock.json)" >> "$GITHUB_OUTPUT"
+  - uses: actions/cache@v4
+    with:
+      path: ${{ github.workspace }}/.kubo-cache
+      key: ${{ runner.os }}-${{ runner.arch }}-kubo-${{ steps.kubo.outputs.version }}
+      restore-keys: ${{ runner.os }}-${{ runner.arch }}-kubo-
+  - uses: actions/setup-node@v4
+    with: { node-version: 24, cache: npm }
+  - run: npm ci
+```
+
+From npm v12 on, install scripts are opt-in, so the `postinstall` download is
+skipped and the binary is fetched on first use instead (see [Installing without
+lifecycle scripts](#installing-without-lifecycle-scripts)). `actions/cache` still
+saves the directory at the end of the job, so the first run that uses the binary
+warms the cache for later runs.
 
 ### Overriding with `KUBO_BINARY` env
 
 If the `KUBO_BINARY` env variable is set at runtime this will override the path of the binary used.
 
 This must point to the file, not the directory containing the file.
 
+### Overriding the download source
+
+Binaries are fetched from `https://github.com/ipfs/kubo/releases`. To use a
+mirror, set `KUBO_RELEASES_URL` (or the `kubo.releasesUrl` field in your
+`package.json`) to a base that serves the same files,
+`<base>/download/<version>/<asset>`. This works with GitHub releases and any
+HTTP server that mirrors them.
+
+The old `KUBO_DIST_URL` and `GO_IPFS_DIST_URL` env vars and the `kubo.distUrl`
+config still work. They keep using the older dist.ipfs.tech layout and print a
+warning, so switch to `KUBO_RELEASES_URL` when you can.
+
+### Installing without lifecycle scripts
+
+Some setups skip lifecycle scripts on install: `npm install --ignore-scripts`,
+the [npm v12 default](https://www.aikido.dev/blog/npm-v12-block-postinstall),
+pnpm, and hardened CI. This package downloads the binary from its `postinstall`
+script, and when that script does not run the binary is downloaded on first use
+instead, either from `path()` or from the `ipfs` CLI shim. The download is
+cached, so it happens once.
+
+To control it:
+
+- set `KUBO_BINARY` to point at an existing binary and skip the download entirely
+- call `path({ autoDownload: false })` to handle a missing binary yourself
+
 ## Development
 
-**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. A pre-commit hook exists and should protect against this, but better safe than sorry.
+**Warning**: `bin/ipfs` is a small shim that gets replaced by a symlink to the
+downloaded binary after install, or the first time the binary is used. The
+symlink then shows up as a change in git. **Do not commit it**, or you would
+commit and publish a large binary. A pre-commit hook restores the shim, but
+better safe than sorry.
 
 ### Publish a new version
 

bin/ipfs

@@ -1,3 +1,25 @@
 #!/usr/bin/env node
+'use strict'
 
-console.log('IPFS is not installed.')
+// After a normal install this file is replaced by a symlink to the downloaded
+// kubo binary. When install scripts are skipped, the default from npm v12 on,
+// the symlink is never created, so this shim resolves the binary, downloading
+// it on first use, and runs it.
+const { spawnSync } = require('child_process')
+const { path: kuboPath } = require('..')
+
+let binary
+try {
+  binary = kuboPath()
+} catch (err) {
+  console.error(err.message)
+  process.exit(1)
+}
+
+const { status, error } = spawnSync(binary, process.argv.slice(2), { stdio: 'inherit' })
+
+if (error) {
+  throw error
+}
+
+process.exit(status ?? 1)

src/download.js

@@ -28,6 +28,8 @@ const hasha = require('hasha')
 const cproc = require('child_process')
 const isWin = process.platform === 'win32'
 
+const DEFAULT_RELEASES_URL = 'https://github.com/ipfs/kubo/releases'
+
 /**
  * avoid expensive fetch if file is already in cache
  * @param {string} url
@@ -48,14 +50,18 @@ async function cachingFetchAndVerify (url) {
   }
   if (!fs.existsSync(cachedFilePath)) {
     console.info(`Downloading ${url} to ${cacheDir}`)
-    // download file
-    fs.writeFileSync(cachedFilePath, await got(url).buffer())
+    // Fetch the archive and its checksum first, then write them. Write the
+    // checksum, then put the archive in place with a temp file and a rename, so
+    // the archive (what the cache check looks for) only shows up once it is
+    // complete. A failed download or write then leaves nothing behind, instead
+    // of an archive with no checksum that breaks every later run.
+    const archive = await got(url).buffer()
+    const checksum = await got(`${url}.sha512`).buffer()
+    fs.writeFileSync(cachedHashPath, checksum)
+    const partFile = `${cachedFilePath}.part`
+    fs.writeFileSync(partFile, archive)
+    fs.renameSync(partFile, cachedFilePath)
     console.info(`Downloaded ${url}`)
-
-    // ..and checksum
-    console.info(`Downloading ${filename}.sha512`)
-    fs.writeFileSync(cachedHashPath, await got(`${url}.sha512`).buffer())
-    console.info(`Downloaded ${filename}.sha512`)
   } else {
     console.info(`Found ${cachedFilePath}`)
   }
@@ -116,25 +122,49 @@ function cleanArguments (version, platform, arch, installPath) {
     cwd: process.env.INIT_CWD || process.cwd(),
     defaults: {
       version: 'v' + pkg.version.replace(/-[0-9]+/, ''),
-      distUrl: 'https://dist.ipfs.tech'
+      releasesUrl: DEFAULT_RELEASES_URL,
+      distUrl: ''
     }
   })
 
+  // KUBO_DIST_URL / GO_IPFS_DIST_URL / kubo.distUrl are the old dist.ipfs.tech
+  // overrides. They still work via the legacy path in download(), so pass them
+  // through here instead of rejecting them. distUrl is empty when none is set.
   return {
     version: process.env.TARGET_VERSION || version || conf.version,
     platform: process.env.TARGET_OS || platform || goenv.GOOS,
     arch: process.env.TARGET_ARCH || arch || goenv.GOARCH,
     distUrl: process.env.KUBO_DIST_URL || process.env.GO_IPFS_DIST_URL || conf.distUrl,
+    releasesUrl: process.env.KUBO_RELEASES_URL || conf.releasesUrl,
     installPath: installPath ? path.resolve(installPath) : process.cwd()
   }
 }
 
 /**
+ * Build the GitHub release asset URL. The download itself validates that the
+ * asset exists (a missing release asset returns 404), so no version index or
+ * manifest is fetched.
+ *
+ * @param {string} version
+ * @param {string} platform
+ * @param {string} arch
+ * @param {string} releasesUrl
+ */
+function getDownloadURL (version, platform, arch, releasesUrl) {
+  const base = releasesUrl.replace(/\/+$/, '')
+  const extension = platform === 'windows' ? 'zip' : 'tar.gz'
+  const asset = `kubo_${version}_${platform}-${arch}.${extension}`
+  return `${base}/download/${version}/${asset}`
+}
+
+/**
+ * Check that a version exists on a dist.ipfs.tech-style server. Used only by the
+ * deprecated KUBO_DIST_URL path.
+ *
  * @param {string} version
  * @param {string} distUrl
  */
 async function ensureVersion (version, distUrl) {
-  console.info(`${distUrl}/kubo/versions`)
   const versions = (await got(`${distUrl}/kubo/versions`).text()).trim().split('\n')
 
   if (versions.indexOf(version) === -1) {
@@ -143,12 +173,15 @@ async function ensureVersion (version, distUrl) {
 }
 
 /**
+ * Resolve the asset URL from a dist.ipfs.tech-style server (the dist.json
+ * layout). Used only by the deprecated KUBO_DIST_URL path.
+ *
  * @param {string} version
  * @param {string} platform
  * @param {string} arch
  * @param {string} distUrl
  */
-async function getDownloadURL (version, platform, arch, distUrl) {
+async function getLegacyDownloadURL (version, platform, arch, distUrl) {
   await ensureVersion(version, distUrl)
 
   const data = await got(`${distUrl}/kubo/${version}/dist.json`).json()
@@ -165,17 +198,72 @@ async function getDownloadURL (version, platform, arch, distUrl) {
   return `${distUrl}/kubo/${version}${link}`
 }
 
+let warnedDistUrlDeprecation = false
+
+// Tell the user once that the dist.ipfs.tech overrides are deprecated, and how
+// to switch to KUBO_RELEASES_URL.
+function warnDeprecatedDistUrl () {
+  if (warnedDistUrlDeprecation) {
+    return
+  }
+  warnedDistUrlDeprecation = true
+  console.error([
+    'kubo: KUBO_DIST_URL, GO_IPFS_DIST_URL and the kubo.distUrl config are deprecated.',
+    'They still work and keep using the old dist.ipfs.tech layout, but please switch',
+    'to KUBO_RELEASES_URL. It works with GitHub releases and any HTTP mirror of them.',
+    `Example: KUBO_RELEASES_URL=${DEFAULT_RELEASES_URL}`
+  ].join('\n'))
+}
+
+/**
+ * Check whether a release exists for this version, so a missing asset can be
+ * told apart from a missing release. Best effort: returns false if the check
+ * itself fails (for example a mirror with no tag page).
+ *
+ * @param {string} version
+ * @param {string} releasesUrl
+ */
+async function releaseExists (version, releasesUrl) {
+  const base = releasesUrl.replace(/\/+$/, '')
+  try {
+    await got(`${base}/tag/${version}`, { method: 'HEAD' })
+    return true
+  } catch (err) {
+    return false
+  }
+}
+
 /**
  * @param {object} options
  * @param {string} options.version
  * @param {string} options.platform
  * @param {string} options.arch
  * @param {string} options.installPath
- * @param {string} options.distUrl
+ * @param {string} [options.releasesUrl]
+ * @param {string} [options.distUrl] deprecated dist.ipfs.tech-style base
  */
-async function download ({ version, platform, arch, installPath, distUrl }) {
-  const url = await getDownloadURL(version, platform, arch, distUrl)
-  const data = await cachingFetchAndVerify(url)
+async function download ({ version, platform, arch, installPath, releasesUrl = DEFAULT_RELEASES_URL, distUrl }) {
+  let url
+  if (distUrl) {
+    warnDeprecatedDistUrl()
+    url = await getLegacyDownloadURL(version, platform, arch, distUrl)
+  } else {
+    url = getDownloadURL(version, platform, arch, releasesUrl)
+  }
+
+  let data
+  try {
+    data = await cachingFetchAndVerify(url)
+  } catch (err) {
+    const e = /** @type {{ response?: { statusCode?: number } }} */ (err)
+    if (e.response && e.response.statusCode === 404 && !distUrl) {
+      if (await releaseExists(version, releasesUrl)) {
+        throw new Error(`kubo ${version} is released, but it has no ${platform}-${arch} file. Your platform may be unsupported, or the release may still be publishing. If this is a new version, wait a few hours and try again. (${url})`)
+      }
+      throw new Error(`kubo ${version} is not available: there is no ${version} release. (${url})`)
+    }
+    throw err
+  }
 
   await unpack(url, installPath, data)
   console.info(`Unpacked ${installPath}`)
@@ -206,6 +294,10 @@ async function link ({ depBin, version }) {
     fs.unlinkSync(localBin)
   }
 
+  // Make sure the bin directory exists. For git installs npm runs postinstall
+  // before it is created, so the symlink below would otherwise fail with ENOENT.
+  fs.mkdirSync(path.dirname(localBin), { recursive: true })
+
   console.info('Linking', depBin, 'to', localBin)
   try {
     fs.symlinkSync(depBin, localBin)
@@ -255,14 +347,6 @@ async function link ({ depBin, version }) {
   return localBin
 }
 
-/**
-* @param {object} options
-* @param {string} options.version
-* @param {string} options.platform
-* @param {string} options.arch
-* @param {string} options.installPath
-* @param {string} options.distUrl
-*/
 module.exports.download = download
 
 /**