diff --git a/README.md b/README.md
index f1d21d3..17c0ff2 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ ipfs version v0.23.0
## Usage
-This module downloads Kubo (go-ipfs) binaries from https://dist.ipfs.tech into your project.
+This module downloads Kubo (go-ipfs) binaries from [GitHub releases](https://github.com/ipfs/kubo/releases) into your project.
It will download the kubo version that matches the npm version of this module. So depending on `kubo@0.23.0` will install `kubo v0.23.0` for your current system architecture, in to your project at `node_modules/kubo/kubo/ipfs` and additional symlink to it at `node_modules/kubo/bin/ipfs`.
@@ -60,11 +60,56 @@ import { path } from 'kubo'
console.info('kubo is installed at', path())
```
-An error will be thrown if the path to the binary cannot be resolved.
+If the binary has not been downloaded yet, `path()` fetches it on the first
+call and returns once it is ready, so it works whether or not the install
+script ran (see [Installing without lifecycle scripts](#installing-without-lifecycle-scripts)).
+Pass `path({ autoDownload: false })` to resolve only an existing binary and
+throw when it is missing.
### Caching
-Downloaded archives are placed in OS-specific cache directory which can be customized by setting `NPM_KUBO_CACHE` in env.
+Downloaded archives (the `.tar.gz` and its `.sha512`) are cached in an
+OS-specific directory, keyed by version, OS, and architecture, so the binary is
+fetched once and reused across every project on the machine:
+
+- Linux: `~/.cache/npm-kubo` (or `$XDG_CACHE_HOME/npm-kubo`)
+- macOS: `~/Library/Caches/npm-kubo`
+- Windows: `%LOCALAPPDATA%\npm-kubo\Cache`
+
+Set `NPM_KUBO_CACHE` to an absolute path to override it. On a cache hit nothing
+is fetched over the network: the download URL is derived from the version, OS,
+and architecture.
+
+#### Caching in GitHub Actions
+
+`actions/setup-node` with `cache: npm` only persists npm's own cache (`~/.npm`),
+not the directory above, so cache it explicitly. Point `NPM_KUBO_CACHE` at an
+absolute, stable path and restore it with `actions/cache` before installing,
+keyed on the kubo version:
+
+```yaml
+env:
+ NPM_KUBO_CACHE: ${{ github.workspace }}/.kubo-cache
+steps:
+ - uses: actions/checkout@v4
+ # read the pinned kubo version so the cache key follows it, no manual bump
+ - id: kubo
+ run: echo "version=$(jq -r '.packages["node_modules/kubo"].version' package-lock.json)" >> "$GITHUB_OUTPUT"
+ - uses: actions/cache@v4
+ with:
+ path: ${{ github.workspace }}/.kubo-cache
+ key: ${{ runner.os }}-${{ runner.arch }}-kubo-${{ steps.kubo.outputs.version }}
+ restore-keys: ${{ runner.os }}-${{ runner.arch }}-kubo-
+ - uses: actions/setup-node@v4
+ with: { node-version: 24, cache: npm }
+ - run: npm ci
+```
+
+From npm v12 on, install scripts are opt-in, so the `postinstall` download is
+skipped and the binary is fetched on first use instead (see [Installing without
+lifecycle scripts](#installing-without-lifecycle-scripts)). `actions/cache` still
+saves the directory at the end of the job, so the first run that uses the binary
+warms the cache for later runs.
### Overriding with `KUBO_BINARY` env
@@ -72,9 +117,39 @@ If the `KUBO_BINARY` env variable is set at runtime this will override the path
This must point to the file, not the directory containing the file.
+### Overriding the download source
+
+Binaries are fetched from `https://github.com/ipfs/kubo/releases`. To use a
+mirror, set `KUBO_RELEASES_URL` (or the `kubo.releasesUrl` field in your
+`package.json`) to a base that serves the same files,
+`/download//`. This works with GitHub releases and any
+HTTP server that mirrors them.
+
+The old `KUBO_DIST_URL` and `GO_IPFS_DIST_URL` env vars and the `kubo.distUrl`
+config still work. They keep using the older dist.ipfs.tech layout and print a
+warning, so switch to `KUBO_RELEASES_URL` when you can.
+
+### Installing without lifecycle scripts
+
+Some setups skip lifecycle scripts on install: `npm install --ignore-scripts`,
+the [npm v12 default](https://www.aikido.dev/blog/npm-v12-block-postinstall),
+pnpm, and hardened CI. This package downloads the binary from its `postinstall`
+script, and when that script does not run the binary is downloaded on first use
+instead, either from `path()` or from the `ipfs` CLI shim. The download is
+cached, so it happens once.
+
+To control it:
+
+- set `KUBO_BINARY` to point at an existing binary and skip the download entirely
+- call `path({ autoDownload: false })` to handle a missing binary yourself
+
## Development
-**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. A pre-commit hook exists and should protect against this, but better safe than sorry.
+**Warning**: `bin/ipfs` is a small shim that gets replaced by a symlink to the
+downloaded binary after install, or the first time the binary is used. The
+symlink then shows up as a change in git. **Do not commit it**, or you would
+commit and publish a large binary. A pre-commit hook restores the shim, but
+better safe than sorry.
### Publish a new version
diff --git a/bin/ipfs b/bin/ipfs
index f73e2d3..a7fb243 100644
--- a/bin/ipfs
+++ b/bin/ipfs
@@ -1,3 +1,25 @@
#!/usr/bin/env node
+'use strict'
-console.log('IPFS is not installed.')
+// After a normal install this file is replaced by a symlink to the downloaded
+// kubo binary. When install scripts are skipped, the default from npm v12 on,
+// the symlink is never created, so this shim resolves the binary, downloading
+// it on first use, and runs it.
+const { spawnSync } = require('child_process')
+const { path: kuboPath } = require('..')
+
+let binary
+try {
+ binary = kuboPath()
+} catch (err) {
+ console.error(err.message)
+ process.exit(1)
+}
+
+const { status, error } = spawnSync(binary, process.argv.slice(2), { stdio: 'inherit' })
+
+if (error) {
+ throw error
+}
+
+process.exit(status ?? 1)
diff --git a/src/download.js b/src/download.js
index 363dea8..c8db058 100644
--- a/src/download.js
+++ b/src/download.js
@@ -28,6 +28,8 @@ const hasha = require('hasha')
const cproc = require('child_process')
const isWin = process.platform === 'win32'
+const DEFAULT_RELEASES_URL = 'https://github.com/ipfs/kubo/releases'
+
/**
* avoid expensive fetch if file is already in cache
* @param {string} url
@@ -48,14 +50,18 @@ async function cachingFetchAndVerify (url) {
}
if (!fs.existsSync(cachedFilePath)) {
console.info(`Downloading ${url} to ${cacheDir}`)
- // download file
- fs.writeFileSync(cachedFilePath, await got(url).buffer())
+ // Fetch the archive and its checksum first, then write them. Write the
+ // checksum, then put the archive in place with a temp file and a rename, so
+ // the archive (what the cache check looks for) only shows up once it is
+ // complete. A failed download or write then leaves nothing behind, instead
+ // of an archive with no checksum that breaks every later run.
+ const archive = await got(url).buffer()
+ const checksum = await got(`${url}.sha512`).buffer()
+ fs.writeFileSync(cachedHashPath, checksum)
+ const partFile = `${cachedFilePath}.part`
+ fs.writeFileSync(partFile, archive)
+ fs.renameSync(partFile, cachedFilePath)
console.info(`Downloaded ${url}`)
-
- // ..and checksum
- console.info(`Downloading ${filename}.sha512`)
- fs.writeFileSync(cachedHashPath, await got(`${url}.sha512`).buffer())
- console.info(`Downloaded ${filename}.sha512`)
} else {
console.info(`Found ${cachedFilePath}`)
}
@@ -116,25 +122,49 @@ function cleanArguments (version, platform, arch, installPath) {
cwd: process.env.INIT_CWD || process.cwd(),
defaults: {
version: 'v' + pkg.version.replace(/-[0-9]+/, ''),
- distUrl: 'https://dist.ipfs.tech'
+ releasesUrl: DEFAULT_RELEASES_URL,
+ distUrl: ''
}
})
+ // KUBO_DIST_URL / GO_IPFS_DIST_URL / kubo.distUrl are the old dist.ipfs.tech
+ // overrides. They still work via the legacy path in download(), so pass them
+ // through here instead of rejecting them. distUrl is empty when none is set.
return {
version: process.env.TARGET_VERSION || version || conf.version,
platform: process.env.TARGET_OS || platform || goenv.GOOS,
arch: process.env.TARGET_ARCH || arch || goenv.GOARCH,
distUrl: process.env.KUBO_DIST_URL || process.env.GO_IPFS_DIST_URL || conf.distUrl,
+ releasesUrl: process.env.KUBO_RELEASES_URL || conf.releasesUrl,
installPath: installPath ? path.resolve(installPath) : process.cwd()
}
}
/**
+ * Build the GitHub release asset URL. The download itself validates that the
+ * asset exists (a missing release asset returns 404), so no version index or
+ * manifest is fetched.
+ *
+ * @param {string} version
+ * @param {string} platform
+ * @param {string} arch
+ * @param {string} releasesUrl
+ */
+function getDownloadURL (version, platform, arch, releasesUrl) {
+ const base = releasesUrl.replace(/\/+$/, '')
+ const extension = platform === 'windows' ? 'zip' : 'tar.gz'
+ const asset = `kubo_${version}_${platform}-${arch}.${extension}`
+ return `${base}/download/${version}/${asset}`
+}
+
+/**
+ * Check that a version exists on a dist.ipfs.tech-style server. Used only by the
+ * deprecated KUBO_DIST_URL path.
+ *
* @param {string} version
* @param {string} distUrl
*/
async function ensureVersion (version, distUrl) {
- console.info(`${distUrl}/kubo/versions`)
const versions = (await got(`${distUrl}/kubo/versions`).text()).trim().split('\n')
if (versions.indexOf(version) === -1) {
@@ -143,12 +173,15 @@ async function ensureVersion (version, distUrl) {
}
/**
+ * Resolve the asset URL from a dist.ipfs.tech-style server (the dist.json
+ * layout). Used only by the deprecated KUBO_DIST_URL path.
+ *
* @param {string} version
* @param {string} platform
* @param {string} arch
* @param {string} distUrl
*/
-async function getDownloadURL (version, platform, arch, distUrl) {
+async function getLegacyDownloadURL (version, platform, arch, distUrl) {
await ensureVersion(version, distUrl)
const data = await got(`${distUrl}/kubo/${version}/dist.json`).json()
@@ -165,17 +198,72 @@ async function getDownloadURL (version, platform, arch, distUrl) {
return `${distUrl}/kubo/${version}${link}`
}
+let warnedDistUrlDeprecation = false
+
+// Tell the user once that the dist.ipfs.tech overrides are deprecated, and how
+// to switch to KUBO_RELEASES_URL.
+function warnDeprecatedDistUrl () {
+ if (warnedDistUrlDeprecation) {
+ return
+ }
+ warnedDistUrlDeprecation = true
+ console.error([
+ 'kubo: KUBO_DIST_URL, GO_IPFS_DIST_URL and the kubo.distUrl config are deprecated.',
+ 'They still work and keep using the old dist.ipfs.tech layout, but please switch',
+ 'to KUBO_RELEASES_URL. It works with GitHub releases and any HTTP mirror of them.',
+ `Example: KUBO_RELEASES_URL=${DEFAULT_RELEASES_URL}`
+ ].join('\n'))
+}
+
+/**
+ * Check whether a release exists for this version, so a missing asset can be
+ * told apart from a missing release. Best effort: returns false if the check
+ * itself fails (for example a mirror with no tag page).
+ *
+ * @param {string} version
+ * @param {string} releasesUrl
+ */
+async function releaseExists (version, releasesUrl) {
+ const base = releasesUrl.replace(/\/+$/, '')
+ try {
+ await got(`${base}/tag/${version}`, { method: 'HEAD' })
+ return true
+ } catch (err) {
+ return false
+ }
+}
+
/**
* @param {object} options
* @param {string} options.version
* @param {string} options.platform
* @param {string} options.arch
* @param {string} options.installPath
- * @param {string} options.distUrl
+ * @param {string} [options.releasesUrl]
+ * @param {string} [options.distUrl] deprecated dist.ipfs.tech-style base
*/
-async function download ({ version, platform, arch, installPath, distUrl }) {
- const url = await getDownloadURL(version, platform, arch, distUrl)
- const data = await cachingFetchAndVerify(url)
+async function download ({ version, platform, arch, installPath, releasesUrl = DEFAULT_RELEASES_URL, distUrl }) {
+ let url
+ if (distUrl) {
+ warnDeprecatedDistUrl()
+ url = await getLegacyDownloadURL(version, platform, arch, distUrl)
+ } else {
+ url = getDownloadURL(version, platform, arch, releasesUrl)
+ }
+
+ let data
+ try {
+ data = await cachingFetchAndVerify(url)
+ } catch (err) {
+ const e = /** @type {{ response?: { statusCode?: number } }} */ (err)
+ if (e.response && e.response.statusCode === 404 && !distUrl) {
+ if (await releaseExists(version, releasesUrl)) {
+ throw new Error(`kubo ${version} is released, but it has no ${platform}-${arch} file. Your platform may be unsupported, or the release may still be publishing. If this is a new version, wait a few hours and try again. (${url})`)
+ }
+ throw new Error(`kubo ${version} is not available: there is no ${version} release. (${url})`)
+ }
+ throw err
+ }
await unpack(url, installPath, data)
console.info(`Unpacked ${installPath}`)
@@ -206,6 +294,10 @@ async function link ({ depBin, version }) {
fs.unlinkSync(localBin)
}
+ // Make sure the bin directory exists. For git installs npm runs postinstall
+ // before it is created, so the symlink below would otherwise fail with ENOENT.
+ fs.mkdirSync(path.dirname(localBin), { recursive: true })
+
console.info('Linking', depBin, 'to', localBin)
try {
fs.symlinkSync(depBin, localBin)
@@ -255,14 +347,6 @@ async function link ({ depBin, version }) {
return localBin
}
-/**
-* @param {object} options
-* @param {string} options.version
-* @param {string} options.platform
-* @param {string} options.arch
-* @param {string} options.installPath
-* @param {string} options.distUrl
-*/
module.exports.download = download
/**
diff --git a/src/index.js b/src/index.js
index 3475090..1453679 100644
--- a/src/index.js
+++ b/src/index.js
@@ -2,25 +2,75 @@
const fs = require('fs')
const path = require('path')
+const { execFileSync } = require('child_process')
const { download } = require('./download')
module.exports.download = download
-module.exports.path = function () {
+const packageRoot = path.join(__dirname, '..')
+const binaryLocations = [
+ path.join(packageRoot, 'kubo', 'ipfs'),
+ path.join(packageRoot, 'kubo', 'ipfs.exe')
+]
+
+// Returns the installed binary, or undefined if it has not been downloaded.
+const findBinary = () => binaryLocations.find(file => fs.existsSync(file))
+
+const NOT_FOUND_ERROR = 'kubo binary not found, it may not be installed or an error may have occurred during installation'
+
+// Download and link the binary by running the installer in a child process.
+// The installer is async, so running it this way keeps path() synchronous while
+// reusing the same caching, checksum and symlink logic. From npm v12 on,
+// install scripts are opt-in, so the binary usually arrives here on first use
+// rather than from the postinstall script.
+const installBinary = () => {
+ execFileSync(process.execPath, [path.join(__dirname, 'post-install.js')], {
+ cwd: packageRoot,
+ // Send the installer's progress (download.js logs via console.info/log,
+ // which go to stdout) to this process's stderr (fd 2), so a programmatic
+ // path() keeps its own stdout clean for the value it returns.
+ stdio: ['ignore', 2, 2],
+ // Under Electron, process.execPath is Electron rather than node;
+ // ELECTRON_RUN_AS_NODE runs the script as plain node. A normal node process
+ // ignores it.
+ env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }
+ })
+}
+
+/**
+ * Resolve the path to the kubo binary, downloading it on first use when it has
+ * not been installed yet.
+ *
+ * Set the KUBO_BINARY env var to use your own binary, or pass
+ * { autoDownload: false } to resolve only an installed binary and throw when it
+ * is missing.
+ *
+ * @param {{ autoDownload?: boolean }} [options]
+ * @returns {string}
+ */
+module.exports.path = (options = {}) => {
+ const { autoDownload = true } = options
+
if (process.env.KUBO_BINARY) {
return process.env.KUBO_BINARY
}
- const paths = [
- path.resolve(path.join(__dirname, '..', 'kubo', 'ipfs')),
- path.resolve(path.join(__dirname, '..', 'kubo', 'ipfs.exe'))
- ]
+ let binary = findBinary()
- for (const bin of paths) {
- if (fs.existsSync(bin)) {
- return bin
+ if (!binary && autoDownload) {
+ try {
+ installBinary()
+ } catch (err) {
+ // The installer prints the underlying failure to stderr; surface the
+ // same error consumers saw before on-demand download existed.
+ throw new Error(NOT_FOUND_ERROR)
}
+ binary = findBinary()
+ }
+
+ if (!binary) {
+ throw new Error(NOT_FOUND_ERROR)
}
- throw new Error('kubo binary not found, it may not be installed or an error may have occurred during installation')
+ return binary
}
diff --git a/src/types.d.ts b/src/types.d.ts
index 98b4897..f65ca25 100644
--- a/src/types.d.ts
+++ b/src/types.d.ts
@@ -1 +1 @@
-export function path(): string
\ No newline at end of file
+export function path(options?: { autoDownload?: boolean }): string
diff --git a/test/download.js b/test/download.js
index 7920021..7d9750b 100644
--- a/test/download.js
+++ b/test/download.js
@@ -23,39 +23,70 @@ test('Ensure ipfs gets downloaded (current version and platform)', async (t) =>
test('Returns an error when version unsupported', async (t) => {
await clean()
- await t.rejects(downloadAndUpdateBin('bogusversion', 'linux'), /Error: Version 'bogusversion' not available/)
+ await t.rejects(downloadAndUpdateBin('bogusversion', 'linux'), /kubo bogusversion is not available/)
t.end()
})
-test('Returns an error when KUBO_DIST_URL is 404', async (t) => {
+test('KUBO_DIST_URL still works via the legacy layout and warns', async (t) => {
await clean()
- process.env.KUBO_DIST_URL = 'https://dist.ipfs.tech/notfound'
+ process.env.KUBO_DIST_URL = 'https://github.com/ipfs/kubo/notfound'
+ const warnings = []
+ const realError = console.error
+ console.error = (...args) => warnings.push(args.join(' '))
- await t.rejects(downloadAndUpdateBin(), /404/)
+ try {
+ // The base 404s, so the legacy /kubo/versions lookup fails fast; we only
+ // need to confirm the legacy path ran and printed the deprecation warning.
+ await t.rejects(downloadAndUpdateBin('v0.0.0'))
+ } finally {
+ console.error = realError
+ delete process.env.KUBO_DIST_URL
+ }
- delete process.env.KUBO_DIST_URL
+ t.ok(warnings.some(w => /deprecated/.test(w)), 'prints a deprecation warning')
t.end()
})
-test('Returns an error when legacy GO_IPFS_DIST_URL is 404', async (t) => {
+test('A released version with no asset for this platform says so', async (t) => {
await clean()
- process.env.GO_IPFS_DIST_URL = 'https://dist.ipfs.tech/notfound'
+ // v0.42.0 exists but ships no plan9 build, so the asset 404s while the
+ // release tag is present.
+ await t.rejects(downloadAndUpdateBin('v0.42.0', 'plan9'), /no plan9-.* file/)
- await t.rejects(downloadAndUpdateBin(), /404/)
+ t.end()
+})
+
+test('Path throws when no binary is present and autoDownload is disabled', async (t) => {
+ await clean()
- delete process.env.GO_IPFS_DIST_URL
+ t.throws(() => detectLocation({ autoDownload: false }), /not found/, 'Path throws if binary is not installed')
t.end()
})
-test('Path returns undefined when no binary has been downloaded', async (t) => {
+test('Path downloads the binary on first call when it is missing', async (t) => {
await clean()
- t.throws(detectLocation, /not found/, 'Path throws if binary is not installed')
+ const binPath = detectLocation()
+ const stats = await fs.stat(binPath)
+
+ t.ok(stats, 'kubo binary was downloaded on first path() call')
+
+ t.end()
+})
+
+test('Path throws a clean error when an on-demand download fails', async (t) => {
+ await clean()
+
+ process.env.TARGET_VERSION = 'v0.0.0-notfound'
+
+ t.throws(() => detectLocation(), /not found/, 'Path throws the kubo-binary-not-found error when the download fails')
+
+ delete process.env.TARGET_VERSION
t.end()
})
@@ -71,7 +102,7 @@ test('Ensure calling download function manually with static values works', async
version: `v${version}`,
platform: 'darwin',
arch: 'arm64',
- distUrl: 'https://dist.ipfs.tech',
+ releasesUrl: 'https://github.com/ipfs/kubo/releases',
installPath: tempDir
})
console.log(kuboPath)