feat(tests): mostly bypass npm when installing deps to fixtures

This is still slower than it should be but let's see what happens.

Author: sorccu

package-lock.json

@@ -13,7 +13,8 @@
     "test:e2e": "npm run test:e2e --workspaces"
   },
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "utilities/*"
   ],
   "devDependencies": {
     "@commitlint/cli": "^17.8.1",
@@ -34,4 +35,4 @@
   "lint-staged": {
     "*.{ts,js,mjs}": "npm run lint"
   }
-}
+}

package.json

@@ -6,7 +6,7 @@ import Debug from 'debug'
 
 const debug = Debug('checkly:cli:testing:fixture-sandbox')
 
-import { detectNearestPackageJson, detectPackageManager, PackageManager } from '../services/check-parser/package-files/package-manager'
+import { detectNearestPackageJson } from '../services/check-parser/package-files/package-manager'
 
 export interface CreateFixtureSandboxOptions {
   /**
@@ -24,13 +24,6 @@ export interface CreateFixtureSandboxOptions {
    */
   root?: string
 
-  /**
-  * The package manager used to manage the fixture.
-  *
-  * If not provided, the package manager is detected automatically.
-  */
-  packageManager?: PackageManager
-
   /**
    * Whether to install packages using the package manager.
    *
@@ -48,34 +41,34 @@ export interface CreateFixtureSandboxOptions {
 }
 
 interface FixtureSandboxOptions {
-  packageManager: PackageManager
   root: string
 }
 
 export class FixtureSandbox {
   #root: string
-  #packageManager: PackageManager
 
-  private constructor ({ root, packageManager }: FixtureSandboxOptions) {
+  static installer: any
+
+  private constructor ({ root }: FixtureSandboxOptions) {
     this.#root = root
-    this.#packageManager = packageManager
   }
 
   get root (): string {
     return this.#root
   }
 
-  get packageManager (): PackageManager {
-    return this.#packageManager
-  }
-
   static async create (options: CreateFixtureSandboxOptions): Promise<FixtureSandbox> {
-    const { execa } = await import('execa')
+    // @ts-expect-error Not a TypeScript module.
+    const { createInstaller, fileHashSha256 } = await import('dependency-layer-installer')
+
+    const installer = FixtureSandbox.installer ?? await createInstaller({
+      archiveDir: path.join(tmpdir(), 'fixture-sandbox-installer-archive'),
+      extractDir: path.join(tmpdir(), 'fixture-sandbox-installer-extracted'),
+    })
 
     const {
       source,
       root: maybeRoot,
-      packageManager: maybePackageManager,
       installPackages = true,
       injectPackedSelf = true,
     } = options
@@ -92,41 +85,7 @@ export class FixtureSandbox {
       recursive: true,
     })
 
-    const packageManager = maybePackageManager
-      ?? await detectPackageManager(root)
-
-    debug(`Detected package manager ${packageManager.name}`)
-
     if (installPackages) {
-      const { executable, args, unsafeDisplayCommand } = packageManager.installCommand()
-
-      debug(`Installing packages via ${unsafeDisplayCommand}`)
-
-      await execa(executable, args, {
-        cwd: root,
-      })
-    }
-
-    if (installPackages && injectPackedSelf) {
-      debug('Injecting containing package')
-
-      const lockfile = packageManager.representativeLockfile
-
-      // Take a backup of the original package.json so that we can restore
-      // it later.
-      await fs.cp(
-        path.join(root, 'package.json'),
-        path.join(root, 'package.json.backup'),
-      )
-
-      // Same for the lockfile.
-      if (lockfile) {
-        await fs.cp(
-          path.join(root, lockfile),
-          path.join(root, `${lockfile}.backup`),
-        )
-      }
-
       const packageJson = await detectNearestPackageJson(__dirname)
 
       const sourcePath = path.join(
@@ -137,30 +96,22 @@ export class FixtureSandbox {
       // Make sure the archive exists.
       await fs.access(sourcePath, fs.constants.R_OK)
 
-      const { executable, args } = packageManager.installCommand()
+      await installer.install(root, async (plan: any) => {
+        await plan.installDependencies()
 
-      await execa(executable, [...args, '--save-dev', `file:${sourcePath}`], {
-        cwd: root,
-      })
+        if (injectPackedSelf) {
+          await plan.injectPackedDependency(packageJson.name, {
+            file: sourcePath,
+            key: await fileHashSha256(packageJson.meta.filePath),
+          })
+        }
 
-      // Restore original package.json.
-      await fs.rename(
-        path.join(root, 'package.json.backup'),
-        path.join(root, 'package.json'),
-      )
-
-      // Restore original lockfile.
-      if (lockfile) {
-        await fs.rename(
-          path.join(root, `${lockfile}.backup`),
-          path.join(root, lockfile),
-        )
-      }
+        await plan.updateLockfile()
+      })
     }
 
     return new FixtureSandbox({
       root,
-      packageManager,
     })
   }
 

packages/cli/src/testing/fixture-sandbox.ts

@@ -0,0 +1,20 @@
+import { createLayer } from '../create.mjs'
+
+if (process.argv.length < 3) {
+  throw new Error('Missing required argument PACKAGE[@VERSION=latest]')
+}
+
+const input = {
+  pkg: process.argv[2],
+  outputDir: process.env['OUTDIR'] ?? process.cwd(),
+  outputFile: process.env['OUTFILE'],
+  onlyDeps: process.env['ONLY_DEPENDENCIES'] === '1',
+}
+
+try {
+  // eslint-disable-next-line no-console
+  console.log(await createLayer(input))
+} catch (err) {
+  // eslint-disable-next-line no-console
+  console.error(err)
+}

utilities/dependency-layer-installer/bin/dependency-layer-create.mjs

@@ -0,0 +1,126 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import os from 'node:os'
+
+import * as tar from 'tar'
+import Debug from 'debug'
+
+import { fetchLatestVersion, readPackageJson, splitPkg } from './util.mjs'
+import { npm } from './npm.mjs'
+
+const debug = Debug('checkly:utility:dependency-layer-installer:create')
+
+export async function createLayer ({ pkg, outputDir, outputFile, onlyDeps }) {
+  const [packageName, requestedVersion = 'latest'] = splitPkg(pkg)
+
+  const version = requestedVersion === 'latest'
+    ? await fetchLatestVersion(packageName)
+    : requestedVersion
+
+  debug('Creating build directory...')
+  const buildDir = await fs.mkdtemp(path.join(os.tmpdir(), 'make-npm-install-layer-'))
+  debug(`Build directory: ${buildDir}`)
+
+  const buildPath = (...paths) => path.join(buildDir, ...paths)
+  const sourcePath = (...paths) => buildPath('node_modules', ...paths)
+  const targetPath = (...paths) => buildPath('new_node_modules', ...paths)
+
+  try {
+    debug(`Installing ${packageName}@${version}...`)
+    await npm({
+      cwd: buildPath(),
+      args: ['install', '--ignore-scripts', `${packageName}@${version}`],
+    })
+    debug(`Installation complete`)
+
+    const pkgPath = packageName.split('/')
+
+    debug(`Reading package.json...`)
+    const {
+      version: installedVersion,
+      bin,
+    } = await readPackageJson(sourcePath(...pkgPath))
+
+    debug(`Installed version: ${installedVersion}`)
+
+    debug('Creating target path...')
+    await fs.mkdir(targetPath(), {
+      recursive: true,
+    })
+
+    if (bin !== undefined) {
+      debug(`Shuffling binaries...`)
+      await fs.mkdir(targetPath('.bin'), {
+        recursive: true,
+      })
+
+      for (const executable of Object.keys(bin)) {
+        debug(`Moving binary: ${executable}`)
+        await fs.rename(
+          sourcePath('.bin', executable),
+          targetPath('.bin', executable),
+        )
+      }
+    }
+
+    debug(`Removing cruft...`)
+    await fs.rm(sourcePath('.package-lock.json'), {
+      force: true,
+    })
+
+    debug(`Preparing target package path...`)
+    await fs.mkdir(targetPath(...pkgPath), {
+      recursive: true,
+    })
+
+    if (!onlyDeps) {
+      debug(`Moving source package to target...`)
+      await fs.rename(sourcePath(...pkgPath), targetPath(...pkgPath))
+    } else {
+      debug(`Removing source package...`)
+      await fs.rm(sourcePath(...pkgPath), {
+        recursive: true,
+      })
+    }
+
+    if (pkgPath.length > 1) {
+      try {
+        await fs.rmdir(pkgPath[0])
+      } catch {
+        // It has other contents, ignore.
+      }
+    }
+
+    debug(`Moving dependencies to target package...`)
+    await fs.rename(sourcePath(), targetPath(...pkgPath, 'node_modules'))
+
+    debug(`Renaming target to source...`)
+    await fs.rename(targetPath(), sourcePath())
+
+    const defaultOutputFile = `node_modules_layer_${pkgPath.join('_')}-v${installedVersion}.tgz`
+
+    const outputFilePath = path.resolve(outputDir, outputFile ?? defaultOutputFile)
+
+    debug('Creating archive...')
+    await tar.create({
+      cwd: buildPath(),
+      gzip: true,
+      file: outputFilePath,
+    }, [
+      'node_modules',
+    ])
+
+    const stat = await fs.stat(outputFilePath)
+
+    return {
+      layerFile: outputFilePath,
+      size: stat.size,
+      version: installedVersion,
+    }
+  } finally {
+    debug('Cleaning up build path...')
+    await fs.rm(buildDir, {
+      recursive: true,
+    })
+  }
+}

utilities/dependency-layer-installer/create.mjs

@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises'
+
+import * as tar from 'tar'
+import Debug from 'debug'
+
+const debug = Debug('checkly:utility:dependency-layer-installer:extract')
+
+export async function extractLayer ({ layerFile, extractDir }) {
+  await fs.mkdir(extractDir, {
+    recursive: true,
+  })
+
+  debug(`Extracting ${layerFile} to ${extractDir}...`)
+  await tar.extract({
+    file: layerFile,
+    cwd: extractDir,
+  })
+}

utilities/dependency-layer-installer/extract.mjs

@@ -0,0 +1,14 @@
+import fs from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import crypto from 'node:crypto'
+
+export async function fileHash (filePath, hash, { limit = 12 } = {}) {
+  const stream = fs.createReadStream(filePath)
+  await pipeline(stream, hash)
+  const digest = hash.digest('hex')
+  return digest.slice(0, limit)
+}
+
+export async function fileHashSha256 (filePath, { limit = 12 } = {}) {
+  return await fileHash(filePath, crypto.createHash('sha256'), { limit })
+}

utilities/dependency-layer-installer/hash.mjs

@@ -0,0 +1,2 @@
+export { createInstaller } from './installer.mjs'
+export { fileHash, fileHashSha256 } from './hash.mjs'