feat: buildkit volume caching (experimental)

AGENTS.md

@@ -0,0 +1,10 @@
+# setup-earthbuild GitHub Action
+
+## Tooling
+
+The primary development tool is "earth". You can use it to build and test the action.
+
+* `earth +compile` - Build the action.
+* `earth +test` - Run the tests.
+* `earth +all` - Build, lint, and test the action.
+* `earth doc` - List all targets.

action.yml

@@ -16,6 +16,10 @@ inputs:
     default: 'true'
   github-token:
     description: 'GitHub token for fetching EarthBuild version list.'
+  experimental-buildkit-volume-cache:
+    description: 'whether to use the experimental buildkit volume caching'
+    default: 'false'
+
 runs:
   using: node24
   main: dist/setup/index.js

dist/cache-save/index.js

@@ -39044,8 +39044,9 @@ var __webpack_exports__ = {};
 
 // EXPORTS
 __nccwpck_require__.d(__webpack_exports__, {
-  g: () => (/* binding */ cacheBinary),
-  e: () => (/* binding */ run)
+  gD: () => (/* binding */ cacheBinary),
+  eF: () => (/* binding */ run),
+  gs: () => (/* binding */ saveBuildkitCache)
 });
 
 // NAMESPACE OBJECT: ./node_modules/@azure/storage-blob/dist/esm/generated/src/models/mappers.js
@@ -39233,9 +39234,6 @@ __nccwpck_require__.d(mappers_namespaceObject, {
   UserDelegationKey: () => (UserDelegationKey)
 });
 
-;// CONCATENATED MODULE: external "fs"
-const external_fs_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("fs");
-var external_fs_default = /*#__PURE__*/__nccwpck_require__.n(external_fs_namespaceObject);
 // EXTERNAL MODULE: external "os"
 var external_os_ = __nccwpck_require__(857);
 ;// CONCATENATED MODULE: ./node_modules/@actions/core/lib/utils.js
@@ -39368,6 +39366,9 @@ function escapeProperty(s) {
 //# sourceMappingURL=command.js.map
 ;// CONCATENATED MODULE: external "crypto"
 const external_crypto_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("crypto");
+;// CONCATENATED MODULE: external "fs"
+const external_fs_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("fs");
+var external_fs_default = /*#__PURE__*/__nccwpck_require__.n(external_fs_namespaceObject);
 ;// CONCATENATED MODULE: ./node_modules/@actions/core/lib/file-command.js
 // For internal use, subject to change.
 // We use any as a valid input type
@@ -94657,18 +94658,6 @@ function saveCacheV2(paths_1, key_1, options_1) {
     });
 }
 //# sourceMappingURL=cache.js.map
-;// CONCATENATED MODULE: ./src/constants.ts
-var State;
-(function (State) {
-    State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
-    State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
-    State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
-})(State || (State = {}));
-var Outputs;
-(function (Outputs) {
-    Outputs["CacheHit"] = "cache-hit";
-})(Outputs || (Outputs = {}));
-
 ;// CONCATENATED MODULE: ./src/cache-utils.ts
 
 
@@ -94689,12 +94678,28 @@ function isCacheFeatureAvailable() {
     return true;
 }
 
+;// CONCATENATED MODULE: ./src/constants.ts
+var State;
+(function (State) {
+    State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
+    State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
+    State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
+    State["BuildkitCacheMatchedKey"] = "BUILDKIT_CACHE_RESULT";
+})(State || (State = {}));
+var Outputs;
+(function (Outputs) {
+    Outputs["CacheHit"] = "cache-hit";
+})(Outputs || (Outputs = {}));
+
 ;// CONCATENATED MODULE: ./src/cache-save.ts
 
 
 
 
 
+
+
+
 // Catch and log any unhandled exceptions.  These exceptions can leak out of the uploadChunk method in
 // @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to
 // throw an uncaught exception.  Instead of failing this action, just warn.
@@ -94705,6 +94710,7 @@ process.on('uncaughtException', (e) => {
 async function run() {
     try {
         await cacheBinary();
+        await saveBuildkitCache();
     }
     catch (error) {
         if (error instanceof Error) {
@@ -94756,10 +94762,72 @@ const cacheBinary = async () => {
         }
     }
 };
+const saveBuildkitCache = async () => {
+    if (!isCacheFeatureAvailable()) {
+        return;
+    }
+    const useBuildkitCache = getInput('experimental-buildkit-volume-cache') === 'true';
+    if (!useBuildkitCache) {
+        return;
+    }
+    const state = getState(State.BuildkitCacheMatchedKey);
+    const primaryKey = `earth-volume-cache-'}`;
+    const containerName = process.env.EARTHLY_INSTALLATION_NAME || 'earth-buildkitd';
+    const volumeName = 'earth-cache';
+    if (primaryKey === state) {
+        info(`Buildkit cache hit occurred on the primary key ${primaryKey}, not saving cache.`);
+        return;
+    }
+    try {
+        info(`Stopping buildkit container ${containerName}...`);
+        await exec_exec('docker', ['stop', containerName], {
+            ignoreReturnCode: true,
+        });
+        const cacheFile = external_path_.join(process.env.RUNNER_TEMP || external_os_.tmpdir(), 'earthbuild-buildkit-cache.tar.zst');
+        try {
+            const volumePath = `/var/lib/docker/volumes/${volumeName}/_data`;
+            info(`Compressing buildkit volume ${volumePath} to ${cacheFile}...`);
+            await exec_exec('sudo', [
+                'tar',
+                '-c',
+                '--use-compress-program=zstd -T0',
+                '-f',
+                cacheFile,
+                '-C',
+                volumePath,
+                '.',
+            ]);
+            await exec_exec('sudo', ['chmod', '666', cacheFile]);
+            await cache_saveCache([cacheFile], primaryKey);
+            info(`Buildkit cache saved with the key: ${primaryKey}`);
+        }
+        finally {
+            await external_fs_default().promises.rm(cacheFile, { force: true });
+        }
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.name === ValidationError.name) {
+                throw error;
+            }
+            else if (error.name === ReserveCacheError.name) {
+                info(error.message);
+            }
+            else {
+                warning(error.message);
+            }
+        }
+        else {
+            core_error(`unknown error encountered saving buildkit cache: ${String(error)}`);
+            throw error;
+        }
+    }
+};
 void run();
 
-var __webpack_exports__cacheBinary = __webpack_exports__.g;
-var __webpack_exports__run = __webpack_exports__.e;
-export { __webpack_exports__cacheBinary as cacheBinary, __webpack_exports__run as run };
+var __webpack_exports__cacheBinary = __webpack_exports__.gD;
+var __webpack_exports__run = __webpack_exports__.eF;
+var __webpack_exports__saveBuildkitCache = __webpack_exports__.gs;
+export { __webpack_exports__cacheBinary as cacheBinary, __webpack_exports__run as run, __webpack_exports__saveBuildkitCache as saveBuildkitCache };
 
 //# sourceMappingURL=index.js.map

dist/setup/index.js

@@ -39142,6 +39142,18 @@ module.exports = /*#__PURE__*/JSON.parse('{"name":"@actions/cache","version":"6.
 /******/ }
 /******/ 
 /************************************************************************/
+/******/ /* webpack/runtime/compat get default export */
+/******/ (() => {
+/******/ 	// getDefaultExport function for compatibility with non-harmony modules
+/******/ 	__nccwpck_require__.n = (module) => {
+/******/ 		var getter = module && module.__esModule ?
+/******/ 			() => (module['default']) :
+/******/ 			() => (module);
+/******/ 		__nccwpck_require__.d(getter, { a: getter });
+/******/ 		return getter;
+/******/ 	};
+/******/ })();
+/******/ 
 /******/ /* webpack/runtime/create fake namespace object */
 /******/ (() => {
 /******/ 	var getProto = Object.getPrototypeOf ? (obj) => (Object.getPrototypeOf(obj)) : (obj) => (obj.__proto__);
@@ -39394,6 +39406,7 @@ __nccwpck_require__.d(mappers_namespaceObject, {
 
 ;// CONCATENATED MODULE: external "node:fs/promises"
 const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:fs/promises");
+var promises_default = /*#__PURE__*/__nccwpck_require__.n(promises_namespaceObject);
 // EXTERNAL MODULE: external "os"
 var external_os_ = __nccwpck_require__(857);
 // EXTERNAL MODULE: external "path"
@@ -95612,6 +95625,7 @@ var State;
     State["CachePrimaryKey"] = "EARTHBUILD_CACHE_KEY";
     State["CacheMatchedKey"] = "EARTHBUILD_CACHE_RESULT";
     State["BinaryPath"] = "EARTHBUILD_BINARY_PATH";
+    State["BuildkitCacheMatchedKey"] = "BUILDKIT_CACHE_RESULT";
 })(State || (State = {}));
 var Outputs;
 (function (Outputs) {
@@ -95643,6 +95657,10 @@ function isCacheFeatureAvailable() {
 
 
 
+
+
+
+
 const cache_restore_restoreCache = async (path, version) => {
     if (!isCacheFeatureAvailable()) {
         return false;
@@ -95663,6 +95681,43 @@ const cache_restore_restoreCache = async (path, version) => {
     info(`Cache restored from key: ${cacheKey}`);
     return Boolean(cacheKey);
 };
+const restoreBuildkitCache = async () => {
+    if (!isCacheFeatureAvailable()) {
+        return false;
+    }
+    const useBuildkitCache = getInput('experimental-buildkit-volume-cache') === 'true';
+    if (!useBuildkitCache) {
+        return false;
+    }
+    const cacheKeyInput = `earth-volume-cache-${process.env.GITHUB_SHA || 'unknown'}`;
+    const restoreKeys = ['earth-volume-cache-'];
+    const volumeName = 'earth-cache';
+    const cacheFile = external_path_.join(process.env.RUNNER_TEMP || external_os_.tmpdir(), 'earthbuild-buildkit-cache.tar.zst');
+    try {
+        const cacheKey = await restoreCache([cacheFile], cacheKeyInput, restoreKeys);
+        if (!cacheKey) {
+            info('EarthBuild buildkit volume cache not found');
+            return false;
+        }
+        saveState(State.BuildkitCacheMatchedKey, cacheKey);
+        info(`Buildkit cache restored from key: ${cacheKey}`);
+        const volumePath = `/var/lib/docker/volumes/${volumeName}/_data`;
+        info(`Extracting cache to ${volumePath}`);
+        await exec_exec('sudo', ['mkdir', '-p', volumePath]);
+        try {
+            await promises_default().access(cacheFile);
+            await exec_exec('sudo', ['tar', '-xf', cacheFile, '-C', volumePath]);
+            info('Buildkit cache successfully extracted');
+        }
+        catch (err) {
+            warning(`Failed to extract buildkit cache: ${String(err)}`);
+        }
+        return true;
+    }
+    finally {
+        await promises_default().rm(cacheFile, { force: true });
+    }
+};
 
 ;// CONCATENATED MODULE: ./node_modules/universal-user-agent/index.js
 function getUserAgent() {
@@ -97485,13 +97540,15 @@ async function run() {
         if (toolcacheDir) {
             addPath(toolcacheDir);
             info(`using earthbuild from toolcache (${toolcacheDir})`);
+            await restoreBuildkitCache();
             return;
         }
         // then try to restore earthbuild from the github action cache
         addPath(installationDir);
         const restored = await cache_restore_restoreCache(installationPath, node_modules_semver.clean(tag_name) || tag_name.substring(1));
         if (restored) {
             await promises_namespaceObject.chmod(installationPath, 0o755);
+            await restoreBuildkitCache();
             return;
         }
         // finally, dowload EarthBuild release binary
@@ -97503,6 +97560,7 @@ async function run() {
         core_debug(`successfully downloaded ${buildURL} to ${downloaded}`);
         await promises_namespaceObject.chmod(installationPath, 0o755);
         await cacheDir(external_path_.join(destination, 'bin'), pkgName, node_modules_semver.clean(tag_name) || tag_name.substring(1), external_os_.arch());
+        await restoreBuildkitCache();
     }
     catch (error) {
         if (error instanceof Error) {

package.json

@@ -24,6 +24,7 @@
   "dependencies": {
     "@actions/cache": "6.1.0",
     "@actions/core": "3.0.1",
+    "@actions/exec": "3.0.0",
     "@actions/tool-cache": "4.0.0",
     "@octokit/core": "7.0.6",
     "@octokit/plugin-paginate-rest": "14.0.0",

src/cache-restore.ts

@@ -1,6 +1,9 @@
 import * as cache from '@actions/cache';
 import * as core from '@actions/core';
-
+import * as exec from '@actions/exec';
+import fs from 'node:fs/promises';
+import * as os from 'os';
+import * as path from 'path';
 import { State } from './constants';
 
 import * as utils from './cache-utils';
@@ -34,3 +37,56 @@ export const restoreCache = async (
   core.info(`Cache restored from key: ${cacheKey}`);
   return Boolean(cacheKey);
 };
+
+export const restoreBuildkitCache = async (): Promise<boolean> => {
+  if (!utils.isCacheFeatureAvailable()) {
+    return false;
+  }
+
+  const useBuildkitCache =
+    core.getInput('experimental-buildkit-volume-cache') === 'true';
+  if (!useBuildkitCache) {
+    return false;
+  }
+
+  const cacheKeyInput = `earth-volume-cache-${process.env.GITHUB_SHA || 'unknown'}`;
+  const restoreKeys = ['earth-volume-cache-'];
+  const volumeName = 'earth-cache';
+
+  const cacheFile = path.join(
+    process.env.RUNNER_TEMP || os.tmpdir(),
+    'earthbuild-buildkit-cache.tar.zst',
+  );
+  try {
+    const cacheKey = await cache.restoreCache(
+      [cacheFile],
+      cacheKeyInput,
+      restoreKeys,
+    );
+
+    if (!cacheKey) {
+      core.info('EarthBuild buildkit volume cache not found');
+      return false;
+    }
+
+    core.saveState(State.BuildkitCacheMatchedKey, cacheKey);
+    core.info(`Buildkit cache restored from key: ${cacheKey}`);
+
+    const volumePath = `/var/lib/docker/volumes/${volumeName}/_data`;
+    core.info(`Extracting cache to ${volumePath}`);
+
+    await exec.exec('sudo', ['mkdir', '-p', volumePath]);
+
+    try {
+      await fs.access(cacheFile);
+      await exec.exec('sudo', ['tar', '-xf', cacheFile, '-C', volumePath]);
+      core.info('Buildkit cache successfully extracted');
+    } catch (err) {
+      core.warning(`Failed to extract buildkit cache: ${String(err)}`);
+    }
+
+    return true;
+  } finally {
+    await fs.rm(cacheFile, { force: true });
+  }
+};