From 57709cdf9ab3e6a9f0907f4f4930fd6d7417588d Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Tue, 6 Aug 2024 14:37:27 +0100 Subject: [PATCH] Updates --- 404.html | 4 +- CNAME | 4 +- _next/data/j64IiYEEq_eJG_mFKDqPk/blog.json | 1 + .../blog/actuated-for-gitlab.json | 1 + .../blog/amd-zenbleed-update-now.json | 2 +- .../blog/arm-ci-cncf-ampere.json | 1 + .../automate-packer-qemu-image-builds.json | 1 + .../blog/blazing-fast-ci-with-microvms.json | 1 + .../blog/burst-billing-capacity.json | 2 +- .../blog/caching-in-github-actions.json | 1 + .../blog/calyptia-case-study-arm.json | 1 + ...-bring-your-own-bare-metal-to-actions.json | 1 + .../blog/cncf-arm-march-update.json | 1 + .../blog/custom-sizes-bpf-kvm.json | 1 + .../blog/develop-a-great-go-cli.json | 1 + .../blog/faster-nix-builds.json | 1 + .../blog/faster-self-hosted-cache.json | 1 + .../blog/firecracker-container-lab.json | 1 + .../blog/github-actions-usage-cli.json | 1 + .../blog/gpus-for-github-actions.json | 1 + ...how-to-run-multi-arch-builds-natively.json | 1 + ...elf-hosted-runner-safe-github-actions.json | 1 + .../blog/kvm-in-github-actions.json | 1 + .../local-caching-for-github-actions.json | 1 + .../blog/managing-github-actions.json | 1 + .../blog/millions-of-cncf-minutes.json | 1 + .../multi-arch-docker-github-actions.json | 1 + .../blog/native-arm64-for-github-actions.json | 1 + .../blog/oidc-proxy-for-openfaas.json | 1 + .../blog/ollama-in-github-actions.json | 1 + .../blog/right-sizing-vms-github-actions.json | 2 +- .../blog/sbom-in-github-actions.json | 1 + .../blog/secure-microvm-ci-gitlab.json | 1 + _next/data/qP6XrePfh_ktdNhbSnok_/blog.json | 1 - .../blog/arm-ci-cncf-ampere.json | 1 - .../automate-packer-qemu-image-builds.json | 1 - .../blog/blazing-fast-ci-with-microvms.json | 1 - .../blog/caching-in-github-actions.json | 1 - .../blog/calyptia-case-study-arm.json | 1 - ...-bring-your-own-bare-metal-to-actions.json | 1 - .../blog/cncf-arm-march-update.json | 1 - .../blog/custom-sizes-bpf-kvm.json | 1 - .../blog/develop-a-great-go-cli.json | 1 - .../blog/faster-nix-builds.json | 1 - .../blog/faster-self-hosted-cache.json | 1 - .../blog/firecracker-container-lab.json | 1 - .../blog/github-actions-usage-cli.json | 1 - .../blog/gpus-for-github-actions.json | 1 - ...how-to-run-multi-arch-builds-natively.json | 1 - ...elf-hosted-runner-safe-github-actions.json | 1 - .../blog/kvm-in-github-actions.json | 1 - .../local-caching-for-github-actions.json | 1 - .../blog/managing-github-actions.json | 1 - .../blog/millions-of-cncf-minutes.json | 1 - .../multi-arch-docker-github-actions.json | 1 - .../blog/native-arm64-for-github-actions.json | 1 - .../blog/oidc-proxy-for-openfaas.json | 1 - .../blog/ollama-in-github-actions.json | 1 - .../blog/sbom-in-github-actions.json | 1 - .../blog/secure-microvm-ci-gitlab.json | 1 - ...2dc4f1b3be.js => _app-70ff176000ac1f10.js} | 2 +- ...3e7e15fca.js => index-5c484ad9161573db.js} | 2 +- ...59bb5a6.js => pricing-bc3f197b23b20726.js} | 2 +- ...b45a25d8b.js => terms-4de8b8e50459fe3f.js} | 2 +- .../j64IiYEEq_eJG_mFKDqPk/_buildManifest.js | 1 + .../_ssgManifest.js | 0 .../qP6XrePfh_ktdNhbSnok_/_buildManifest.js | 1 - blog.html | 2 +- blog/actuated-for-gitlab.html | 190 ++++++ blog/amd-zenbleed-update-now.html | 10 +- blog/arm-ci-cncf-ampere.html | 18 +- blog/automate-packer-qemu-image-builds.html | 22 +- blog/blazing-fast-ci-with-microvms.html | 30 +- blog/burst-billing-capacity.html | 6 +- blog/caching-in-github-actions.html | 12 +- blog/calyptia-case-study-arm.html | 34 +- ...-bring-your-own-bare-metal-to-actions.html | 22 +- blog/cncf-arm-march-update.html | 10 +- blog/custom-sizes-bpf-kvm.html | 18 +- blog/develop-a-great-go-cli.html | 32 +- blog/faster-nix-builds.html | 14 +- blog/faster-self-hosted-cache.html | 22 +- blog/firecracker-container-lab.html | 32 +- blog/github-actions-usage-cli.html | 14 +- blog/gpus-for-github-actions.html | 20 +- ...how-to-run-multi-arch-builds-natively.html | 16 +- ...elf-hosted-runner-safe-github-actions.html | 16 +- blog/kvm-in-github-actions.html | 16 +- blog/local-caching-for-github-actions.html | 14 +- blog/managing-github-actions.html | 30 +- blog/millions-of-cncf-minutes.html | 18 +- blog/multi-arch-docker-github-actions.html | 20 +- blog/native-arm64-for-github-actions.html | 16 +- blog/oidc-proxy-for-openfaas.html | 10 +- blog/ollama-in-github-actions.html | 22 +- blog/right-sizing-vms-github-actions.html | 8 +- blog/sbom-in-github-actions.html | 12 +- blog/secure-microvm-ci-gitlab.html | 14 +- images/2024-07-gitlab/background.png | Bin 0 -> 80772 bytes images/2024-07-gitlab/chart-warning.png | Bin 0 -> 7856 bytes images/2024-07-gitlab/peering.png | Bin 0 -> 47181 bytes images/2024-07-gitlab/project-runners.png | Bin 0 -> 82648 bytes index.html | 2 +- pricing.html | 2 +- rss.xml | 611 ++++++++++++------ terms.html | 2 +- 106 files changed, 899 insertions(+), 517 deletions(-) create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/actuated-for-gitlab.json rename _next/data/{qP6XrePfh_ktdNhbSnok_ => j64IiYEEq_eJG_mFKDqPk}/blog/amd-zenbleed-update-now.json (51%) create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/arm-ci-cncf-ampere.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/automate-packer-qemu-image-builds.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/blazing-fast-ci-with-microvms.json rename _next/data/{qP6XrePfh_ktdNhbSnok_ => j64IiYEEq_eJG_mFKDqPk}/blog/burst-billing-capacity.json (71%) create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/caching-in-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/calyptia-case-study-arm.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/case-study-bring-your-own-bare-metal-to-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/cncf-arm-march-update.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/custom-sizes-bpf-kvm.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/develop-a-great-go-cli.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-nix-builds.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-self-hosted-cache.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/firecracker-container-lab.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/github-actions-usage-cli.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/gpus-for-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/how-to-run-multi-arch-builds-natively.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/is-the-self-hosted-runner-safe-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/kvm-in-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/local-caching-for-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/managing-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/millions-of-cncf-minutes.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/multi-arch-docker-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/native-arm64-for-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/oidc-proxy-for-openfaas.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/ollama-in-github-actions.json rename _next/data/{qP6XrePfh_ktdNhbSnok_ => j64IiYEEq_eJG_mFKDqPk}/blog/right-sizing-vms-github-actions.json (56%) create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/sbom-in-github-actions.json create mode 100644 _next/data/j64IiYEEq_eJG_mFKDqPk/blog/secure-microvm-ci-gitlab.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/arm-ci-cncf-ampere.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/automate-packer-qemu-image-builds.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/blazing-fast-ci-with-microvms.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/caching-in-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/calyptia-case-study-arm.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/case-study-bring-your-own-bare-metal-to-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/cncf-arm-march-update.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/custom-sizes-bpf-kvm.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/develop-a-great-go-cli.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-nix-builds.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-self-hosted-cache.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/firecracker-container-lab.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/github-actions-usage-cli.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/gpus-for-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/how-to-run-multi-arch-builds-natively.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/is-the-self-hosted-runner-safe-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/kvm-in-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/local-caching-for-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/managing-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/millions-of-cncf-minutes.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/multi-arch-docker-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/native-arm64-for-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/oidc-proxy-for-openfaas.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/ollama-in-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/sbom-in-github-actions.json delete mode 100644 _next/data/qP6XrePfh_ktdNhbSnok_/blog/secure-microvm-ci-gitlab.json rename _next/static/chunks/pages/{_app-a058602dc4f1b3be.js => _app-70ff176000ac1f10.js} (99%) rename _next/static/chunks/pages/{index-44069e03e7e15fca.js => index-5c484ad9161573db.js} (98%) rename _next/static/chunks/pages/{pricing-728f7d3fc59bb5a6.js => pricing-bc3f197b23b20726.js} (98%) rename _next/static/chunks/pages/{terms-d47029db45a25d8b.js => terms-4de8b8e50459fe3f.js} (97%) create mode 100644 _next/static/j64IiYEEq_eJG_mFKDqPk/_buildManifest.js rename _next/static/{qP6XrePfh_ktdNhbSnok_ => j64IiYEEq_eJG_mFKDqPk}/_ssgManifest.js (100%) delete mode 100644 _next/static/qP6XrePfh_ktdNhbSnok_/_buildManifest.js create mode 100644 blog/actuated-for-gitlab.html create mode 100644 images/2024-07-gitlab/background.png create mode 100644 images/2024-07-gitlab/chart-warning.png create mode 100644 images/2024-07-gitlab/peering.png create mode 100644 images/2024-07-gitlab/project-runners.png diff --git a/404.html b/404.html index a0d52abb..29b92f06 100644 --- a/404.html +++ b/404.html @@ -4,7 +4,7 @@ gtag('js', new Date()); gtag('config', 'G-M5YNDNX7VT'); -
ActuatedActuated logo

404

This page could not be found.

\ No newline at end of file + }

404

This page could not be found.

\ No newline at end of file diff --git a/CNAME b/CNAME index bc1b0f79..ee28f4e7 100644 --- a/CNAME +++ b/CNAME @@ -1,2 +1,2 @@ -actuated.dev -www.actuated.dev \ No newline at end of file +actuated.com +www.actuated.com \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog.json new file mode 100644 index 00000000..8ec225a2 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog.json @@ -0,0 +1 @@ +{"pageProps":{"posts":[{"slug":"actuated-for-gitlab","fileName":"2024-07-26-actuated-for-gitlab.md","title":"Secure your GitLab jobs with microVMs and Actuated","description":"Learn how microVMs provide more flexibility and security for CI workloads, by removing the need for privileged containers or Docker In Docker.","tags":["gitlab","security","docker"],"author_img":"welteki","image":"/images/2024-07-gitlab/background.png","date":"2024-07-26"},{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"},{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"},{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"},{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"},{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"},{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"},{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"},{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"},{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"},{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"},{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"},{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"},{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"},{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"},{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"},{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"},{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"},{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"},{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"},{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"},{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"},{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"},{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"},{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"},{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"},{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"},{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"},{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"},{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}]},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/actuated-for-gitlab.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/actuated-for-gitlab.json new file mode 100644 index 00000000..7a7d7767 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/actuated-for-gitlab.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"actuated-for-gitlab","fileName":"2024-07-26-actuated-for-gitlab.md","contentHtml":"

Last year we introduced the tech preview for Actuated for GitLab CI, since then we've had customer interest from enterprise companies who wanted to improve their security posture and to lower overheads. Actuated reduces management overheads of self-hosted runners and provides a secure, ephemeral microVM for every job.

\n

We've made a lot of progress since the original version and are looking for additional customers who want to deliver an improved CI experience. In this article we will give you an overview of some of the available features and how they can benefit your GitLab CI.

\n\n

Why are microVMs the future of CI?

\n

It can be challenging to run GitLab CI/CD jobs that build and publish Docker images or jobs that require extensive system access in a safe way. Docker-in-Docker (DIND) requires the docker executor to run containers in privileged mode. Using the shell executor would give a job full access to the runner host and network. They can also leave behind side effects between builds as the runner is reused.

\n

Using both approaches causes a significant security concern and the GitLab runner security docs warn against it.

\n

\"Docker

\n
\n

Security notice displayed by the GitLab Helm chart to explain why docker in docker is disabled by default for security purposes.

\n
\n

With Actuated, jobs run in ephemeral microVMs using Linux KVM for secure isolation. After the job is completed, the VM will be destroyed and removed from the GitLab instance. This allows us to safely run DIND and the shell executor in a fresh isolated environment for each job.

\n

There are no horrible Kernel tricks or workarounds required to be able to use user namespaces, no need to change your tooling from what developers love - Docker, to Kaniko or Buildah or similar. You have sudo access and full VM with systemd available, things like Kubernetes will also work out of the box if you need them for end to end testing.

\n

\"GitLab

\n
\n

Runners get automatically added to a project and are removed again when they finish running a job.

\n
\n

Run jobs in microVMs with Actuated

\n

When a pipeline is triggered through a commit, merge request or in the UI the Actuated control plane gets notified through a webhook. For every job we schedule and run a new microVM and register it as a runner to the project. After the job is completed, the VM will be destroyed and removed from the GitLab instance. Scheduling and launching VMs is very fast. On average a new VM is booting up and running the job within 1 second.

\n

\"actuated

\n

The agent will use either Firecracker or Cloud Hypervisor to launch microVMS depending on whether GPU support is required. microVMs boot almost instantly and in most cases will be faster than Kubernetes since the image is optimized and already available on each server.

\n

To run jobs on Actuated the actuated tag has to be added to a job. One feature our customers like is the ability to configure the VM size for a job through the tag. Using the tag actuated-4cpu-8gb will schedule a VM with 4 vCPUs and 8 gigabytes of RAM.

\n

You can pick any combination for vCPU and RAM. There's no need to pick a predefined runner size. This means that runners can be sized accordingly for the job they need to run so that the available CPU and memory resources can be used more efficiently.

\n

Example .gitlab-ci.yaml that runs a job on Actuated runners using the docker executor:

\n
image: ruby:2.7\nservices:\n  - postgres:9.3\n\nbefore_script:\n  - bundle install\ntest:\n  script:\n    - bundle exec rake spec\n  tags:\n    - actuated-4cpu-8gb\n
\n

Mixed Docker and Shell executors

\n

GitLab supports a number of executors to run builds in different environments. With Actuated we support running jobs with the docker and shell executor.

\n

There is no need to pre-configure the type of executors you want to use. Actuated allows you to quickly select the executor for a job by adding an additional tag. Adding the shell tag to a job will launch a VM and register the GitLab runners using the shell executor. If no tag is provided the docker executor is used by default.

\n
build-job:\n  stage: build\n  script:\n    - echo \"Hi $GITLAB_USER_LOGIN!\"\n  tags:\n    - actuated-2cpu-4gb\n    - shell\n
\n

With Actuated the shell executor can be used securely without leaving side effects behind that can influence job execution. A clean isolated build environment is provided for every job since the GitLab runner is started on an ephemeral VM that is removed as soon as the job has completed.

\n

Using the shell executor in an isolated VM lets you safely run workloads like:

\n\n

These kinds of jobs can be difficult to run in a docker container or would require the container to run in privileged mode which is unsafe and advised against in GitLab runner security guidelines.

\n

Since jobs run in ephemeral VMs with Actuated it is also possible to run the docker executor safely in privileged mode. If you are already using the docker executor in privileged mode Actuated can improve the security of your jobs without making changes to your existing pipelines.

\n

The following .gitlab-ci.yaml runs two jobs. The first job uses the docker executor to build and push a container image for an OpenFaaS function with Docker and the faas-cli. The second job sets the additional shell tag to request Actuated to run the job with the shell executor. By running the jobs with the shell executor we get access to the full ephemeral VM that is launched for the job. This makes it easy to bootstrap a K3s Kubernetes cluster with k3sup for E2E testing the function with OpenFaaS.

\n
stages:\n  - push\n  - e2e\n\nvariables:\n  DOCKER_DRIVER: overlay2\n  DOCKER_TLS_CERT_DIR: \"\"\n\npush_job:\n  stage: push\n  image: docker:latest\n  before_script:\n    # Install dependencies: faas-cli\n    - apk add --no-cache git curl\n    - if [ -f \"./faas-cli\" ] ; then cp ./faas-cli /usr/local/bin/faas-cli || 0 ; fi\n    - if [ ! -f \"/usr/local/bin/faas-cli\" ] ; then apk add --no-cache curl git &&\n      curl -sSL https://cli.openfaas.com |\n      sh && chmod +x /usr/local/bin/faas-cli &&\n      cp /usr/local/bin/faas-cli ./faas-cli ; fi\n  script:\n    - echo $CI_JOB_TOKEN | docker login $CI_REGISTRY \\\n        -u $CI_REGISTRY_USER \\\n        --password-stdin\n\n    # Build and push an OpenFaaS function\n    - /usr/local/bin/faas-cli template pull stack\n    - /usr/local/bin/faas-cli publish\n\n  tags:\n    - actuated-4cpu-8gb\n\ne2e_job:\n  stage: e2e\n  before_script:\n    # Install dependencies: faas-cli kubectl kubectx k3sup\n    - curl -SLs https://get.arkade.dev | sh\n    - export PATH=$PATH:$HOME/.arkade/bin/\n    - arkade get faas-cli kubectl kubectx k3sup --progress=false\n  script:\n    # Deploy a K3s cluster.\n    - |\n      mkdir -p ~/.kube/\n      k3sup install --local --local-path ~/.kube/config\n      k3sup ready\n    - kubectl get nodes\n\n    # Install OpenFaaS on the local cluster.\n    - mkdir -p ~/.openfaas && echo $OPENFAAS_LICENSE > ~/.openfaas/LICENSE\n    - |\n      arkade install openfaas \\\n        --license-file ~/.openfaas/LICENSE \\\n        --operator \\\n        --clusterrole \\\n        --jetstream \\\n        --autoscaler\n      kubectl get secret -n openfaas\n      echo $OPENFAAS_LICENSE | wc\n    - kubectl create secret docker-registry gitlab-ci-pull \\\n        --docker-server=$CI_REGISTRY \\\n        --docker-username=$CI_REGISTRY_USER \\\n        --docker-password=$CI_JOB_TOKEN \\\n        --docker-email=docker@example.com \\\n        --namespace openfaas-fn\n    - kubectl rollout status -n openfaas deploy/gateway --timeout=60s\n    - |\n      kubectl port-forward -n openfaas svc/gateway 8080:8080 &>/dev/null &\n      echo -n \"$!\" > kubectl-pid.txt\n\n      echo PID for port-fowarding: $(cat kubectl-pid.txt)\n\n    - faas-cli ready --attempts 120\n\n    - |\n      kubectl patch serviceaccount \\\n        -n openfaas-fn default \\\n        -p '{\"imagePullSecrets\": [{\"name\": \"gitlab-ci-pull\"}]}'\n\n    # Deploy the function that we build and pushed in the previous stage.\n    - |\n      echo $(kubectl get secret \\\n        -n openfaas basic-auth \\\n        -o jsonpath=\"{.data.basic-auth-password}\" | base64 --decode; echo) |\n      faas-cli login --username admin --password-stdin\n    - faas-cli template pull stack\n    - CI_REGISTRY=$CI_REGISTRY CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA faas-cli deploy\n\n    # Test the OpenFaaS function by invoking it.\n    - faas-cli ready bcrypt\n    - curl -i http://127.0.0.1:8080/function/bcrypt -d \"\"\n\n    - kill -9 \"$(cat kubectl-pid.txt)\"\n\n  tags:\n    - actuated-4cpu-8gb\n    - shell\n
\n

What does an actuated server look like?

\n

An actuated server is where VMs are run for your CI jobs. It needs a minimal server operating system, and there's rarely a reason to even log into the host once it's setup and connected to our control plane.

\n

Actuated servers need to make use of /dev/kvm, so the processor and Kernel must support virtualization, most bare-metal servers support this without any additional configuration. Certain cloud VMs, OpenStack and VMware support KVM through nested virtualization.

\n

The recommended Operating System is the LTS version of Ubuntu Server, then you just need to install our agent and enroll it with the actuated control plane.

\n

If there happens to be some serious hardware failure or a major OS upgrade is required, then you can just re-image the disk and install the agent again, the whole process takes a couple of minutes.

\n

Private peering

\n

You can host your actuated servers on the public cloud or on-premises in your own data center. For Internet facing hosts, just open port 80 and 443.

\n

For hosts behind a private network, you can enable peering, which makes an outbound connection that passes through most firewalls, NAT, HTTP Proxies and VPNs without any additional configuration.

\n

Agent peering simplifies agent setup and improves security:

\n\n

\"Private

\n
\n

Peering example for an enterprise with two agents within their own private firewalls.

\n
\n

Wrapping up

\n

Compared to GitLab CI's built-in solution, there is no Kubernetes cluster required. VMs are densely packed into a fleet of servers efficiently or kept in a queue until capacity becomes available.

\n

Each VM is destroyed after the job has completed, so there are no side-effects to be concerned about, or server maintenance to be done.

\n

Docker can be used natively without any of the risks with the built-in Kubernetes runners which use privileged Pods.

\n

There is no need to add side-cars for BuildKit, Kaniko or to have to fight with the issues surrounding user-namespaces.

\n

Everything is tested by our team, so you don't have to fine-tune or configure a Kernel, we ship the OS image and Kernel together and update them remotely.

\n

Actuated for GitLab is available for self-hosted GitLab instances hosted on-premise or on the public cloud.

\n

Talk to us, if you would like to see how Actuated can improve your GitLab CI.

","title":"Secure your GitLab jobs with microVMs and Actuated","description":"Learn how microVMs provide more flexibility and security for CI workloads, by removing the need for privileged containers or Docker In Docker.","tags":["gitlab","security","docker"],"author_img":"welteki","image":"/images/2024-07-gitlab/background.png","date":"2024-07-26"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/amd-zenbleed-update-now.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/amd-zenbleed-update-now.json similarity index 51% rename from _next/data/qP6XrePfh_ktdNhbSnok_/blog/amd-zenbleed-update-now.json rename to _next/data/j64IiYEEq_eJG_mFKDqPk/blog/amd-zenbleed-update-now.json index c8ef0073..97707758 100644 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/amd-zenbleed-update-now.json +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/amd-zenbleed-update-now.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","contentHtml":"

On 24th July 2023, The Register covered a new exploit for certain AMD CPUs based upon the Zen architecture. The exploit, dubbed Zenbleed, allows an attacker to read arbitrary physical memory locations on the host system. It works by allowing memory to be read after it's been set to be freed up aka \"use-after-free\". This is a serious vulnerability, and you should update your AMD hosts as soon as possible.

\n

The Register made the claim that \"any level of emulation such as QEMU\" would prevent the exploit from working. This is misleading because QEMU only makes sense in production when used with hardware acceleration (KVM). We were able to run the exploit with a GitHub Action using actuated on an AMD Epyc server from Equinix Metal using Firecracker and KVM.

\n
\n

\"If you stick any emulation layer in between, such as Qemu, then the exploit understandably fails.\"

\n
\n

The editors at The Register have since reached out and updated their article.

\n

Even Firecracker with its isolated guest Kernel is vulnerable, which shows how serious the bug is, it's within the hardware itself. Of course it goes without saying that this also affects containerd, Docker (and by virtue Kubernetes) which share the host Kernel.

\n

To test this, we ran a GitHub Actions matrix build that creates many VMs running different versions of K3s. About the same time, we triggered a build which runs a Zenbleed exploit PoC written by Tavis Ormandy, a security researcher at Google.

\n

We found that the exploit was able to read the memory of the host system, and that the exploit was able to read the memory of other VMs running on the same host.

\n
name: build\n\non:\n    pull_request:\n        branches:\n        - '*'\n    push:\n        branches:\n        - master\n        - main\n    workflow_dispatch:\n\njobs:\n    build:\n        name: specs\n        runs-on: actuated\n        steps:\n        - uses: actions/checkout@v1\n        - name: Download exploit\n          run: |\n            curl -L -S -s -O https://lock.cmpxchg8b.com/files/zenbleed-v5.tar.gz\n            tar -xvf zenbleed-v5.tar.gz\n            sudo apt install -qy build-essential make nasm\n        - name: Build exploit\n          run: |\n            cd zenbleed\n            make\n            chmod +x ./zenbleed\n        - name: Run exploit for 1000 pieces of data\n          run: |\n            ./zenbleed/zenbleed -m 1000\n
\n

Full details of the exploit can be found on a microsite created by the security researcher who discovered the vulnerability. The -m 1000 flag reads 1000 pieces of memory and then exits.

\n

Zenbleed by Tavis Ormandy

\n

We didn't see any secrets printed out during the scan, but we did see part of a public SSH key, console output from etcd running within K3s, and instructions from containerd. So we can assume anything that was within memory within one of the other VMs on the host, or even the host itself, could be read by the exploit.

\n

\"GHA

\n
\n

GHA output from the zenbleed exploit

\n
\n

Mitigation

\n

AMD has already released a mitigation for the Zenbleed exploit, which requires an update to the CPU's microcode.

\n

Ed Vielmetti, Developer Partner Manager at Equinix told us that mitigation is three-fold:

\n
    \n
  1. There is an incoming BIOS update from certain vendors that will update the microcode.
    2. \n
  2. Updating some OSes like the Ubuntu 20.04 and 22.04 will upgrade the microcode of the CPU.
    3. \n
  3. There is a \"chicken bit\" that can be enabled which prevents the exploit from working.
    4. \n
\n

I probably don't need to spell this out, but a system update looks like the following, and the reboot is required:

\n
sudo apt update -qy && \\\n    sudo apt upgrade -yq && \\\n    sudo reboot \n
\n

\"An

\n

For some unknown reason, both of the Equinix AMD hosts that we use internally broke after running the OS upgrade, so I had to reinstall Ubuntu 22.04 using the dashboard. If for whatever reason the machine won't come up after the microcode update, then you should reinstall the Operating System (OS) using your vendor's rescue system or out of band console, both Equinix Metal and Hetzner have an \"easy button\" that you can click for this. If there is still an issue after that, reach out to your vendor's support team.

\n

New machines provisioned after this date should already contain the microcode fix or have the \"chicken bit\" enabled. We provisioned a new AMD Epyc server on Equinix Metal to make sure, and as expected, thanks to their hard work - it was not vulnerable.

\n

We offer 500 USD of free credit for new Equinix Metal customers to use with actuated, and Equinix Metal have also written up their own guide on workaround here:

\n\n

Verifying the mitigation

\n

Since actuated VMs use Firecracker, you should run the above workflow before and after to verify the exploit was a) present and b) mitigated.

\n

\"What

\n
\n

Above: What it looks like when the mitigation is in place

\n
\n

You can also run the exploit on the host by copying and pasting the commands from the GitHub Action above.

\n

My workstation uses a Ryzen 9 CPU, so when I ran the exploit I just saw a blocking message instead of memory regions:

\n
$ grep \"model name\" /proc/cpuinfo |uniq\nmodel name\t: AMD Ryzen 9 5950X 16-Core Processor\n\n./zenbleed -m 100\n*** EMBARGOED SECURITY ISSUE --  DO NOT DISTRIBUTE! ***\nZenBleed Testcase -- taviso@google.com\n\nNOTE: Try -h to see configuration options\n\nSpawning 32 Threads...\nThread 0x7fd0d40e8700 running on CPU 0\nThread 0x7fd0d38e7700 running on CPU 1\n...\n
\n

You can also run the following command to print out the microcode version. This is the output from the Equinix Metal server (c3.medium) that ran an OS update on:

\n
$ grep 'microcode' /proc/cpuinfo\nmicrocode\t: 0x830107a\n
\n

Wrapping up

\n

Actuated uses Firecracker, an open source Virtual Machine Manager (VMM) that works with Linux KVM to run isolated systems on a host. We have verified that the exploit works on Firecracker, and that the mitigation works too. So whilst VM-level isolation and an immutable filesystem is much more appropriate than a container for CI, this is an example of why we must still be vigilant and ready to respond to security vulnerabilities.

\n

This is an unfortunate, and serious vulnerability. It affects bare-metal, VMs and containers, which is why it's important to update your systems as soon as possible.

","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","contentHtml":"

On 24th July 2023, The Register covered a new exploit for certain AMD CPUs based upon the Zen architecture. The exploit, dubbed Zenbleed, allows an attacker to read arbitrary physical memory locations on the host system. It works by allowing memory to be read after it's been set to be freed up aka \"use-after-free\". This is a serious vulnerability, and you should update your AMD hosts as soon as possible.

\n

The Register made the claim that \"any level of emulation such as QEMU\" would prevent the exploit from working. This is misleading because QEMU only makes sense in production when used with hardware acceleration (KVM). We were able to run the exploit with a GitHub Action using actuated on an AMD Epyc server from Equinix Metal using Firecracker and KVM.

\n
\n

\"If you stick any emulation layer in between, such as Qemu, then the exploit understandably fails.\"

\n
\n

The editors at The Register have since reached out and updated their article.

\n

Even Firecracker with its isolated guest Kernel is vulnerable, which shows how serious the bug is, it's within the hardware itself. Of course it goes without saying that this also affects containerd, Docker (and by virtue Kubernetes) which share the host Kernel.

\n

To test this, we ran a GitHub Actions matrix build that creates many VMs running different versions of K3s. About the same time, we triggered a build which runs a Zenbleed exploit PoC written by Tavis Ormandy, a security researcher at Google.

\n

We found that the exploit was able to read the memory of the host system, and that the exploit was able to read the memory of other VMs running on the same host.

\n
name: build\n\non:\n    pull_request:\n        branches:\n        - '*'\n    push:\n        branches:\n        - master\n        - main\n    workflow_dispatch:\n\njobs:\n    build:\n        name: specs\n        runs-on: actuated\n        steps:\n        - uses: actions/checkout@v1\n        - name: Download exploit\n          run: |\n            curl -L -S -s -O https://lock.cmpxchg8b.com/files/zenbleed-v5.tar.gz\n            tar -xvf zenbleed-v5.tar.gz\n            sudo apt install -qy build-essential make nasm\n        - name: Build exploit\n          run: |\n            cd zenbleed\n            make\n            chmod +x ./zenbleed\n        - name: Run exploit for 1000 pieces of data\n          run: |\n            ./zenbleed/zenbleed -m 1000\n
\n

Full details of the exploit can be found on a microsite created by the security researcher who discovered the vulnerability. The -m 1000 flag reads 1000 pieces of memory and then exits.

\n

Zenbleed by Tavis Ormandy

\n

We didn't see any secrets printed out during the scan, but we did see part of a public SSH key, console output from etcd running within K3s, and instructions from containerd. So we can assume anything that was within memory within one of the other VMs on the host, or even the host itself, could be read by the exploit.

\n

\"GHA

\n
\n

GHA output from the zenbleed exploit

\n
\n

Mitigation

\n

AMD has already released a mitigation for the Zenbleed exploit, which requires an update to the CPU's microcode.

\n

Ed Vielmetti, Developer Partner Manager at Equinix told us that mitigation is three-fold:

\n
    \n
  1. There is an incoming BIOS update from certain vendors that will update the microcode.
    2. \n
  2. Updating some OSes like the Ubuntu 20.04 and 22.04 will upgrade the microcode of the CPU.
    3. \n
  3. There is a \"chicken bit\" that can be enabled which prevents the exploit from working.
    4. \n
\n

I probably don't need to spell this out, but a system update looks like the following, and the reboot is required:

\n
sudo apt update -qy && \\\n    sudo apt upgrade -yq && \\\n    sudo reboot \n
\n

\"An

\n

For some unknown reason, both of the Equinix AMD hosts that we use internally broke after running the OS upgrade, so I had to reinstall Ubuntu 22.04 using the dashboard. If for whatever reason the machine won't come up after the microcode update, then you should reinstall the Operating System (OS) using your vendor's rescue system or out of band console, both Equinix Metal and Hetzner have an \"easy button\" that you can click for this. If there is still an issue after that, reach out to your vendor's support team.

\n

New machines provisioned after this date should already contain the microcode fix or have the \"chicken bit\" enabled. We provisioned a new AMD Epyc server on Equinix Metal to make sure, and as expected, thanks to their hard work - it was not vulnerable.

\n

We offer 500 USD of free credit for new Equinix Metal customers to use with actuated, and Equinix Metal have also written up their own guide on workaround here:

\n\n

Verifying the mitigation

\n

Since actuated VMs use Firecracker, you should run the above workflow before and after to verify the exploit was a) present and b) mitigated.

\n

\"What

\n
\n

Above: What it looks like when the mitigation is in place

\n
\n

You can also run the exploit on the host by copying and pasting the commands from the GitHub Action above.

\n

My workstation uses a Ryzen 9 CPU, so when I ran the exploit I just saw a blocking message instead of memory regions:

\n
$ grep \"model name\" /proc/cpuinfo |uniq\nmodel name\t: AMD Ryzen 9 5950X 16-Core Processor\n\n./zenbleed -m 100\n*** EMBARGOED SECURITY ISSUE --  DO NOT DISTRIBUTE! ***\nZenBleed Testcase -- taviso@google.com\n\nNOTE: Try -h to see configuration options\n\nSpawning 32 Threads...\nThread 0x7fd0d40e8700 running on CPU 0\nThread 0x7fd0d38e7700 running on CPU 1\n...\n
\n

You can also run the following command to print out the microcode version. This is the output from the Equinix Metal server (c3.medium) that ran an OS update on:

\n
$ grep 'microcode' /proc/cpuinfo\nmicrocode\t: 0x830107a\n
\n

Wrapping up

\n

Actuated uses Firecracker, an open source Virtual Machine Manager (VMM) that works with Linux KVM to run isolated systems on a host. We have verified that the exploit works on Firecracker, and that the mitigation works too. So whilst VM-level isolation and an immutable filesystem is much more appropriate than a container for CI, this is an example of why we must still be vigilant and ready to respond to security vulnerabilities.

\n

This is an unfortunate, and serious vulnerability. It affects bare-metal, VMs and containers, which is why it's important to update your systems as soon as possible.

","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/arm-ci-cncf-ampere.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/arm-ci-cncf-ampere.json new file mode 100644 index 00000000..42d38482 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/arm-ci-cncf-ampere.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","contentHtml":"

In this post, we'll cover why Ampere Computing and The Cloud Native Computing Foundation (CNCF) are sponsoring a pilot of actuated for open source projects, how you can get involved.

\n

We'll also give you a quick one year recap on actuated, if you haven't checked in with us for a while.

\n

Managed Arm CI for CNCF projects

\n

At KubeCon EU, I spoke to Chris Aniszczyk, CTO at the Cloud Native Computing Foundation (CNCF), and told him about some of the results we'd been seeing with actuated customers, including Fluent Bit, which is a CNCF project. Chris told me that many teams were either putting off Arm support all together, were suffering with the slow builds that come from using QEMU, or were managing their own infrastructure which was underutilized.

\n

Equinix provides a generous amount of credits to the CNCF under CNCF Community Infrastructure Lab (CIL), including access to powerful Ampere Q80 Arm servers (c3.large.arm64), that may at times be required by Equinix customers for their own Arm workloads.

\n

You can find out more about Ampere's Altra platform here, which is being branded as a \"Cloud Native\" CPU, due to its low power consumption, high core count, and ubiquitous availability across Google Cloud, Oracle Cloud Platform, Azure, Equinix Metal, Hetzner Cloud, and Alibaba Cloud.

\n

As you can imagine, over time, different projects have deployed 1-3 of their own runner servers, each with 256GB of RAM and 80 Cores, which remain idle most of the time, and are not available to other projects or Equinix customers when they may need them suddenly. So, if actuated can reduce this number, whilst also improving the experience for maintainers, then that's a win-win.

\n

Around the same time as speaking to Chris, Ampere reached out and asked how they could help secure actuated for a number of CNCF projects.

\n

Together, Ampere and the CNCF are now sponsoring an initial 1-year pilot of managed Arm CI provided by actuated, for CNCF projects, with the view to expand it, if the pilot is a success.

\n

Ed Vielmetti, Developer Partner Manager at Equinix said:

\n
\n

I'm really happy to see this all come together. If all goes according to plan, we'll have better runner isolation, faster builds, and a smaller overall machine footprint.

\n
\n

Dave Neary, Director of Developer Relations at Ampere Computing added:

\n
\n

Actuated offers a faster, more secure way for projects to run 64-bit Arm builds, and will also more efficiently use the Ampere Altra-based servers being used by the projects.

\n

We're happy to support CNCF projects running their CI on Ampere Computing's Cloud Native Processors, hosted by Equinix.

\n
\n

One year recap on actuated

\n

In case you are hearing about actuated for the first time, I wanted to give you a quick one year recap.

\n

Just over 12 months ago, we announced the work we'd been doing with actuated to improve self-hosted runner security and management. We were pleasantly surprised with the amount of people that responded who'd had a common experience with slow builds, running out of RAM, limited disk space, and a lack of an easy and secure way to run self-hosted runners.

\n

Fast forward to today, and we have run over 140,000 individual Firecracker VMs for customers on their own hardware. Rather than the fully managed service that GitHub offers, we believe that you should be able to bring your own hardware, and pay a flat-rate fee for the service, rather than being charged per-minute.

\n

The CNCF project brings about 64-bit Arm support, but we see a good mix of x86_64 and Arm builds from customers, with both closed and open-source repositories being used.

\n

The main benefits are having access to bigger, faster and more specialist hardware.

\n\n

Vendors and consumers are becoming increasingly aware of the importance of the supply chain, GitHub's self-hosted runner is not recommended for open source repos. Why? Due to the way side-effects can be left over between builds. Actuated uses a fresh, immutable, Firecracker VM for every build which boots up in less than 1 second and is destroyed after the build completes, which removes this risk.

\n

If you're wanting to know more about why we think microVMs are the only tool that makes sense for secure CI, then I'd recommend my talk from Cloud Native Rejekts earlier in the year: Face off: VMs vs. Containers vs Firecracker.

\n

What are maintainers saying?

\n

Ellie Huxtable is the maintainer of Atuin, a popular open-source tool to sync, search and backup shell history. Her Rust build for the CLI took 90 minutes with QEMU, but was reduced to just 3 minutes with actuated, and a native Arm server.

\n

Thanks to @selfactuated, Atuin now has very speedy ARM docker builds in our GitHub actions! Thank you @alexellisuk 🙏

Docker builds on QEMU: nearly 90 mins
Docker builds on ARM with Actuated: ~3 mins

— Ellie Huxtable (@ellie_huxtable) October 20, 2023
\n

For Fluent Bit, one of their Arm builds was taking over 6 hours, which meant it always failed with a timed-out on a hosted runner. Patrick Stephens, Tech Lead of Infrastructure at Calyptia reached out to work with us. We got the time down to 5 minutes by changing runs-on: ubuntu-latest to runs-on: actuated-arm64-4cpu-16gb, and if you need more or less RAM/CPU, you can tune those numbers as you wish.

\n

Patrick shares about the experience on the Calyptia blog, including the benefits to their x86_64 builds for the commercial Calyptia product: Scaling ARM builds with Actuated.

\n

A number of CNCF maintainers and community leaders such as Davanum Srinivas (Dims), Principal Engineer at AWS have come forward with project suggestions, and we're starting to work through them, with the first two being Fluent Bit and etcd.

\n

Fluent Bit describes itself as:

\n
\n

..a super fast, lightweight, and highly scalable logging and metrics processor and forwarder. It is the preferred choice for cloud and containerized environments.

\n
\n

etcd is a core component of almost every Kubernetes installation and is responsible for storing the state of the cluster.

\n
\n

A distributed, reliable key-value store for the most critical data of a distributed system

\n
\n

In the case of etcd, there were two servers being maintained by five individual maintainers, all of that work goes away by adopting actuated.

\n

We even sent etcd a minimal Pull Request to make the process smoother.

\n

James Blair, Specialist Solution Architect at Red Hat, commented:

\n
\n

I believe managed on demand arm64 CI hosts will definitely be a big win for the project. Keen to trial this.

\n
\n

Another maintainer also commented that they will no longer need to worry about \"leaky containers\".

\n

\"One

\n
\n

One of the first nightly workflows running within 4x separate isolated Firecracker VMs, one per job

\n
\n

Prior to adopting actuated, the two servers were only configured to run one job at a time, afterwards, the jobs are scheduled by the control-plane, according to the amount of available RAM and CPU in the target servers.

\n

How do we get access?

\n

If you are working on a CNCF project and would like access, please contact us via this form. If your project gets selected for the pilot, there are a couple of things you may need to do.

\n
    \n
  1. If you are already using the GitHub Actions runner hosted on Equinix, then change the runs-on: self-hosted label to: runs-on: actuated-arm64-8cpu-16gb. Then after you've seen a build or two pass, delete the old runner.
    2. \n
  2. If you are using QEMU, the best next step is to \"split\" the build into two jobs which can run on x86_64 and arm64 natively, that's what Ellie did and it only took her a few minutes.
    3. \n
\n

The label for runs-on: allows for dynamic configuration of vCPU and GBs of RAM, just edit the label to match your needs, for etcd, the team asked for 8vCPU and 32GB of RAM, so they used runs-on: actuated-arm64-8cpu-32gb.

\n

I had to split the docker build so that the ARM half would build on ARM, and x86 on x86, and then a step to combine the two - overall this works out to be a very significant improvementhttps://t.co/69cIxjYRcW

— Ellie Huxtable (@ellie_huxtable) October 20, 2023
\n

We have full instructions for 2, in the following tutorial: How to split up multi-arch Docker builds to run natively.

\n

Is there access for AMD64?

\n

This program is limited to CNCF projects and Arm CI only. That said, most actuated customers run AMD64 builds with us.

\n

GitHub already provides access to AMD64 runners for free for open source projects, that should cover most OSS project's needs.

\n

So why would you want dedicated AMD64 support from actuated? Firstly, our recommended provider makes builds up to 3x quicker, secondly, you can run on private repos if required, without accuring a large bill.

\n

What are all the combinations of CPU and RAM?

\n

We get this question very often, but have tried to be as clear as possible in this blog post and in the docs. There are no set combinations. You can come up with what you need.

\n

That helps us make best use of the hardware, you can even have just a couple of cores, and max out to 256GB of RAM, if that's what your build needs.

\n

What if the sponsored program is full?

\n

The program has been very popular and there is a limit to the budget and number of projects that Ampere and the CNCF agreed to pay for. If you contact us and we tell you the limit has been reached, then your employer could sponsor the subscription, and we'll give you a special discount - you could get started immediately. Or you'll need to contact Chris Aniszczyk and tell him why it would be of value to the OSS project you represent to have native Arm CI. If you get in touch with us, we can introduce you to him via email if needed.

\n

Learn more about actuated

\n

We're initially offering access to managed Arm CI for CNCF projects, but if you're working for a company that is experiencing friction with CI, please reach out to us to talk using this form.

\n

Ampere who are co-sponsoring our service with the CNCF have their own announcement here: Ampere Computing and CNCF Supporting Arm Native CI for CNCF Projects.

\n\n
\n

Did you know? Actuated for GitLab CI is now in technical preview, watch a demo here.

\n
","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/automate-packer-qemu-image-builds.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/automate-packer-qemu-image-builds.json new file mode 100644 index 00000000..e3d97f0f --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/automate-packer-qemu-image-builds.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","contentHtml":"

One of the most popular tools for creating images for virtual machines is Packer by Hashicorp. Packer automates the process of building images for a variety of platforms from a single source configuration. Different builders can be used to create machines and generate images from those machines.

\n

In this tutorial we will use the QEMU builder to create a KVM virtual machine image.

\n

We will see how the Packer build can be completely automated by integrating Packer into a continuous integration (CI) pipeline with GitHub Actions. The workflow will automatically trigger image builds on changes and publish the resulting images as GitHub release artifacts.

\n

Actuated supports nested virtualsation where a VM can make use of KVM to launch additional VMs within a GitHub Action. This makes it possible to run the Packer QEMU builder in GitHub Action workflows. Something that is not possible with GitHub's default hosted runners.

\n
\n

See also: How to run KVM guests in your GitHub Actions

\n
\n

Create the Packer template

\n

We will be starting from a Ubuntu Cloud Image and modify it to suit our needs. If you need total control of what goes into the image you can start from scratch using the ISO.

\n

Variables are used in the packer template to set the iso_url and iso_checksum. In addition to these we also use variables to configure the disk_size, ram, cpu, ssh_password and ssh_username:

\n
variable \"cpu\" {\n  type    = string\n  default = \"2\"\n}\n\nvariable \"disk_size\" {\n  type    = string\n  default = \"40000\"\n}\n\nvariable \"headless\" {\n  type    = string\n  default = \"true\"\n}\n\nvariable \"iso_checksum\" {\n  type    = string\n  default = \"sha256:d699ae158ec028db69fd850824ee6e14c073b02ad696b4efb8c59d37c8025aaa\"\n}\n\nvariable \"iso_url\" {\n  type    = string\n  default = \"https://cloud-images.ubuntu.com/jammy/20230719/jammy-server-cloudimg-amd64.img\"\n}\n\nvariable \"name\" {\n  type    = string\n  default = \"jammy\"\n}\n\nvariable \"ram\" {\n  type    = string\n  default = \"2048\"\n}\n\nvariable \"ssh_password\" {\n  type    = string\n  default = \"ubuntu\"\n}\n\nvariable \"ssh_username\" {\n  type    = string\n  default = \"ubuntu\"\n}\n\nvariable \"version\" {\n  type    = string\n  default = \"\"\n}\n\nvariable \"format\" {\n  type    = string\n  default = \"qcow2\"\n}\n
\n

The Packer source configuration:

\n
source \"qemu\" \"jammy\" {\n  accelerator      = \"kvm\"\n  boot_command     = []\n  disk_compression = true\n  disk_interface   = \"virtio\"\n  disk_image       = true\n  disk_size        = var.disk_size\n  format           = var.format\n  headless         = var.headless\n  iso_checksum     = var.iso_checksum\n  iso_url          = var.iso_url\n  net_device       = \"virtio-net\"\n  output_directory = \"artifacts/qemu/${var.name}${var.version}\"\n  qemuargs = [\n    [\"-m\", \"${var.ram}M\"],\n    [\"-smp\", \"${var.cpu}\"],\n    [\"-cdrom\", \"cidata.iso\"]\n  ]\n  communicator           = \"ssh\"\n  shutdown_command       = \"echo '${var.ssh_password}' | sudo -S shutdown -P now\"\n  ssh_password           = var.ssh_password\n  ssh_username           = var.ssh_username\n  ssh_timeout            = \"10m\"\n}\n
\n

Some notable settings in the source configuration:

\n\n

In the next section we will see how cloud-init is used to setup user account with the correct password that Packer needs for provisioning.

\n

The full example of the packer file is available on GitHub.

\n

Create the user-data file

\n

Cloud images provided by Canonical do not have users by default. The Ubuntu images use cloud-init to pre-configure the system during boot.

\n

Packer uses provisioners to install and configure the machine image after booting. To run these provisioners Packer needs to be able to communicate with the machine. By default this happens by establishing an ssh connection to the machine.

\n

Create a user-data file that sets the password of the default user so that it can be used by Packer to connect over ssh:

\n
#cloud-config\npassword: ubuntu\nssh_pwauth: true\nchpasswd:\n  expire: false\n
\n

Next create an ISO that can be referenced by our Packer template and presented to the VM:

\n
genisoimage -output cidata.iso -input-charset utf-8 -volid cidata -joliet -r \\\n
\n

The ISO can be mounted by QEMU to provide the configuration data to cloud-init while the VM boots.

\n

The -cdrom flag is used in the qemuargs field to mount the cidata.iso file:

\n
  qemuargs = [\n    [\"-m\", \"${var.ram}M\"],\n    [\"-smp\", \"${var.cpu}\"],\n    [\"-cdrom\", \"cidata.iso\"]\n  ]\n
\n

Provision the image

\n

The build section of the Packer template is used to define provisioners that can run scripts and commands to install software and configure the machine.

\n

In this example we are installing python3 but you can run any script you want or use tools like Ansible to automate the configuration.

\n
build {\n  sources = [\"source.qemu.jammy\"]\n\n  provisioner \"shell\" {\n    execute_command = \"{{ .Vars }} sudo -E bash '{{ .Path }}'\"\n    inline          = [\"sudo apt update\", \"sudo apt install python3\"]\n  }\n\n  post-processor \"shell-local\" {\n    environment_vars = [\"IMAGE_NAME=${var.name}\", \"IMAGE_VERSION=${var.version}\", \"IMAGE_FORMAT=${var.format}\"]\n    script           = \"scripts/prepare-image.sh\"\n  }\n}\n
\n

Prepare the image for publishing.

\n

Packer supports post-processors. They only run after Packer saves an instance as an image. Post-processors are commonly used to compress artifacts, upload them into a cloud, etc. See the Packer docs for more use-cases and examples.

\n

We will add a post processing step to the packer template to run the prepare-image.sh script. This script renames the image artifacts and calculates the shasum to prepare them to be uploaded as release artifacts on GitHub.

\n
  post-processor \"shell-local\" {\n    environment_vars = [\"IMAGE_NAME=${var.name}\", \"IMAGE_VERSION=${var.version}\", \"IMAGE_FORMAT=${var.format}\"]\n    script           = \"scripts/prepare-image.sh\"\n  }\n
\n

Launch the build locally

\n

If your local system is setup correctly, it has the packer binary and qemu installed, you can build with just:

\n
packer build .\n
\n

The artifacts folder will contain the resulting machine image and shasum file after the build completes.

\n
artifacts\n└── qemu\n    └── jammy\n        ├── jammy.qcow2\n        └── jammy.qcow2.sha256sum\n
\n

Automate image releases with GitHub Actions.

\n

For the QEMU builder to run at peak performance it requires hardware acceleration. This is not always possible in CI runners. GitHub's hosted runners do not support nested virtualization. With Actuated we added support for launching Virtual Machines in GitHub Action pipelines. This makes it possible to run the Packer QEMU builder in your workflows.

\n

Support for KVM is not enabled by default on Actuated and there are some prerequisites:

\n\n

To configure your Actuated Agent for KVM support follow the instructions in the docs.

\n

The GitHub actions workflow

\n

The default GitHub hosted runners come with Packer pre-installed. On self-hosted runners you will need a step to install the Packer binary. The official [setup-packer][https://github.com/hashicorp/setup-packer] action can be used for this.

\n

We set runs-on to actuated so that the build workflow will run on an Actuated runner:

\n
name: Build\n\non:\n  push:\n    tags: [\"v[0-9].[0-9]+.[0-9]+\"]\n    branches:\n      - \"main\"\njobs:\n  build-image:\n    name: Build\n    runs-on: actuated\n    ##...\n
\n

The build job runs the following steps:

\n
    \n
  1. \n

    Retrieve the Packer configuration by checking out the GitHub repository.

    \n
    - name: Checkout Repository\n  uses: actions/checkout@v3\n
    \n
    2. \n
  2. \n

    Install QEMU to ensure Packer is able to launch kvm/qemu virtual machines.

    \n
    - name: Install qemu\n  run: sudo apt-get update && sudo apt-get install qemu-system -y\n
    \n
    3. \n
  3. \n

    Setup packet to ensure the binary is available in the path.

    \n
    - name: Setup packer\n  uses: hashicorp/setup-packer@main\n
    \n
    4. \n
  4. \n

    Initialize the packer template and install all plugins referenced by the template.

    \n
    - name: Packer Init\n  run: packer init .\n
    \n
    5. \n
  5. \n

    Build the images defined in the root directory. Before we run the packer build command we make /dev/kvm world read-writable so that the QEMU builder can use it.

    \n
    - name: Packer Build\n  run: |\n    sudo chmod o+rw /dev/kvm\n    packer build .\n
    \n
    6. \n
  6. \n

    Upload the images as GitHub release artifacts. This job only runs for tagged commits.

    \n
    - name: Upload images and their SHA to Github Release\n  if: startsWith(github.ref, 'refs/tags/v')\n  uses: alexellis/upload-assets@0.4.0\n  env:\n    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n  with:\n    asset_paths: '[\"./artifacts/qemu/*/*\"]'\n
    \n
    7. \n
\n

Taking it further

\n

We created a GitHub actions workflow that can run a Packer build with QEMU to create a custom Ubuntu image. The resulting qcow2 image is automatically uploaded to the GitHub release assets on each release.

\n

The released image can be downloaded and used to spin up a VM instance on your private hardware or on different cloud providers.

\n

We exported the image in qcow2 format but you might need a different image format. The QEMU builder also supports outputting images in raw format. In our Packer template the output format can be changed by setting the format variable.

\n

Additional tools like the qemu disk image utility can also be used to convert images between different formats. A post-processor would be the ideal place for these kinds of extra processing steps.

\n

AWS also supports importing VM images and converting them to an AMI so they can be used to launch EC2 instances. See: Create an AMI from a VM image

\n

If you'd like to know more about nested virtualisation support, check out: How to run KVM guests in your GitHub Actions

","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/blazing-fast-ci-with-microvms.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/blazing-fast-ci-with-microvms.json new file mode 100644 index 00000000..59e8a492 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/blazing-fast-ci-with-microvms.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","contentHtml":"

Around 6-8 months ago I started exploring MicroVMs out of curiosity. Around the same time, I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.

\n

There's three parts to this post:

\n
    \n
  1. A quick debrief on Firecracker and MicroVMs vs legacy solutions
    2. \n
  2. Exploring friction with GitHub Actions from a hosted and self-hosted perspective
    3. \n
  3. Blazing fast CI with Actuated, and additional materials for learning more about Firecracker
    4. \n
\n
\n

We're looking for customers who want to solve the problems explored in this post.\nRegister for the pilot

\n
\n

1) A quick debrief on Firecracker 🔥

\n
\n

Firecracker is an open source virtualization technology that is purpose-built for creating and managing secure, multi-tenant container and function-based services.

\n
\n

I learned about Firecracker mostly by experimentation, building bigger and more useful prototypes. This helped me see what the experience was going to be like for users and the engineers working on a solution. I met others in the community and shared notes with them. Several people asked \"Are microVMs the next thing that will replace containers?\" I don't think they are, but they are an important tool where hard isolation is necessary.

\n

Over time, one thing became obvious:

\n
\n

MicroVMs fill a need that legacy VMs and containers can't.

\n
\n

If you'd like to know more about how Firecracker works and how it compares to traditional VMs and Docker, you can replay my deep dive session with Richard Case, Principal Engineer (previously Weaveworks, now at SUSE).

\n\n
\n

Join Alex and Richard Case for a cracking time. The pair share what's got them so excited about Firecracker, the kinds of use-cases they see for microVMs, fundamentals of Linux Operating Systems and plenty of demos.

\n
\n

2) So what's wrong with GitHub Actions?

\n

First let me say that I think GitHub Actions is a far better experience than Travis ever was, and we have moved all our CI for OpenFaaS, inlets and actuated to Actions for public and private repos. We've built up a good working knowledge in the community and the company.

\n

I'll split this part into two halves.

\n

What's wrong with hosted runners?

\n

Hosted runners are constrained

\n

Hosted runners are incredibly convenient, and for most of us, that's all we'll ever need, especially for public repositories with fast CI builds.

\n

Friction starts when the 7GB of RAM and 2 cores allocated causes issues for us - like when we're launching a KinD cluster, or trying to run E2E tests and need more power. Running out of disk space is also a common problem when using Docker images.

\n

GitHub recently launched new paid plans to get faster runners, however the costs add up, the more you use them.

\n

What if you could pay a flat fee, or bring your own hardware?

\n

They cannot be used with public repos

\n

From GitHub.com:

\n
\n

We recommend that you only use self-hosted runners with private repositories. This is because forks of your public repository can potentially run dangerous code on your self-hosted runner machine by creating a pull request that executes the code in a workflow.

\n
\n
\n

This is not an issue with GitHub-hosted runners because each GitHub-hosted runner is always a clean isolated virtual machine, and it is destroyed at the end of the job execution.

\n
\n
\n

Untrusted workflows running on your self-hosted runner pose significant security risks for your machine and network environment, especially if your machine persists its environment between jobs.

\n
\n

Read more about the risks: Self-hosted runner security

\n

Despite a stern warning from GitHub, at least one notable CNCF project runs self-hosted ARM64 runners on public repositories.

\n

On one hand, I don't blame that team, they have no other option if they want to do open source, it means a public repo, which means risking everything knowingly.

\n

Is there another way we can help them?

\n

I spoke to the GitHub Actions engineering team, who told me that using an ephemeral VM and an immutable OS image would solve the concerns.

\n

There's no access to ARM runners

\n

Building with QEMU is incredibly slow as Frederic Branczyk, Co-founder, Polar Signals found out when his Parca project was taking 33m5s to build.

\n

I forked it and changed a line: runs-on: actuated-aarch64 and reduced the total build time to 1m26s.

\n

This morning @fredbrancz said that his ARM64 build was taking 33 minutes using QEMU in a GitHub Action and a hosted runner.

I ran it on @selfactuated using an ARM64 machine and a microVM.

That took the time down to 1m 26s!! About a 22x speed increase. https://t.co/zwF3j08vEV pic.twitter.com/ps21An7B9B

— Alex Ellis (@alexellisuk) October 20, 2022
\n

They limit maximum concurrency

\n

On the free plan, you can only launch 20 hosted runners at once, this increases as you pay GitHub more money.

\n

Builds on private repos are billed per minute

\n

I think this is a fair arrangement. GitHub donates Azure VMs to open source users or any public repo for that matter, and if you want to build closed-source software, you can do so by renting VMs per hour.

\n

There's a free allowance for free users, then Pro users like myself get a few more build minutes included. However, These are on the standard, 2 Core 7GB RAM machines.

\n

What if you didn't have to pay per minute of build time?

\n

What's wrong with self-hosted runners?

\n

It's challenging to get all the packages right as per a hosted runner

\n

I spent several days running and re-running builds to get all the software required on a self-hosted runner for the private repos for OpenFaaS Pro. Guess what?

\n

I didn't want to touch that machine again afterwards, and even if I built up a list of apt packages, it'd be wrong in a few weeks. I then had a long period of tweaking the odd missing package and generating random container image names to prevent Docker and KinD from conflicting and causing side-effects.

\n

What if we could get an image that had everything we needed and was always up to date, and we didn't have to maintain that?

\n

Self-hosted runners cause weird bugs due to caching

\n

If your job installs software like apt packages, the first run will be different from the second. The system is mutable, rather than immutable and the first problem I faced was things clashing like container names or KinD cluster names.

\n

You get limited to one job per machine at a time

\n

The default setup is for a self-hosted Actions Runner to only run one job at a time to avoid the issues I mentioned above.

\n

What if you could schedule as many builds as made sense for the amount of RAM and core the host has?

\n

Docker isn't isolated at all

\n

If you install Docker, then the runner can take over that machine since Docker runs at root on the host. If you try user-namespaces, many things break in weird and frustrating aways like Kubernetes.

\n

Container images and caches can cause conflicts between builds.

\n

Kubernetes isn't a safe alternative

\n

Adding a single large machine isn't a good option because of the dirty cache, weird stateful errors you can run into, and side-effects left over on the host.

\n

So what do teams do?

\n

They turn to a controller called Actions Runtime Controller (ARC).

\n

ARC is non trivial to set up and requires you to create a GitHub App or PAT (please don't do that), then to provision, monitor, maintain and upgrade a bunch of infrastructure.

\n

This controller starts a number of re-usable (not one-shot) Pods and has them register as a runner for your jobs. Unfortunately, they still need to use Docker or need to run Kubernetes which leads us to two awful options:

\n
    \n
  1. Sharing a Docker Socket (easy to become root on the host)
    2. \n
  2. Running Docker In Docker (requires a privileged container, root on the host)
    3. \n
\n

There is a third option which is to use a non-root container, but that means you can't use sudo in your builds. You've now crippled your CI.

\n

What if you don't need to use Docker build/run, Kaniko or Kubernetes in CI at all? Well ARC may be a good solution for you, until the day you do need to ship a container image.

\n

3) Can we fix it? Yes we can.

\n

Actuated (\"cause (a machine or device) to operate.\") is a semi-managed solution that we're building at OpenFaaS Ltd.

\n

\"A

\n
\n

A semi-managed solution, where you provide hosts and we do the rest.

\n
\n

You provide your own hosts to run jobs, we schedule to them and maintain a VM image with everything you need.

\n

You install our GitHub App, then change runs-on: ubuntu-latest to runs-on: actuated or runs-on: actuated-aarch64 for ARM.

\n

Then, provision one or more VMs with nested virtualisation enabled on GCP, DigitalOcean or Azure, or a bare-metal host, and install our agent. That's it.

\n

If you need ARM support for your project, the a1.metal from AWS is ideal with 16 cores and 32GB RAM, or an Ampere Altra machine like the c3.large.arm64 from Equinix Metal with 80 Cores and 256GB RAM if you really need to push things. The 2020 M1 Mac Mini also works well with Asahi Linux, and can be maxed out at 16GB RAM / 8 Cores. I even tried Frederic's Parca job on my Raspberry Pi and it was 26m30s quicker than a hosted runner!

\n

Whenever a build is triggered by a repo in your organisation, the control plane will schedule a microVM on one of your own servers, then GitHub takes over from there. When the GitHub runner exits, we forcibly delete the VM.

\n

You get:

\n\n

It's capable of running Docker and Kubernetes (KinD, kubeadm, K3s) with full isolation. You'll find some examples in the docs, but anything that works on a hosted runner we expect to work with actuated also.

\n

Here's what it looks like:

\n\n

Want the deeply technical information and comparisons? Check out the FAQ

\n

You may also be interested in a debug experience that we're building for GitHub Actions. It can be used to launch a shell session over SSH with hosted and self-hosted runners: Debug GitHub Actions with SSH and launch a cloud shell

\n

Wrapping up

\n

We're piloting actuated with customers today. If you're interested in faster, more isolated CI without compromising on security, we would like to hear from you.

\n

Register for the pilot

\n

We're looking for customers to participate in our pilot.

\n

Register for the pilot 📝

\n

Actuated is live in pilot and we've already run thousands of VMs for our customers, but we're only just getting started here.

\n

\"VM

\n
\n

Pictured: VM launch events over the past several days

\n
\n

Other links:

\n\n

What about GitLab?

\n

We're focusing on GitHub Actions users for the pilot, but have a prototype for GitLab. If you'd like to know more, reach out using the Apply for the pilot form.

\n

Just want to play with Firecracker or learn more about microVMs vs legacy VMs and containers?

\n\n

What are people saying about actuated?

\n
\n

\"We've been piloting Actuated recently. It only took 30s create 5x isolated VMs, run the jobs and tear them down again inside our on-prem environment (no Docker socket mounting shenanigans)! Pretty impressive stuff.\"

\n

Addison van den Hoeven - DevOps Lead, Riskfuel

\n
\n
\n

\"Actuated looks super cool, interested to see where you take it!\"

\n

Guillermo Rauch, CEO Vercel

\n
\n
\n

\"This is great, perfect for jobs that take forever on normal GitHub runners. I love what Alex is doing here.\"

\n

Richard Case, Principal Engineer, SUSE

\n
\n
\n

\"Thank you. I think actuated is amazing.\"

\n

Alan Sill, NSF Cloud and Autonomic Computing (CAC) Industry-University Cooperative Research Center

\n
\n
\n

\"Nice work, security aspects alone with shared/stale envs on self-hosted runners.\"

\n

Matt Johnson, Palo Alto Networks

\n
\n
\n

\"Is there a way to pay github for runners that suck less?\"

\n

Darren Shepherd, Acorn Labs

\n
\n
\n

\"Excited to try out actuated! We use custom actions runners and I think there's something here 🔥\"

\n

Nick Gerace, System Initiative

\n
\n
\n

It is awesome to see the work of Alex Ellis with Firecracker VMs. They are provisioning and running Github Actions in isolated VMs in seconds (vs minutes).\"

\n

Rinat Abdullin, ML & Innovation at Trustbit

\n
\n
\n

This is awesome!\" (After reducing Parca build time from 33.5 minutes to 1 minute 26s)

\n

Frederic Branczyk, Co-founder, Polar Signals

\n
","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/burst-billing-capacity.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/burst-billing-capacity.json similarity index 71% rename from _next/data/qP6XrePfh_ktdNhbSnok_/blog/burst-billing-capacity.json rename to _next/data/j64IiYEEq_eJG_mFKDqPk/blog/burst-billing-capacity.json index d2017187..d0321ae2 100644 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/burst-billing-capacity.json +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/burst-billing-capacity.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","contentHtml":"

Why did we make Actuated? Actuated provides a securely isolated, managed, white-glove experience for customers who want to run CI/CD on GitHub Actions, with access to fast and private hardware. It's ideal for moving your organisation off Jenkins, customers who spend far too much on hosted runners, or those who cringe at the security implications of building with Docker using Kubernetes.

\n

Today, we're introducing two new features for actuated customers: burst billing and burst capacity (on shared servers). With burst billing, you can go over the concurrency limit of your plan for the day, and pay for the extra usage at a slightly higher rate. With burst capacity, you can opt into running more jobs than your current pool of servers allow for by using our hardware.

\n

Why we charge on concurrency, not minutes

\n

Having run over 320k VMs for customer CI jobs on GitHub Actions, we've seen a lot of different workloads and usage patterns. Some teams have a constant stream of hundreds of jobs per hour due to the use of matrix builds, some have a plan that's a little to big for them, and others have a plan that's a little to small, so they get the odd delay whilst they wait for jobs to finish.

\n

We decided to charge customers based not upon how many jobs they launched, how many minutes they consumed, but on the maximum amount of jobs they wanted to run at any one time (concurrency). Since customers already bring their own hardware, and pay per minute, hour or month for it, we didn't want them to have to pay again per minute and to be limited by how many jobs they could run per month.

\n

For high usage customers, this is a great deal. You get to run unlimited minutes, and in one case we had a customer who consumed 100k minutes within one week. With GitHub's current pricing that would have cost them 3,200 USD per week, or 12,800 USD per month. So you can see how the actuated plans, based upon concurrency alone are a great deal cheaper here.

\n

What if your plan is too small?

\n

So let's take a team that has the 5 concurrent build plan, but their server can accommodate 10 builds at once. What happens there?

\n

If 8 builds are queued, 5 will be scheduled, and the other 3 will remain in a queue. Once one of the first 5 completes, one of those 3 pending will get scheduled, and so forth, until all the work is done.

\n

\"Concurrency

\n
\n

The plan size is 5, but there are 8 jobs in the queue. The first 5 are scheduled, and the other 3 are pending.

\n
\n

Prior to today, the team would have only had two options: stay on the 5 concurrent build plan, and just accept that sometimes they'll have to wait for builds to complete, or upgrade to the 10 concurrent build plan, and have that extra capacity available to them whenever it's needed.

\n

CI usage can be unpredictable

\n

In the world of Kubernetes and autoscaling Pods, it might seem counterintuitive to plan out your capacity, but what we've seen is that when it comes to CI/CD, the customers we've worked with so far have very predictable usage patterns and can help them right-size their servers and plans.

\n

As part of our white-glove service, we monitor the usage of customers to see when they're hitting their limits, or encountering delays. We'll also let them know if they are under or overutilising their servers based upon free RAM and CPU usage.

\n

Then, there are two additional free tools we offer for self-service:

\n\n

Extra billing on your own servers

\n

If you're on a plan that has 20 concurrent builds, and it's \"Release day\", which means you ideally need 40 concurrent builds, but only for that day, it doesn't make sense to upgrade to a 40 concurrent build plan for the whole month. So we've introduced billing for burst billing where you pay extra for the extra concurrency you use, but only for the days in which you use it.

\n

This is a great way to get the extra capacity you need, without having to pay for it all the time, you'll need to have the extra capacity available on your own servers, but we can help you set that up.

\n

In the below diagram, the customer is on a 5 concurrent build plan, but is bursting to 8 builds on his own servers.

\n

\"Burst

\n
\n

Make use of excess capacity on your own servers for a day, without increasing your plan size for the whole month.

\n
\n

Extra concurrency on our servers

\n

At no extra cost, we're offering burst capacity for customers who need it onto hardware which we run. This is turned off by default, so you'll have to ask us to access, but once it's available, you'll be able to run more jobs than your servers allow for. When you use our servers, you'll be billed for burst concurrency, which is slightly higher than the normal rate.

\n

Below, the customer has used up the capacity of her own servers and is now bursting onto our servers for 2x x86_64 and 2x Arm builds.

\n

\"Use

\n
\n

Make use of our servers to run more jobs than your servers allow for.

\n
\n

Wrapping up

\n

If you'd like to try burst billing on your own servers, where you pay a little more on the days where you need to go over your plan, or if you'd like to use our server capacity to be able to keep the number of servers you run down, then please get in touch with us via the actuated Slack.

\n

How much extra is burst billing?

\n

Burst billing when used for spiky workloads is far cheaper than upgrading your plan for the whole month. The launch rate is 40%, however this may be subject to change.

\n

What if we go over the plan every day?

\n

If you go over the plan every day, or most days, then it may be cheaper to upgrade to the next plan size. For instance, if the average daily usage is 14 builds, and your plan size is 10, you should save money by adopting the 15 build plan.

\n

That said, if your plan size is 10, and the monthly average is 12, you'll be better off with burst billing.

\n

Can we set a limit on burst billing?

\n

Burst billing is off by default. You can set an upper bound on the amount by telling us how high you want to go.

\n

How do we get burst capacity with actuated servers?

\n

Tell us the burst limit you'd like to enable, then we can enable x86_64 or Arm64 server capacity on your account.

\n

When is burst capacity used?

\n

Burst capacity is used when a job cannot be scheduled onto your own server capacity, and there is available capacity on our servers. Another way to think about burst capacity is as using on shared servers. Each job runs in its own microVM, which provides hard isolation.

\n

You may also like:

\n","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","contentHtml":"

Why did we make Actuated? Actuated provides a securely isolated, managed, white-glove experience for customers who want to run CI/CD on GitHub Actions, with access to fast and private hardware. It's ideal for moving your organisation off Jenkins, customers who spend far too much on hosted runners, or those who cringe at the security implications of building with Docker using Kubernetes.

\n

Today, we're introducing two new features for actuated customers: burst billing and burst capacity (on shared servers). With burst billing, you can go over the concurrency limit of your plan for the day, and pay for the extra usage at a slightly higher rate. With burst capacity, you can opt into running more jobs than your current pool of servers allow for by using our hardware.

\n

Why we charge on concurrency, not minutes

\n

Having run over 320k VMs for customer CI jobs on GitHub Actions, we've seen a lot of different workloads and usage patterns. Some teams have a constant stream of hundreds of jobs per hour due to the use of matrix builds, some have a plan that's a little to big for them, and others have a plan that's a little to small, so they get the odd delay whilst they wait for jobs to finish.

\n

We decided to charge customers based not upon how many jobs they launched, how many minutes they consumed, but on the maximum amount of jobs they wanted to run at any one time (concurrency). Since customers already bring their own hardware, and pay per minute, hour or month for it, we didn't want them to have to pay again per minute and to be limited by how many jobs they could run per month.

\n

For high usage customers, this is a great deal. You get to run unlimited minutes, and in one case we had a customer who consumed 100k minutes within one week. With GitHub's current pricing that would have cost them 3,200 USD per week, or 12,800 USD per month. So you can see how the actuated plans, based upon concurrency alone are a great deal cheaper here.

\n

What if your plan is too small?

\n

So let's take a team that has the 5 concurrent build plan, but their server can accommodate 10 builds at once. What happens there?

\n

If 8 builds are queued, 5 will be scheduled, and the other 3 will remain in a queue. Once one of the first 5 completes, one of those 3 pending will get scheduled, and so forth, until all the work is done.

\n

\"Concurrency

\n
\n

The plan size is 5, but there are 8 jobs in the queue. The first 5 are scheduled, and the other 3 are pending.

\n
\n

Prior to today, the team would have only had two options: stay on the 5 concurrent build plan, and just accept that sometimes they'll have to wait for builds to complete, or upgrade to the 10 concurrent build plan, and have that extra capacity available to them whenever it's needed.

\n

CI usage can be unpredictable

\n

In the world of Kubernetes and autoscaling Pods, it might seem counterintuitive to plan out your capacity, but what we've seen is that when it comes to CI/CD, the customers we've worked with so far have very predictable usage patterns and can help them right-size their servers and plans.

\n

As part of our white-glove service, we monitor the usage of customers to see when they're hitting their limits, or encountering delays. We'll also let them know if they are under or overutilising their servers based upon free RAM and CPU usage.

\n

Then, there are two additional free tools we offer for self-service:

\n\n

Extra billing on your own servers

\n

If you're on a plan that has 20 concurrent builds, and it's \"Release day\", which means you ideally need 40 concurrent builds, but only for that day, it doesn't make sense to upgrade to a 40 concurrent build plan for the whole month. So we've introduced billing for burst billing where you pay extra for the extra concurrency you use, but only for the days in which you use it.

\n

This is a great way to get the extra capacity you need, without having to pay for it all the time, you'll need to have the extra capacity available on your own servers, but we can help you set that up.

\n

In the below diagram, the customer is on a 5 concurrent build plan, but is bursting to 8 builds on his own servers.

\n

\"Burst

\n
\n

Make use of excess capacity on your own servers for a day, without increasing your plan size for the whole month.

\n
\n

Extra concurrency on our servers

\n

At no extra cost, we're offering burst capacity for customers who need it onto hardware which we run. This is turned off by default, so you'll have to ask us to access, but once it's available, you'll be able to run more jobs than your servers allow for. When you use our servers, you'll be billed for burst concurrency, which is slightly higher than the normal rate.

\n

Below, the customer has used up the capacity of her own servers and is now bursting onto our servers for 2x x86_64 and 2x Arm builds.

\n

\"Use

\n
\n

Make use of our servers to run more jobs than your servers allow for.

\n
\n

Wrapping up

\n

If you'd like to try burst billing on your own servers, where you pay a little more on the days where you need to go over your plan, or if you'd like to use our server capacity to be able to keep the number of servers you run down, then please get in touch with us via the actuated Slack.

\n

How much extra is burst billing?

\n

Burst billing when used for spiky workloads is far cheaper than upgrading your plan for the whole month. The launch rate is 40%, however this may be subject to change.

\n

What if we go over the plan every day?

\n

If you go over the plan every day, or most days, then it may be cheaper to upgrade to the next plan size. For instance, if the average daily usage is 14 builds, and your plan size is 10, you should save money by adopting the 15 build plan.

\n

That said, if your plan size is 10, and the monthly average is 12, you'll be better off with burst billing.

\n

Can we set a limit on burst billing?

\n

Burst billing is off by default. You can set an upper bound on the amount by telling us how high you want to go.

\n

How do we get burst capacity with actuated servers?

\n

Tell us the burst limit you'd like to enable, then we can enable x86_64 or Arm64 server capacity on your account.

\n

When is burst capacity used?

\n

Burst capacity is used when a job cannot be scheduled onto your own server capacity, and there is available capacity on our servers. Another way to think about burst capacity is as using on shared servers. Each job runs in its own microVM, which provides hard isolation.

\n

You may also like:

\n","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/caching-in-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/caching-in-github-actions.json new file mode 100644 index 00000000..0edcbb9d --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/caching-in-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","contentHtml":"

GitHub provides a cache action that allows caching dependencies and build outputs to improve workflow execution time.

\n

A common use case would be to cache packages and dependencies from tools such as npm, pip, Gradle, ... . If you are using Go, caching go modules and the build cache can save you a significant amount of build time as we will see in the next section.

\n

Caching can be configured manually, but a lot of setup actions already use the actions/cache under the hood and provide a configuration option to enable caching.

\n

We use the actions cache to speed up workflows for building the Actuated base images. As part of those workflows we build a kernel and then a rootfs. Since the kernel’s configuration is changed infrequently it makes sense to cache that output.

\n

\"Build

\n
\n

Comparing workflow execution times with and without caching.

\n
\n

Building the kernel takes around 1m20s on our aarch-64 Actuated runner and 4m10s for the x86-64 build so we get some significant time improvements by caching the kernel.

\n

The output of the cache action can also be used to do something based on whether there was a cache hit or miss. We use this to skip the kernel publishing step when there was a cache hit.

\n
- if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}\n  name: Publish Kernel\n  run: make publish-kernel-x86-64\n
\n

Caching Go dependency files and build outputs

\n

In this minimal example we are going to setup caching for Go dependency files and build outputs. As an example we will be building alexellis/registry-creds. This is a Kubernetes operator that can be used to replicate Kubernetes ImagePullSecrets to all namespaces.

\n

It has the K8s API as a dependency which is quite large so we expect to save some time by cashing the Go mod download. By also caching the Go build cache it should be possible to speed up the workflow even more.

\n

Configure caching manually

\n

We will first create the workflow and run it without any caching.

\n
name: ci\n\non: push\n\njobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: \"alexellis/registry-creds\"\n      - name: Setup Golang\n        uses: actions/setup-go@v3\n        with:\n          go-version: ~1.19\n      - name: Build\n        run: |\n          CGO_ENABLED=0 GO111MODULE=on \\\n          go build -ldflags \"-s -w -X main.Release=dev -X main.SHA=dev\" -o controller\n
\n

The checkout action is used to check out the registry-creds repo so the workflow can access it. The next step sets up Go using the setup-go action and as a last step we run go build.

\n

\"No

\n

When triggering this workflow we see that each run takes around 1m20s.

\n

Modify the workflow and add an additional step to configure the caches using the cache action:

\n
steps:\n  - name: Setup Golang\n    uses: actions/setup-go@v3\n    with:\n      go-version: ~1.19\n  - name: Setup Golang caches\n    uses: actions/cache@v3\n    with:\n      path: |\n        ~/.cache/go-build\n        ~/go/pkg/mod\n      key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}\n      restore-keys: |\n        ${{ runner.os }}-golang-\n
\n

The path parameter is used to set the paths on the runner to cache or restore. The key parameter sets the key used when saving the cache. A hash of the go.sum file is used as part of the cache key.

\n

Optionally the restore-keys are used to find and restore a cache if there was no hit for the key. In this case we always restore the cache even if there was no specific hit for the go.sum file.

\n

The first time this workflow is run the cache is not populated so we see a similar execution time as without any cache of around 1m20s.

\n

\"Comparing

\n

Running the workflow again we can see that it now completes in just 18s.

\n

Use setup-go built-in caching

\n

The V3 edition of the setup-go action has support for caching built-in. Under the hood it also uses the actions/cache with a similar configuration as in the example above.

\n

The advantage of using the built-in functionality is that it requires less configuration settings. Caching can be enabled by adding a single line to the workflow configuration:

\n
name: ci\n\non: push\n\njobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: \"alexellis/registry-creds\"\n      - name: Setup Golang\n        uses: actions/setup-go@v3\n        with:\n          go-version: ~1.19\n+         cache: true\n      - name: Build\n        run: |\n          CGO_ENABLED=0 GO111MODULE=on \\\n          go build -ldflags \"-s -w -X main.Release=dev -X main.SHA=dev\" -o controller\n
\n

Triggering the workflow with the build-in cache yields similar time gains as with the manual cache configuration.

\n

Conclusion

\n

We walked you through a short example to show you how to set up caching for a Go project and managed to build the project 4x faster.

\n

If you are building with Docker you can use Docker layer caching to make your builds faster. Buildkit automatically caches the build results and allows exporting the cache to an external location. It has support for uploading the build cache to GitHub Actions cache

\n

See also: GitHub: Caching dependencies in Workflows

\n

Keep in mind that there are some limitations to the GitHub Actions cache. Cache entries that have not been accessed in over 7 days will be removed. There is also a limit on the total cache size of 10 GB per repository.

\n

Some points to take away:

\n\n
\n

Want to learn more about Go and GitHub Actions?

\n

Alex's eBook Everyday Golang has a chapter dedicated to building Go programs with Docker and GitHub Actions.

\n
","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/calyptia-case-study-arm.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/calyptia-case-study-arm.json new file mode 100644 index 00000000..c4ef4221 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/calyptia-case-study-arm.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","contentHtml":"

This is a case-study, and guest article by Patrick Stephens, Tech Lead of Infrastructure at Calyptia.

\n

Introduction

\n

Different architecture builds can be slow using the Github Actions hosted runners due to emulation of the non-native architecture for the build. This blog shows a simple way to make use of self-hosted runners for dedicated builds but in a secure and easy to maintain fashion.

\n

Calyptia maintains the OSS and Cloud Native Computing Foundation (CNCF) graduated Fluent projects including Fluent Bit. We then add value to the open-source core by providing commercial services and enterprise-level features.

\n
\n

Fluent Bit is a Fast and Lightweight Telemetry Agent for Logs, Metrics, and Traces for Linux, macOS, Windows, and BSD family operating systems. It has been made with a strong focus on performance to allow the collection and processing of telemetry data from different sources without complexity.

\n

It was originally created by Eduardo Silva and is now an independent project.

\n
\n

To learn about Fluent Bit, the Open Source telemetry agent that Calyptia maintains, check out their docs.

\n

The Problem

\n

One of the best things about Fluent Bit is that we provide native packages (RPMs and DEBs) for a myriad of supported targets (various Linux, macOS and Windows), however to do this is also one of the hardest things to support due to the complexity of building and testing across all these targets.

\n

When PRs are provided we would like to ensure they function across the targets but doing so can take a very long time (hours) and consume a lot of resources (that must be paid for). This means that these long running jobs are only done via exception (manually labelling a PR or on full builds for releases) leading to issues only discovered when a full build & test is done, e.g. during the release process so blocking the release until it is fixed.

\n

The long build time problem came to a head when we discovered we could no longer build for Amazon Linux 2023 (AL2023) because the build time exceeded the 6 hour limit for a single job on Github. We had to disable the AL2023 target for releases which means users cannot then update to the latest release leading to missing features or security problems: See the issue here

\n

In addition to challenges in the OSS, there are also challenges on the commercial side. Here, we are seeing issues with extended build times for ARM64 targets because our CI is based on Github Actions and currently only AMD64 (also called x86-64 or x64) runners are provided for builds. This slows down development and can mean bugs are not caught as early as possible.

\n

Why not use self-hosted runners?

\n

One way to speed up builds is to provide self-hosted ARM64 runners.

\n

Unfortunately, runners pose security implications, particularly for public repositories. In fact, Github recommends against using self-hosted runners: About self-hosted runners - GitHub Docs

\n

In addition to security concerns, there are also infrastructure implications for using self-hosted runners. We have to provide the infrastructure around deploying and managing the self-hosted runners, installing an agent, configuring it for jobs, etc. From a perspective of OSS we want anything we do to be simple and easy for maintenance purposes.

\n

Any change we make needs to be compatible with downstream forks as well. We do not want to break builds for existing users, particularly for those who are contributors as well to the open source project. Therefore we need a solution that does not impact them.

\n

There are various tools that can help with managing self-hosted runners, https://jonico.github.io/awesome-runners/ provides a good curated list. I performed an evaluation of some of the recommended tools but the solution would be non-trivial and require some effort to maintain.

\n

Our considerations

\n

We have the following high level goals in a rough priority order:

\n\n

The solution

\n

At Kubecon EU 2023 I met up with Alex Ellis from Actuated (and of OpenFaaS fame) in-person and we wanted to put Alex and his technology to the test, to see if the Actuated technology could fix the problems we see with our build process.

\n

To understand what Actuated is then it is best to refer to their documentation with this specific blog post being a good overview of why we considered adopting it. We're not the only CNCF project that Alex's team was able to help. He describes how he helped Parca and Network Service Mesh to slash their build teams by using native Arm hardware.

\n

A quick TLDR; though would be that Actuated provides an agent you install which then automatically creates ephemeral VMs on the host for each build job. Actuated seemed to tick the various boxes (see the considerations above) we had for it but never trust a vendor until you’ve tried it yourself!

\n

Quote from Alex:

\n
\n

\"Actuated aims to give teams the closest possible experience to managed runners, but with native arm support flat rate billing, and secure VM-level isolation. Since Calyptia adopted actuated, we’ve also shipped an SSH debug experience (like you’d find with CircleCI) and detailed reports and insights on usage across repos, users and organisations.\"

\n
\n

To use Actuated, you have to provision a machine with the Actuated agent, which is trivial and well documented: https://docs.actuated.com/install-agent/.

\n

We deployed an Ampere Altra Q80 server with 256GB of RAM and 80 cores ARM64 machine via Equinix (Equinix donates resources to the CNCF which we use for Fluent Bit so this satisfies the cost side of things) and installed the Actuated agent on it per the Actuated docs.

\n

The update required to start using Actuated in OSS Fluent Bit is a one-liner. (Thanks in part to my excellent work refactoring the CI workflows, or so I like to think. You can see the actual PR here for the change: https://github.com/fluent/fluent-bit/pull/7527.)

\n

The following is the code required to start using Actuated:

\n
-    runs-on: ubuntu-latest\n+    runs-on: ${{ (contains(matrix.distro, 'arm' ) & 'actuated-arm64') || 'ubuntu-latest' }}\n
\n

For most people, the change will be much simpler:

\n
-    runs-on: ubuntu-latest\n+    runs-on: actuated\n
\n

In Github Actions parlance, the code above translates to “if we are doing an ARM build, then use the Actuated runner; otherwise, use the default Github Hosted (AMD64) Ubuntu runner”.

\n

In the real code, I added an extra check so that we only use Actuated runners for the official source repo which means any forks will also carry on running as before on the Github Hosted runner.

\n

With this very simple change, all the ARM64 builds that used to take hours to complete now finish in minutes. In addition, we can actually build the AL2023 ARM64 target to satisfy those users too. A simple change gave us a massive boost to performance and also provided a missing target.

\n

To demonstrate this is not specific to Equinix hosts or in some fashion difficult to manage in heterogeneous infrastructure (e.g. various hosts/VMs from different providers), we also replicated this for all our commercial offerings using a bare-metal Hetzner host. The process was identical: install the agent and make the runs-on code change as above to use Actuated. Massive improvements in build time were seen again as expected.

\n

The usage of bare-metal (or cloud) hosts providers is invisible and only a choice of which provider you want to put the agent on. In our case we have a mixed set up with no difference in usage or maintenance.

\n

Challenges building containers

\n

The native package (RPM/DEB) building described above was quite simple to integrate via the existing workflows we had.

\n

Building the native packages is done via a process that runs a target-specific container for each of the builds, e.g. we run a CentOS container to build for that target. This allows a complete build to be run on any Linux-compatible machine with a container runtime either in CI or locally. For ARM builds, we were using QEMU emulation for ARM builds hence the slowdown as this has to emulate instructions between architectures.

\n

Container builds are the primary commercial area for improvement as we provide a SAAS solution running on K8S. Container builds were also a trickier proposition for OSS as we were using a single job to build all architectures using the docker/build-push-action. The builds were incredibly slow for ARM and also atomic, which means if you received a transient issue in one of the architecture builds, you would have to repeat the whole lot.

\n

As an example: https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-images.yaml

\n
      - name: Build the production images\n        id: build_push\n        uses: docker/build-push-action@v4\n        with:\n          file: ./dockerfiles/Dockerfile\n          context: .\n          tags: ${{ steps.meta.outputs.tags }}\n          labels: ${{ steps.meta.outputs.labels }}\n          platforms: linux/amd64, linux/arm64, linux/arm/v7\n          target: production\n          # Must be disabled to provide legacy format images from the registry\n          provenance: false\n          push: true\n          load: false\n          build-args: |\n            FLB_NIGHTLY_BUILD=${{ inputs.unstable }}\n            RELEASE_VERSION=${{ inputs.version }}\n
\n

The build step above is a bit more complex to tease out into separate components: we need to run single architecture builds for each target then provide a multi-arch manifest that links them together.

\n

We reached out to Alex on a good way to modify this to work within a split build per architecture approach. The Actuated team has been very responsive on these types of questions along with proactive monitoring of our build queue and runners.

\n

Within Calyptia we have followed the approach Docker provided here and suggested by the Actuated team: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners

\n

Based on what we learned, we recommend the following process is followed:

\n

Build each architecture and push by digest in a set of parallel matrix jobs.\nCapture the output digest of each build.\nCreate the multi-arch manifest made up of each digest we have pushed in step 1 using the artefact from step 2.

\n

This approach provides two key benefits. First, it allows us to run on dedicated runners per-arch. Second, if a job fails we only need to repeat the single job, instead of having to rebuild all architectures.

\n

The new approach reduced the time for the release process for the Calyptia Core K8S Operator from more than an hour to minutes. Additionally, because we can do this so quickly, we now build all architectures for every change rather than just on release. This helps developers who are running ARM locally for development as they have containers always available.

\n

The example time speed up for the Calyptia Core K8S operator process was replicated across all the other components. A very good bang for your buck!

\n

For us, the actuated subscription fee has been of great value. Initially we tested the waters on the Basic Plan, but soon upgraded when we saw more areas where we could use it. The cost for us has been offset against a massive improvement in CI time and development time plus reducing the infrastructure costs of managing the self-hosted runners.

\n

Lessons learned

\n

The package updates were seamless really, however we did encounter some issues with the ecosystem (not with actuated), when refactoring and updating our container builds. The issues with the container builds are covered below to help anyone else with the same problems.

\n

Provenance is now enabled by default

\n

We were using v3 of Docker’s docker/build-push-action, but they made a breaking change which caused us a headache. They changed the default in v4 to create the various extra artifacts for provenance (e.g. SBOMs) which did have a few extra side effects both at the time and even now.

\n

If you do not disable this then it will push manifest lists rather than images so you will subsequently get an error message when you try to create a manifest list of another manifest list.

\n

Separately this also causes issues for older docker clients or organisations that need the legacy Docker schema format from a registry: using it means only OCI format schemas are pushed. This impacted both OSS and our commercial offerings: https://github.com/fluent/fluent-bit/issues/7748.

\n

It meant people on older OS’s or with requirements on only consuming Docker schema (e.g. maybe an internal mirror registry only supports that) could not pull the images.

\n

Invalid timestamps for gcr.io with manifests

\n

A funny problem found with our cloud-run deployments for Calyptia Core SAAS offering was that pushing the manifests to (Google Container Registry) gcr.io meant they ended up with a zero-epoch timestamp. This messed up some internal automation for us when we tried to get the latest version.

\n

To resolve this we just switched back to doing a single architecture build as we do not need multi-arch manifests for cloud-run. Internally we still have multi-arch images in ghcr.io for internal use anyway, this is purely the promotion to gcr.io.

\n

Manifests cannot use sub-paths

\n

This was a fun one: when specifying images to make up your manifest they must be in the same registry of course!

\n

Now, we tend to use sub-paths a lot to handle specific use cases for ghcr.io but unfortunately you cannot use them when trying to construct a manifest.

\n

OK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/product:tag-amd64\nNOK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/amd64/product:tag

\n

As with all good failures, the tooling let me make a broken manifest at build time but unfortunately trying to pull it meant a failure at runtime.

\n

Actuated container registry mirror

\n

All Github hosted runners provide default credentials to authenticate with docker.io for pulling public images. When running on a self-hosted runner you need to authenticate for this otherwise you will hit rate limits and builds may fail as they cannot download required base images.

\n

Actuated provide a registry mirror and Github Action to simplify this so make sure you set it up: https://docs.actuated.com/tasks/registry-mirror/

\n

As part of this, ensure it is set up for anything that uses images (e.g. we run integration tests on KIND that failed as the cluster could not download its images) and that it is done after any buildx config as it creates a dedicated buildx builder for the mirror usage.

\n

Actuated support

\n

The Actuated team helped us in two ways: the first was that we were able to enable Arm builds for our OSS projects and our commercial products, when they timed out with hosted runners. The second way was where our costs were getting out of hand on GitHub’s larger hosted runners: Actuated not only reduced the build time, but the billing model is flat-rate, meaning our costs are now fixed, rather than growing.

\n

As we made suggestions or collaborated with the Actuated team, they updated the documentation, including our suggestions on smoothing out the onboarding of new build servers and new features for the CLI.

\n

The more improvements we’ve made, the more we’ve seen. Next on our list is getting the runtime of a Go release down from 26 minutes by bringing it over to actuated.

\n

Conclusion

\n

Alex Ellis: We've learned a lot working with Patrick and Calyptia and are pleased to see that they were able to save money, whilst getting much quicker, and safer Open Source and commercial builds.

\n

We value getting feedback and suggestions from customers, and Patrick continues to provide plenty of them.

\n

If you'd like to learn more about actuated, reach out to speak to our team by clicking \"Sign-up\" and filling out the form. We'll be in touch to arrange a call.

","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/case-study-bring-your-own-bare-metal-to-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/case-study-bring-your-own-bare-metal-to-actions.json new file mode 100644 index 00000000..960c1cd9 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/case-study-bring-your-own-bare-metal-to-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","contentHtml":"

I'm going to show you how both a regular x86_64 build and an Arm build were made dramatically faster by using Bring Your Own (BYO) bare-metal servers.

\n

At the early stage of a project, GitHub's standard runners with 2x cores, 8GB RAM, and a little free disk space are perfect because they're free for public repos. For private repos they come in at a modest cost, if you keep your usage low.

\n

What's not to love?

\n

Well, Ed Warnicke, Distinguished Engineer at Cisco contacted me a few weeks ago and told me about the VPP project, and some of the problems he was running into trying to build it with hosted runners.

\n
\n

The Fast Data Project (FD.io) is an open-source project aimed at providing the world's fastest and most secure networking data plane through Vector Packet Processing (VPP).

\n
\n

Whilst VPP can be used as a stand-alone project, it is also a key component in the Cloud Computing Foundation's (CNCF's) Open Source Network Service Mesh project.

\n

There were two issues:

\n
    \n
  1. \n

    The x86_64 build was taking 1 hour 25 minutes on a standard runner.

    \n

    Why is that a problem? CI is meant to both validate against regression, but to build binaries for releases. If that process can take 50 minutes before failing, it's incredibly frustrating. For an open source project, it's actively hostile to contributors.

    \n
    2. \n
  2. \n

    The Arm build was hitting the 6 hour limit for GitHub Actions then failing

    \n

    Why? Well it was using QEMU, and I've spoken about this in the past - QEMU is a brilliant, zero cost way to build Arm binaries on a regular machine, but it's slow. And you'll see just how slow in the examples below, including where my Raspberry Pi beat a GitHub runner.

    \n
    3. \n
\n

We explain how to use QEMU in Docker Actions in the following blog post:

\n

The efficient way to publish multi-arch containers from GitHub Actions

\n

Rubbing some bare-metal on it

\n

So GitHub does actually have a beta going for \"larger runners\", and if Ed wanted to try that out, he'd have to apply to a beta waitlist, upgrade to a Team or Enterprise Plan, and then pick a new runner size.

\n

But that wouldn't have covered him for the Arm build, GitHub don't have any support there right now. I'm sure it will come one, day, but here we are unable to release binaries for our Arm users.

\n

With actuated, we have no interest in competing with GitHub's business model of selling compute on demand. We want to do something more unique than that - we want to enable you to bring your own (BYO) devices and then use them as runners, with VM-level isolation and one-shot runners.

\n
\n

What does Bring Your Own (BYO) mean?

\n

\"Your Own\" does not have to mean physical ownership. You do not need to own a datacenter, or to send off a dozen Mac Minis to a Colo.\nYou can provision bare-metal servers on AWS or with Equinix Metal as quickly as you can get an EC2 instance.\nActually, bare-metal isn't strictly needed at all, and even DigitalOcean's and Azure's VMs will work with actuated because they support KVM, which we use to launch Firecracker.

\n
\n

And who is behind actuated? We are a nimble team, but have a pedigree with Cloud Native and self-hosted software going back 6-7 years from OpenFaaS. OpenFaaS is a well known serverless platform which is used widely in production by commercial companies including Fortune 500s.

\n

Actuated uses a Bring Your Own (BYO) server model, but there's very little for you to do once you've installed the actuated agent.

\n

Here's how to set up the agent software: Actuated Docs: Install the Agent.

\n

You then get detailed stats about each runner, the build queue and insights across your whole GitHub organisation, in one place:

\n

Actuated now aggregates usage data at the organisation level, so you can get insights and spot changes in behaviour.

This peak of 57 jobs was when I was quashing CVEs for @openfaas Pro customers in Alpine Linux and a bunch of Go https://t.co/a84wLNYYjo… https://t.co/URaxgMoQGW pic.twitter.com/IuPQUjyiAY

— Alex Ellis (@alexellisuk) March 7, 2023
\n

First up - x86_64

\n

I forked Ed's repo into the \"actuated-samples\" repo, and edited the \"runs-on:\" field from \"ubuntu-latest\" to \"actuated\".

\n

The build which previously took 1 hour 25 minutes now took 18 minutes 58 seconds. That's a 4.4x improvement.

\n

\"Improvements

\n

4.4x doesn't sound like a big number, but look at the actual number.

\n

It used to take well over an hour to get feedback, now you get it in less than 20 minutes.

\n

And for context, this x86_64 build took 17 minutes to build on Ed's laptop, with some existing caches in place.

\n

I used an Equinix Metal m3.small.x86 server, which has 8x Intel Xeon E-2378G cores @ 2.8 GHz. It also comes with a local SSD, local NVMe would have been faster here.

\n

The Firecracker VM that was launched had 12GB of RAM and 8x vCPUs allocated.

\n

Next up - Arm

\n

For the Arm build I created a new branch and had to change a few hard-coded references from \"_amd64.deb\" to \"_arm64.deb\" and then I was able to run the build. This is common enablement work. I've been doing Arm enablement for Cloud Native and OSS since 2015, so I'm very used to spotting this kind of thing.

\n

So the build took 6 hours, and didn't even complete when running with QEMU.

\n

How long did it take on bare-metal? 14 minutes 28 seconds.

\n

\"Improvements

\n

That's a 25x improvement.

\n

The Firecracker VM that we launched had 16GB of RAM and 8x vCPUs allocated.

\n

It was running on a Mac Mini M1 configured with 16GB RAM, running with Asahi Linux. I bought it for development and testing, as a one-off cost, and it's a very fast machine.

\n

But, this case-study is not specifically about using consumer hardware, or hardware plugged in under your desk.

\n

Equinix Metal and Hetzner both have the Ampere Altra bare-metal server available on either an hourly or monthly basis, and AWS customers can get access to the a1.metal instance on an hourly basis too.

\n
\n

To prove the point, that BYO means cloud servers, just as much as physically owned machines, I also ran the same build on an Ampere Altra from Equinix Metal with 20 GB of RAM, and 32 vCPUs, it completed in 9 minutes 39 seconds.

\n
\n

See our hosting recommendations: Actuated Docs: Provision a Server

\n

In October last year, I benchmarked a Raspberry Pi 4 as an actuated server and pitted it directly against QEMU and GitHub's Hosted runners.

\n

It was 24 minutes faster. That's how bad using QEMU can be instead of using bare-metal Arm.

\n

Then, just for run I scheduled the MicroVM on my @Raspberry_Pi instead of an @equinixmetal machine.

Poor little thing has 8GB RAM and 4 Cores with an SSD connected over USB-C.

Anyway, it still beat QEMU by 24 minutes! pic.twitter.com/ITyRpbnwEE

— Alex Ellis (@alexellisuk) October 20, 2022
\n

Wrapping up

\n

So, wrapping up - if you only build x86_64, and have very few build minutes, and are willing to upgrade to a Team or Enterprise Plan on GitHub, \"faster runners\" may be an option you want to consider.

\n

If you don't want to worry about how many minutes you're going to use, or surprise bills because your team got more productive, or grew in size, or is finally running those 2 hour E2E tests every night, then actuated may be faster and better value overall for you.

\n

But if you need Arm runners, and want to use them with public repos, then there are not many options for you which are going to be secure and easy to manage.

\n

A recap on the results

\n

\"The

\n
\n

The improvement on the x86 build

\n
\n

You can see the builds here:

\n

x86_64 - 4.4x improvement

\n\n

Arm - 25x improvement

\n\n

Want to work with us?

\n

Want to get in touch with us and try out actuated for your team?

\n

We're looking for pilot customers who want to speed up their builds, or make self-hosted runners simpler to manager, and ultimately, about as secure as they're going to get with MicroVM isolation.

\n

Set up a 30 min call with me to ask any questions you may have and find out next steps.

\n\n

Learn more about how it compares to other solutions in the FAQ: Actuated FAQ

\n

See also:

\n","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/cncf-arm-march-update.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/cncf-arm-march-update.json new file mode 100644 index 00000000..b2dff00d --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/cncf-arm-march-update.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","contentHtml":"

It's now been 4 months since we kicked off the sponsored program with the Cloud Native Computing Foundation (CNCF) and Ampere to manage CI for the foundation's open source projects. But even before that, Calyptia, the maintainer of Fluent approached us to run Arm CI for the open source fluent repos, so we've been running CI for CNCF projects since June 2023.

\n

Over that time, we've got to work directly with some really bright, friendly, and helpful maintainers, who wanted to have a safe, fast and secure way to create release artifacts, test PRs, and to run end to end tests. Their alternative until this point was either to go against GitHub's own advice, and to run an unsafe, self-hosted runner on an open source repo, or to use QEMU that in the case of Fluent meant their 5 minute build took over 6 hours before failing.

\n

You can find out more about why we put this program together in the original announcement: Announcing managed Arm CI for CNCF projects

\n

Measuring the impact

\n

When we started out, Chris Aniszczyk, the CNCF's CTO wanted to create a small pilot to see if there'd be enough demand for our service. The CNCF partnered with Ampere to co-fund the program, Ampere sell a number of Arm based CPUs - which they brand as \"Cloud Native\" because they're so dense in cores and highly power efficient. Equinix Metal provide the credits and the hosting via the Cloud Native Credits program.

\n

In a few weeks, not only did we fill up all available slots, but we personally hand-held and onboarded each of the project maintainers one by one, over Zoom, via GitHub, and Slack.

\n

Why would maintainers of top-tier projects need our help? Our team and community has extensive experience porting code to Arm, and building for multiple CPUs. We were able to advise on best practices for splitting up builds, how to right-size VMs, were there to turn on esoteric Kernel modules and configurations, and to generally give them a running start.

\n

Today, our records show that the CNCF projects enrolled have run almost 400k minutes. That's almost the equivalent of a computer running tasks 24/7 for a total of 2 months solid, without a break.

\n

Here's a list of the organisations we've onboarded so far, ordered by the total amount of build minutes. We added the date of their first actuated build to help add some context. As I mentioned in the introduction, fluent have been a paying customer since June 2023.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationDate of first actuated build
1etcd-io (etcd, boltdb)2023-10-24
2fluent2023-06-07
3falcosecurity2023-12-06
4containerd2023-12-02
5cilium (tetragon, cilium, ebpf-go)2023-10-31
6cri-o2023-11-27
7open-telemetry2024-02-14
8opencontainers (runc)2023-12-15
9argoproj2024-01-30
\n

Ranked by build minutes consumed

\n

Some organisations have been actuated on multiple projects like etcd-io, with boltdb adding to their minutes, and cilium where tetragon and ebpf-go are also now running Arm builds.

\n

It's tempting to look at build minutes as the only metric, however, now that containerd, runc, cilium, etcd, and various other core projects are built by actuated, the security of the supply chain has become far more certain.

\n

From 10,000ft

\n

Here's what we aimed for and have managed to achieve in a very short period of time:

\n\n

Making efficient use of shared resources

\n

After fluent, etcd was the second project to migrate off self-managed runners. They had the impression that one of their jobs needed 32 vCPU and 32GB of RAM, and when we monitored the shared server pool, we noticed barely any load on the servers. That led me to build a quick Linux profiling tool called vmmeter. When they ran the profiler, it turned out the job used a maximum of 1.3 vCPU and 3GB of RAM, that's not just a rounding error - that's a night and day difference.

\n

You can learn how to try out vmmeter to right-size your jobs on actuated, or on GitHub's hosted runners.

\n

Right sizing VMs for GitHub Actions

\n

The projects have had a fairly stable, steady-state of CI jobs throughout the day and night as contributors from around the globe send PRs and end to end tests run.

\n

But with etcd-io in particular we started to notice on Monday or Tuesday that there was a surge of up to 200 jobs all at once. When we asked them about this, they told us Dependabot was the cause. It would send a number of PRs to bump dependencies and that would in turn trigger dozens of jobs.

\n

\"Thundering

\n
\n

Thundering herd problem from dependabot

\n
\n

It would clear itself down in time, but we spent a little time to automate adding in 1-2 extra servers for this period of the week, and we managed to get the queue cleared several times quicker. When the machines are no longer needed, they drain themselves and get deleted. This is important for efficient use of the CNCF's credits and Equinix Metal's fleet of Ampere Altra Q80 Arm servers.

\n

Giving insights to maintainers

\n

I got to meet up with Phil Estes from the containerd project at FOSDEM. We are old friends and used to be Docker Captains together.

\n

We looked at the daily usage stats, looked at the total amount of contributors that month and how many builds they'd had.

\n

\"Phil

\n

Then we opened up the organisation insights page and found that containerd had accounted for 14% of the total build minutes having only been onboarded in Dec 2023.

\n

\"Detailed

\n

We saw that there was a huge peak in jobs last month compared to this month, so he went off to the containerd Slack to ask about what had happened.

\n

Catching build time increases early

\n

Phil also showed me that he used to have a jimmy-rigged dashboard of his own to track build time increases, and at FOSDEM, my team did a mini hackathon to release our own way to show people their job time increases.

\n

We call it \"Job Outliers\" and it can be used to track increases going back as far as 120 days from today.

\n

\"Job

\n

Clicking \"inspect\" on any of the workflows will open up a separate plot link with deep links to the longest job seen on each day of that period of time.

\n

So what changed for our own actuated VM builds in that week, to add 5+ minutes of build time?

\n

\"Maximum

\n

We started building eBPF into the Kernel image, and the impact was 2x 2.5 minutes of build time.

\n

This feature was originally requested by Toolpath, a commercial user of actuated with very intensive Julia builds, and they have been using it to keep their build times in check. We're pleased to be able to offer every enhancement to the CNCF project maintainers too.

\n

Wrapping up

\n

What are the project maintainers saying?

\n

Antoine Toulme, maintainer of OpenTelemetry collectors:

\n
\n

The OpenTelemetry project has been looking for ways to test arm64 to support it as a top tier distribution. Actuated offers a path for us to test on new operating systems, especially arm64, without having to spend any time setting up or maintaining runners. We were lucky to be the recipient of a loan from Ampere that gave us access to a dedicated ARM server, and it took us months to navigate setting up dedicated runners and has significant maintenance overhead. With Actuated, we just set a tag in our actions and everything else is taken care of.

\n
\n

Luca Guerra, maintainer of Falco:

\n
\n

Falco users need to deploy to ARM64 as a platform, and we as maintainers, need to make sure that this architecture is treated as a first class citizen. Falco is a complex piece of software that employs kernel instrumentation and so it is not trivial to properly test. Thanks to Actuated, we were able to quickly add ARM64 to our GitHub Actions CI/CD pipeline making it much easier to maintain, freeing up engineering time from infrastructure work.

\n
\n

Sascha Grunert, maintainer of Cri-o:

\n
\n

The CRI-O project was able to seamlessly integrate Arm based CI with the support of Actuated. We basically had to convert our existing tests to a GitHub Actions matrix utilizing their powerful Arm runners. Integration and unit testing on Arm is another big step for CRI-O to provide a generally broader platform support. We also had to improve the test suite itself for better compatibility with other architectures than x86_64/arm64. This makes contributing to CRI-O on those platforms even simpler. I personally don’t see any better option than Actuated right now, because managing our own hardware is something we’d like to avoid to mainly focus on open source software development. The simplicity of the integration using Actuated helped us a lot, and our future goal is to extend the CRI-O test scenarios for that.

\n
\n

Summing up the program so far

\n

Through the sponsored program, actuated has now almost 400k build minutes for around 10 CNCF projects, and we've heard from a growing number of projects who would like access.

\n

We've secured the supply chain by removing unsafe runners that GitHub says should definitely not be used for open source repositories, and we've lessened the burden of server management on already busy maintainers.

\n

Whilst the original pilot program is now full, we have the capacity to onboard many other projects and would love to work with you. We are happy to offer a discounted subscription if your employer that sponsors your time on the said CNCF project will pay for it. Otherwise, contact us anyway, and we'll put you into email contact with Chris Aniszczyk so you can let him know how this would help you.

","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/custom-sizes-bpf-kvm.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/custom-sizes-bpf-kvm.json new file mode 100644 index 00000000..ced14e0f --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/custom-sizes-bpf-kvm.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","contentHtml":"

In this December update, we've got three new updates to the platform that we think you'll benefit from. From requesting custom vCPU and RAM per job, to eBPF features, to spreading your plan across multiple machines dynamically, it's all new and makes actuated better value.

\n

New eBPF support and 5.10.201 Kernel

\n

And as part of our work to provide hosted Arm CI for CNCF projects, including Tetragon and Cilium, we've now enabled eBPF and BTF features within the Kernel.

\n
\n

Berkley Packet Filter (BPF) is an advanced way to integrate with the Kernel, for observability, security and networking. You'll see it included in various CNCF projects like Cilium, Falco, Kepler, and others.

\n
\n

Whilst BPF is powerful, it's also a very fast moving space, and was particularly complicated to patch to Firecracker's minimal Kernel configuration. We want to say a thank you to Mahé Tardy\n who maintains Tetragon and to Duffie Coolie both from Isovalent for pointers and collaboration.

\n

We've made a big jump in the supported Kernel version from 5.10.77 up to 5.10.201, with newer revisions being made available on a continual basis.

\n

To update your servers, log in via SSH and edit /etc/default/actuated.

\n

For amd64:

\n
AGENT_IMAGE_REF=\"ghcr.io/openfaasltd/actuated-ubuntu22.04:x86_64-latest\"\nAGENT_KERNEL_REF=\"ghcr.io/openfaasltd/actuated-kernel:x86_64-latest\"\n
\n

For arm64:

\n
AGENT_IMAGE_REF=\"ghcr.io/openfaasltd/actuated-ubuntu22.04:aarch64-latest\"\nAGENT_KERNEL_REF=\"ghcr.io/openfaasltd/actuated-kernel:aarch64-latest\"\n
\n

Once you have the new images in place, reboot the server. Updates to the Kernel and root filesystem will be delivered Over The Air (OTA) automatically by our team.

\n

Request custom vCPU and RAM per job

\n

Our initial version of actuated aimed to set a specific vCPU and RAM value for each build, designed to slice up a machine equally for the best mix of performance and concurrency. We would recommend it to teams during their onboarding call, then mostly leave it as it was. For a machine with 128GB RAM and 32 threads, you may have set it up for 8 jobs with 4x vCPU and 16GB RAM each, or 4 jobs with 8x vCPU and 32GB RAM.

\n

However, whilst working with Justin Gray, CTO at Toolpath, we found that their build needed increasing amounts of RAM to avoid an Out Of Memory (OOM) crash, and so implemented custom labels.

\n

These labels do not have any predetermined values, so you can change them to any value you like, independently. You're not locked into a set combinations.

\n

Small tasks, automation, publishing Helm charts?

\n
runs-on: actuated-2cpu-8gb\n
\n

Building a large application, or training an AI model?

\n
runs-on: actuated-32cpu-128gb\n
\n

Spreading your plan across all available hosts

\n

Previously, if you had a plan with 10 concurrent builds and both an Arm server and an amd64 server, we'd split your plan statically 50/50. So you could run a maximum of 5 Arm and 5 amd64 builds at the same time.

\n

Now, we've made this dynamic, all of your 10 builds can start on the Arm or amd64 server. Or, 1 could start on the Arm server, then 9 on the amd64 server, and so on.

\n

The change makes the product better value for money, and we had always wanted it to work this way.

\n

Thanks to Patrick Stephens at Fluent/Calyptia for the suggestion and for helping us test it out.

\n

KVM acceleration aka running VMs in your CI pipeline

\n

When we started actuated over 12 months ago, there was no support for using KVM acceleration, or running a VM within a GitHub Actions job within GitHub's infrastructure. We made it available for our customers first, with a custom Kernel configuration for x86_64 servers. Arm support for launching VMs within VMs is not currently available in the current generation of Ampere servers, but may be available within the next generation of chips and Kernels.

\n

We have several tutorials including how to run Firecracker itself within a CI job, Packer, Nix and more.

\n

When you run Packer in a VM, instead of with one of the cloud drivers, you save on time and costs, by not having to fire up cloud resources on AWS, GCP, Azure, and so forth. Instead, you can run a local VM to build the image, then convert it to an AMI or another format.

\n

One of our customers has started exploring launching a VM during a CI job in order to test air-gapped support for enterprise customers. This is a great example of how you can use nested virtualisation to test your own product in a repeatable way.

\n

Nix benefits particularly from being able to create a clean, isolated environment within a CI pipeline, to get a repeatable build. Graham Christensen from Determinate Systems reached out to collaborate on testing their Nix installer in actuated.

\n

He didn't expect it to run, but when it worked first time, he remarked: \"Perfect! I'm impressed and happy that our action works out of the box.\"

\n
jobs:\n  specs:\n    name: ci\n    runs-on: [actuated-16cpu-32gb]\n    steps:\n      - uses: DeterminateSystems/nix-installer-action@main\n      - run: |\n            nix-build '<nixpkgs/nixos/tests/doas.nix>'\n
\n\n

Wrapping up

\n

We've now released eBPF/BTF support as part of onboarding CNCF projects, updated to the latest Kernel revision, made scheduling better value for money & easier to customise, and have added a range of tutorials for getting the most out of nested virtualisation.

\n

If you'd like to try out actuated, you can get started same day.

\n

Talk to us..

\n

You may also like:

\n","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/develop-a-great-go-cli.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/develop-a-great-go-cli.json new file mode 100644 index 00000000..116cfcda --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/develop-a-great-go-cli.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","contentHtml":"

Is your project's CLI growing with you? I'll cover some of the lessons learned writing the OpenFaaS, actuated, actions-usage, arkade and k3sup CLIs, going as far back as 2016. I hope you'll find some ideas or inspiration for your own projects - either to start them off, or to improve them as you go along.

\n
\n

Just starting your journey, or want to go deeper?

\n

You can master the fundamentals of Go (also called Golang) with my eBook Everyday Golang, which includes chapters on Go routines, HTTP clients and servers, text templates, unit testing and crafting a CLI. If you're on a budget, I would recommend checkout out the official Go tour, too.

\n
\n

Introduction

\n

The earliest CLI I wrote was for OpenFaaS, called faas-cli. It's a client for a REST API exposed over HTTP, and I remember how it felt to add the first command list functions, then one more, and one more, until it was a fully working CLI with a dozen commands.

\n

But it started with one command - something that was useful to us at the time, that was to list the available functions.

\n

The initial version used Go's built-in flags parser, which is rudimentary, but perfectly functional.

\n
faas-cli -list\nfaas-cli -describe\nfaas-cli -deploy\n
\n

Over time, you may outgrow this simple approach, and drift towards wanting sub-commands, each with their own set of options.

\n

An early contributor John McCabe introduced me to Cobra and asked if he could convert everything over.

\n
faas-cli list\nfaas-cli describe\nfaas-cli deploy\n
\n

Now each sub-command can have its set of flags, and even sub-commands in the case of faas-cli secret list/create/delete

\n

actions-usage is a free analytics tool we wrote for GitHub Actions users to iterate GitHub's API and summarise your usage over a certain period of time. It's also written in Go, but because it's mostly single-purpose, it'll probably never need sub-commands.

\n
actions-usage -days 28 \\\n    -token-file ~/pat.txt \\\n    -org openfaasltd\n
\n

Shortly after launching the tool for teams an open-source organisations, we had a feature request to run it on individual user accounts.

\n

That meant switching up some API calls and adding new CLI flags:

\n
actions-usage -days 7 \\\n    -token-file ~/pat.txt \\\n    -user alexellis\n
\n

We then got a bit clever and started adding some extra reports and details, you can see what it looks in the article Understand your usage of GitHub Actions

\n

What's new for actuated-cli

\n

I'm very much a believer in a Minimal Viable Product (MVP). If you can create some value or utility to users, you should ship it as early as possible, especially if you have a good feedback loop with them.

\n

A quick note about the actuated-cli, it's main use-cases are to:

\n\n

No more owner flags

\n

The actuated-cli was designed to work on a certain organisation, but it meant extra typing, so wherever possible, we've removed the flag completely.

\n
actuated-cli runners --owner openfaasltd\n
\n

becomes:

\n
actuated-cli runners\n
\n

How did we do this? We determine the intersection of organisations for which your account is authorized, and which are enrolled for actuated. It's much less typing and it's more intuitive.

\n

The host flag became a positional argument

\n

This was another exercise in reducing typing. Let's say we wanted to upgrade the agent for a certain host, we'd have to type:

\n
actuated-cli upgrade --owner openfaasltd --host server-1\n
\n

By looking at the \"args\" slice, instead of for a specific command, we can assume that any text after the flags is always the server name:

\n
actuated-cli upgrade --owner openfaasltd server-1\n
\n

Token management

\n

The actuated CLI uses a GitHub personal access token to authenticate with the API. This is a common pattern, but it's not always clear how to manage the token.

\n

We took inspiration from the gh CLI, which is a wrapper around the GitHub API.

\n

The gh CLI has a gh auth command which can be used to obtain a token, and save it to a local file, then any future usage of the CLI will use that token.

\n

Before, you had to create a Personal Access Token in the GitHub UI, then copy and paste it into a file, and decide where to put it, and what to name it. What's more, if you missed a permission, then the token wouldn't work.

\n
actuated-cli --token ~/pat.txt\n
\n

Now, you simply run:

\n
actuated-cli auth\n
\n

And as you saw from the previous commands, there's no longer any need for the --token flag. Unless of course, you want to supply it, then you can.

\n

A good way to have a default for a flag, and then an override, is to use the Cobra package's Changed() function. Read the default, unless .Changed() on the --token or --token-value flags return true.

\n

The --json flag

\n

From early on, I knew that I would want to be able to pipe output into .jq, or perhaps even do some scripting. I've seen this in docker, kubectl and numerous other CLI tools written in Go.

\n
actuated-cli runners --json | jq '.[] | .name'\n\n\"m1m1\"\n\"m1m2\"\n\"of-epyc-lon1\"\n
\n

The JSON format also allows you to get access to certain fields which the API call returns, which may not be printed by the default command's text-based formatter:

\n
|         NAME         |  CUSTOMER   |   STATUS    | VMS  | PING  |   UP    | CPUS |   RAM   | FREE RAM | ARCH  |                 VERSION                  |\n|----------------------|-------------|-------------|------|-------|---------|------|---------|----------|-------|------------------------------------------|\n| of-epyc-lon1         | openfaasltd | running     | 0/5  | 7ms   | 6 days  |   48 | 65.42GB | 62.85GB  | amd64 | 5f702001a952e496a9873d2e37643bdf4a91c229 |\n
\n

Instead, we get:

\n
[  {\n    \"name\": \"of-epyc-lon1\",\n    \"customer\": \"openfaasltd\",\n    \"pingNano\": 30994998,\n    \"uptimeNano\": 579599000000000,\n    \"cpus\": 48,\n    \"memory\": 65423184000,\n    \"memoryAvailable\": 62852432000,\n    \"vms\": 0,\n    \"maxVms\": 5,\n    \"reachable\": true,\n    \"status\": \"running\",\n    \"agentSHA\": \"5f702001a952e496a9873d2e37643bdf4a91c229\",\n    \"arch\": \"amd64\"\n  }\n]\n
\n

SSH commands and doing the right thing

\n

Actuated has a built-in SSH gateway, this means that any job can be debugged - whether running on a hosted or self-hosted runner, just by editing the workflow YAML.

\n

Add the following to the - steps: section, and the id_token: write permission, and your workflow will pause, and then you can connect over SSH using the CLI or the UI.

\n
    - uses: self-actuated/connect-ssh@master\n
\n

There are two sub-commands:

\n\n

Here's an example of having only one connection:

\n
actuated-cli ssh list\n| NO  |   ACTOR   |   HOSTNAME    | RX | TX | CONNECTED |\n|-----|-----------|---------------|----|----|-----------|\n|   1 | alexellis | fv-az1125-168 |  0 |  0 | 32s       |\n
\n

Now how do you think the ssh connect command should work?

\n

Here's the most obvious way:

\n
actuated-cli ssh connect --hostname fv-az1125-168\n
\n

This is a little obtuse, since we only have one server to connect to, we can improve it for the user, with:

\n
actuated-cli ssh connect\n
\n

That's right, we do the right thing, the obvious thing.

\n

Then when there is more than one connection, instead of adding two flags --no or --hostname, we can simply take the positional argument:

\n
actuated-cli ssh connect 1\nactuated-cli ssh connect fv-az1125-168\n
\n

Are there any places where you could simplify your own CLI?

\n

Read the source code here: ssh_connect.go

\n

The --verbose flag

\n

We haven't made any use of the --verbose flag yet in the CLI, but it's a common pattern which has been used in faas-cli and various others. Once your output gets to a certain width, it can be hard to view in a terminal, like the output from the previous command.

\n

To implement --verbose, you should reduce the columns to the absolute minimum to be useful, so maybe we could give up the Version, customer, ping, and CPUs columns in the standard view, then add them back in with --verbose.

\n

Table printing

\n

As you can see from the output of the commands above, we make heavy usage of a table printer.

\n

You don't necessarily need a 3rd-party table printer, Go has a fairly good \"tab writer\" which can create nicely formatted code:

\n
faas-cli list -g https://openfaas.example.com\nFunction                        Invocations     Replicas\nbcrypt                          9               1    \nfiglet                          0               1    \ninception                       0               1    \nnodeinfo                        2               1    \nping-url                        0               1  \n
\n

You can find the standard tabwriter package here.

\n

Or try out the tablewriter package by Olekukonko. We've been able to make use of it in arkade too - a free marketplace for developer tools.

\n

See usage in arkade here: table.go

\n

See usage in actuated-cli's SSH command here: ssh_ls.go

\n

Progress bars

\n

One thing that has been great about having open-source CLIs, is that other people make suggestions and help you learn about new patterns.

\n

For arkade, Ramiro from Okteto sent a PR to add a progress bar to show how long remained to download a big binary like the Kubernetes CLI.

\n
arkade get kubectl\nDownloading: kubectl\nDownloading: https://storage.googleapis.com/kubernetes-release/release/v1.24.2/bin/linux/amd64/kubectl\n\n15.28 MiB / 43.59 MiB [------------------------>____________________________________] 35.05%\n
\n

It's simple, but gives enough feedback to stop you from thinking the program is stuck. In my Human Design Interaction course at university, I learned that anything over 7s triggers uncertainty in an end-user.

\n

See how it's implemented: download.go

\n

HTTP and REST are not the only option

\n

When I wrote K3sup, a tool to install K3s on remote servers, I turned to SSH to automate the process. So rather than making HTTP calls, a Go library for SSH is used to open a connection and run remote commands.

\n

It also simplifies an annoying post-installation task - managing the kubeconfig file. By default this is a protected file on the initial server you set up, k3sup will download the file and merge it with your local kubeconfig.

\n
k3sup install \\\n    --host HOST1 \\\n    --user ubuntu \\\n    --merge \\\n    --local-path ~/.kube/config\n
\n

I'd recommend trying out golang.org/x/crypto/ssh in your own CLIs and tools. It's great for automation, and really simple to use.

\n

Document everything as best as you can

\n

Here's an example of a command with good documentation:

\n
Schedule additional VMs to repair the build queue.\nUse sparingly, check the build queue to see if there is a need for \nmore VMs to be launched. Then, allow ample time for the new VMs to \npick up a job by checking the build queue again for an in_progress\nstatus.\n\nUsage:\n  actuated-cli repair [flags]\n\nExamples:\n  ## Launch VMs for queued jobs in a given organisation\n  actuated repair OWNER\n\n  ## Launch VMs for queued jobs in a given organisation for a customer\n  actuated repair --staff OWNER\n\n\nFlags:\n  -h, --help    help for repair\n  -s, --staff   List staff repair\n\nGlobal Flags:\n  -t, --token string         File to read for Personal Access Token (default \"$HOME/.actuated/PAT\")\n      --token-value string   Personal Access Token\n
\n

Not only does it show example usage, so users can understand what can be done, but it has a detailed explanation of when to use the command.

\n
\tcmd := &cobra.Command{\n\t\tUse:   \"repair\",\n\t\tShort: \"Schedule additional VMs to repair the build queue\",\n\t\tLong: `Schedule additional VMs to repair the build queue.\nUse sparingly, check the build queue to see if there is a need for \nmore VMs to be launched. Then, allow ample time for the new VMs to \npick up a job by checking the build queue again for an in_progress\nstatus.`,\n\t\tExample: `  ## Launch VMs for queued jobs in a given organisation\n  actuated repair OWNER\n\n  ## Launch VMs for queued jobs in a given organisation for a customer\n  actuated repair --staff OWNER\n`\n    }\n
\n

Browse the source code: repair.go

\n

Wrapping up

\n

I covered just a few of the recent changes - some were driven by end-user feedback, others were open source contributions, and in some cases, we just wanted to make the CLI easier to use. I've been writing CLIs for a long time, and I still have a lot to learn.

\n

What CLIs do you maintain? Could you apply any of the above to them?

\n

Do you want to learn how to master the fundamentals of Go? Check out my eBook: Everyday Go.

\n

If you're on a budget, I would recommend checkout out the official Go tour, too. It'll help you understand some of the basics of the language and is a good primer for the e-book.

\n

Read the source code of the CLIs we mentioned:

\n","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-nix-builds.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-nix-builds.json new file mode 100644 index 00000000..67707a43 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-nix-builds.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","contentHtml":"

faasd is a lightweight and portable version of OpenFaaS that was created to run on a single host. In my spare time I maintain faasd-nix, a project that packages faasd and exposes a NixOS module so it can be run with NixOS.

\n

The module itself depends on faasd, containerd and the CNI plugins and all of these binaries are built in CI with Nix and then cached using Cachix to save time on subsequent builds.

\n

I often deploy faasd with NixOS on a Raspberry Pi and to the cloud, so I build binaries for both x86_64 and aarch64. The build usually runs on the default GitHub hosted action runners. Now because GitHub currently doesn't have Arm support, I use QEMU instead which can emulate them. The drawback of this approach is that builds can sometimes be several times slower.

\n
\n

For some of our customers, their builds couldn't even complete in 6 hours using QEMU, and only took between 5-20 minutes using native Arm hardware. Alex Ellis, Founder of Actuated.

\n
\n

While upgrading to the latest nixpkgs release recently I decided to try and build the project on runners managed with Actuated to see the improvements that can be made by switching to both bigger x86_64 iron and native Arm hardware.

\n

Nix and GitHub actions

\n

One of the features Nix offers are reproducible builds. Once a package is declared it can be built on any system. There is no need to prepare your machine with all the build dependencies. The only requirement is that Nix is installed.

\n
\n

If you are new to Nix, then I'd recommend you read the Zero to Nix guide. It's what got me excited about the project.

\n
\n

Because Nix is declarative and offers reproducible builds, it is easy to setup a concise build pipeline for GitHub actions. A lot of steps usually required to setup the build environment can be left out. For instance, faasd requires Go, but there's no need to install it onto the build machine, and you'd normally have to install btrfs-progs to build containerd, but that's not something you have to think about, because Nix will take care of it for you.

\n

Another advantage of the reproducible builds is that if it works on your local machine it most likely also works in CI. No need to debug and find any discrepancies between your local and CI environment.

\n
\n

Of course, if you ever do get frustrated and want to debug a build, you can use the built-in SSH feature in Actuated. Alex Ellis, Founder of Actuated.

\n
\n

This is what the workflow looks like for building faasd and its related packages:

\n
jobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - uses: cachix/install-nix-action@v21\n      - name: Build faasd 🔧\n        run: |\n          nix build -L .#faasd\n      - name: Build containerd 🔧\n        run: |\n          nix build -L .#faasd-containerd\n      - name: Build cni-plugin 🔧\n        run: |\n          nix build -L .#faasd-cni-plugins\n
\n

All this pipeline does is install Nix, using the cachix/install-nix-action and run the nix build command for the packages that need to be built.

\n

Notes on the nix build for aarch64

\n

To build the packages for multiple architectures there are a couple of options:

\n\n

The preferred option would be to compile everything natively on an aarch64 machine as that would result in the best performance. However, at the time of writing GitHub does not provide Arm runners. That is why QEMU is used by many people to compile binaries in CI.

\n

Enabling the binfmt wrapper on NixOS can be done easily through the NixOS configuration. On non-NixOS machines, like on the GitHub runner VM, the QEMU static binaries need to be installed and the Nix daemon configuration updated.

\n

Instructions to configure Nix for compilation with QEMU can be found on the NixOS wiki

\n

The workflow for building aarch64 packages with QEMU on GitHub Actions looks like this:

\n
jobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - uses: docker/setup-qemu-action@v2\n      - uses: cachix/install-nix-action@v21\n        with:\n          extra_nix_config: |\n            extra-platforms = aarch64-linux\n      - name: Build faasd 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd\n      - name: Build containerd 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd-containerd\n      - name: Build cni-plugin 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd-cni-plugins\n\n
\n

Install the QEMU static binaries using docker/setup-qemu-action. Let the nix daemon know that it can build for aarch64 by adding extra-platforms = aarch64-linux via the extra_nix_config input on the install nix action. Update the nix build commands to specify platform e.g. nix build .#packages.aarch64-linux.faasd.

\n

Speeding up the build with a Raspberry Pi

\n

Nix has great support for caching and build speeds can be improved greatly by never building things twice. This project normally uses Cachix for caching and charing binaries across systems. For this comparison caching was disabled. All packages and their dependencies are built from scratch again each time.

\n

Building the project takes around 4 minutes and 20 seconds on the standard GitHub hosted runner. After switching to a more powerful Actuated runner with 4CPUs and 8GB of RAM the build time dropped to 2 minutes and 15 seconds.

\n

\"Comparison

\n
\n

Comparison of more powerful Actuated runner with GitHub hosted runner.

\n
\n

While build times are still acceptable for x86_64 this is not the case for the aarch64 build. It takes around 55 minutes to complete the Arm build with QEMU on a GitHub runner.

\n

Running the same build with QEMU on the Actuated runner already brings down the build time to 19 minutes and 40 seconds. Running the build natively on a Raspberry Pi 4 (8GB) completed in 11 minutes and 47 seconds. Building on a more powerful Arm machine would potentially reduce this time to a couple of minutes.

\n

\"Results

\n
\n

Results of the matrix build comparing the GitHub hosted runner and the 2 Actuated runners.

\n
\n

Running the build natively on the Pi did even beat the fast bare-metal machine that is using QEMU.

\n

My colleague Alex ran the same build on his Raspberry Pi using Actuated and an NVMe mounted over USB-C, he got the build time down even further. Why? Because it increased the I/O performance. In fact, if you build this on server-grade Arm like the Ampere Altra, it would be about 4x faster than the Pi 4.

\n

Building for Arm:

\n\n

Building for x86_64

\n\n

\"Alex's

\n

These results show that whatever the Arm hardware you pick, it'll likely be faster than QEMU, even when QEMU is run on the fastest bare-metal available, the slowest Arm hardware will beat it by minutes.

\n

Wrapping up

\n

Building your projects with Nix allows your GitHub actions pipelines to be concise and easy to maintain.

\n

Even when you are not using Nix to build your project it can still help you to create concise and easy to maintain GitHub Action workflows. With Nix shell environments you can use Nix to declare which dependencies you want to make available inside an isolated shell environment for your project: Streamline your GitHub Actions dependencies using Nix

\n

Building Nix packages or entire NixOS systems on GitHub Actions can be slow especially if you need to build for Arm. Bringing your own metal to GitHub actions can speed up your builds. If you need Arm runners, Actuated is one of the only options for securely isolated CI that is safe for Open Source and public repositories. Alex explains why in: Is the GitHub Actions self-hosted runner safe for Open Source?

\n

Another powerful feature of the Nix ecosystem is the ability to run integration tests using virtual machines (NixOS test). This feature requires hardware acceleration to be available in the CI runner. Actuated makes it possible to run these tests in GitHub Actions CI pipelines: how to run KVM guests in your GitHub Actions.

\n

See also:

\n","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-self-hosted-cache.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-self-hosted-cache.json new file mode 100644 index 00000000..c40445d2 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/faster-self-hosted-cache.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","contentHtml":"

In some of our builds for actuated we cache things like the Linux Kernel, so we don't needlessly rebuild it when we update packages in our base images. It can shave minutes off every build meaning our servers can be used more efficiently. Most customers we've seen so far only make light to modest use of GitHub's hosted cache, so haven't noticed much of a latency problem.

\n

But you don't have to spend too long on the issuer tracker for GitHub Actions to find people complaining about the cache being slow or locking up completely for self-hosted runners.

\n

Go, Rust, Python and other languages don't tend to make heavy use of caches, and Docker has some of its own mechanisms like building cached steps into published images aka inline caching. But for the Node.js ecosystem, the node_modules folder and yarn cache can become huge and take a long time to download. That's one place where you may start to see tension between the speed of self-hosted runners and the latency of the cache. If your repository is a monorepo or has lots of large artifacts, you may get a speed boost by caching that too.

\n

So why is GitHub's cache so fast for hosted runners, and (sometimes) so slow self-hosted runners?

\n

Simply put - GitHub runs VMs and the accompanying cache on the same network, so they can talk over a high speed backbone connection. But when you run a self-hosted runner, then any download or upload operations are taking place over the public Internet.

\n

Something else that can slow builds down is having to download large base images from the Docker Hub. We've already covered how to solve that for actuated in the docs.

\n

Speeding up in the real world

\n

We recently worked with Roderik, the CTO of SettleMint to migrate their CI from a self-hosted Kubernetes solution Actions Runtime Controller (ARC) to actuated. He told me that they originally moved from GitHub's hosted runners to ARC to save money, increase speed and to lower the latency of their builds. Unfortunately, running container builds within Kubernetes provided very poor isolation, and side effects were being left over between builds, even with a pool of ephemeral containers. They also wanted to reduce the amount of effort required to maintain a Kubernetes cluster and control-plane for CI.

\n

Roderik explained that he'd been able to get times down by using pnpm instead of yarn, and said every Node project should try it out to see the speed increases. He believes the main improvement is due to efficient downloading and caching. pnpm is a drop-in replacement for npm and yarn, and is compatible with both.

\n
\n

In some cases, we found that downloading dependencies from the Internet was faster than using GitHub's remote cache. The speed for a hosted runner was often over 100MBs/sec, but for a self-hosted runner it was closer to 20MBs/sec.

\n
\n

That's when we started to look into how we could run a cache directly on the same network as our self-hosted runners, or even on the machine that was scheduling the Firecracker VMs.

\n
\n

\"With the local cache that Alex helped us set up, the cache is almost instantaneous. It doesn't even have time to show a progress bar.\"

\n
\n

Long story short, SettleMint have successfully migrated their CI for x86 and Arm to actuated for the whole developer team:

\n

Super happy with my new self hosted GHA runners powered by @selfactuated, native speeds on both AMD and ARM bare metal monster machines. Our CI now goes brrrr… pic.twitter.com/quZ4qfcLmu

— roderik.eth (@r0derik) May 23, 2023
\n

This post is about speed improvements for caching, but if you're finding that QEMU is too slow to build your Arm containers on hosted runners, you may benefit from switching to actuated with bare-metal Arm servers.

\n

See also:

\n\n

Set up a self-hosted cache for GitHub Actions

\n

In order to set up a self-hosted cache for GitHub Actions, we switched out the official actions/cache@v3 action for tespkg/actions-cache@v1 created by Target Energy Solutions, a UK-based company, which can target S3 instead of the proprietary GitHub cache.

\n

We then had to chose between Seaweedfs and Minio for the self-hosted S3 server. Of course, there's also nothing stopping you from actually using AWS S3, or Google Cloud Storage, or another hosted service.

\n

Then, the question was - should we run the S3 service directly on the server that was running Firecracker VMs, for ultimate near-loopback speed, or on a machine provisioned in the same region, just like GitHub does with Azure?

\n

Either would be a fine option. If you decide to host a public S3 cache, make sure that authentication and TLS are both enabled. You may also want to set up an IP whitelist just to deter any bots that may scan for public endpoints.

\n

Set up Seaweedfs

\n

The Seaweedfs README describes the project as:

\n
\n

\"a fast distributed storage system for blobs, objects, files, and data lake, for billions of files! Blob store has O(1) disk seek, cloud tiering. Filer supports Cloud Drive, cross-DC active-active replication, Kubernetes, POSIX FUSE mount, S3 API, S3 Gateway, Hadoop, WebDAV, encryption, Erasure Coding.\"

\n
\n

We liked it so much that we'd already added it to the arkade marketplace, arkade is a faster, developer-focused alternative to brew.

\n
arkade get seaweedfs\nsudo mv ~/.arkade/bin/seaweedfs /usr/local/bin\n
\n

Define a secret key and access key to be used from the CI jobs in the /etc/seaweedfs/s3.conf file:

\n
{\n  \"identities\": [\n    {\n      \"name\": \"actuated\",\n      \"credentials\": [\n        {\n          \"accessKey\": \"s3cr3t\",\n          \"secretKey\": \"s3cr3t\"\n        }\n      ],\n      \"actions\": [\n        \"Admin\",\n        \"Read\",\n        \"List\",\n        \"Tagging\",\n        \"Write\"\n      ]\n    }\n  ]\n}\n
\n

Create seaweedfs.service:

\n
[Unit]\nDescription=SeaweedFS\nAfter=network.target\n\n[Service]\nUser=root\nExecStart=/usr/local/bin/seaweedfs server -ip=192.168.128.1 -volume.max=0 -volume.fileSizeLimitMB=2048 -dir=/home/runner-cache -s3 -s3.config=/etc/seaweedfs/s3.conf\nRestart=on-failure\n\n[Install]\nWantedBy=multi-user.target\n
\n

We have set -volume.max=0 -volume.fileSizeLimitMB=2048 to minimize the amount of space used and to allow large zip files of up to 2GB, but you can change this to suit your needs. See seaweedfs server --help for more options.

\n

Install it and check that it started:

\n
sudo cp ./seaweedfs.service /etc/systemd/system/seaweedfs.service\nsudo systemctl enable seaweedfs\n\nsudo journalctl -u seaweedfs -f\n
\n

Try it out

\n

You'll need to decide what you want to cache and whether you want to use a hosted, or self-hosted S3 service - either directly on the actuated server or on a separate machine in the same region.

\n

Roderik explained that the pnpm cache was important for node_modules, but that actually caching the git checkout saved a lot of time too. So he added both into his builds.

\n

Here's an example:

\n
    - name: \"Set current date as env variable\"\n      shell: bash\n      run: |\n        echo \"CHECKOUT_DATE=$(date +'%V-%Y')\" >> $GITHUB_ENV\n      id: date\n    - uses: tespkg/actions-cache@v1\n      with:\n        endpoint: \"192.168.128.1\"\n        port: 8333\n        insecure: true\n        accessKey: \"s3cr3t\"\n        secretKey: \"s3cr3t\"\n        bucket: actuated-runners\n        region: local\n        use-fallback: true\n        path: ./.git\n        key: ${{ runner.os }}-checkout-${{ env.CHECKOUT_DATE }}\n        restore-keys: |\n          ${{ runner.os }}-checkout-\n
\n\n

See also: Official GitHub Actions Cache action

\n

You may also want to create a self-signed certificate for the S3 service and then set insecure: false to ensure that the connection is encrypted. If you're running these builds within private repositories, tampering is unlikely.

\n

Roderik explained that the cache key uses a week-year format, rather than a SHA. Why? Because a SHA would change on every build, meaning that a save and load would be performed on every build, using up more space and slowing things down. In this example, There's only ever 52 cache entries per year.

\n
\n

You define a key which is unique if the cache needs to be updated. Then you define a restore key that matches part or all of the key.\nPart means it takes the last one that matches, then updates at the end of the run, in the post part, it then uses the key to upload the zip file if the key is different from the one stored.

\n
\n

In one instance, a cached checkout went from 2m40s to 11s. That kind of time saving adds up quickly if you have a lot of builds.

\n

Roderik's pipeline has multiple steps, and may need to run multiple times, so we're looking at 55s instead of 13 minutes for 5 jobs or runs.

\n

\"Example

\n
\n

One of the team's pipelines

\n
\n

Here's how to enable a cache for pnpm:

\n
    - name: Install PNPM\n      uses: pnpm/action-setup@v2\n      with:\n        run_install: |\n          - args: [--global, node-gyp]\n\n    - name: Get pnpm store directory\n      id: pnpm-cache\n      shell: bash\n      run: |\n        echo \"STORE_PATH=$(pnpm store path)\" >> $GITHUB_OUTPUT\n\n    - uses: tespkg/actions-cache@v1\n      with:\n        endpoint: \"192.168.128.1\"\n        port: 8333\n        insecure: true\n        accessKey: \"s3cr3t\"\n        secretKey: \"s3cr3t\"\n        bucket: actuated-runners\n        region: local\n        use-fallback: true\n        path:\n          ${{ steps.pnpm-cache.outputs.STORE_PATH }}\n          ~/.cache\n          .cache\n        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}\n        restore-keys: |\n          ${{ runner.os }}-pnpm-store-\n\n    - name: Install dependencies\n      shell: bash\n      run: |\n        pnpm install --frozen-lockfile --prefer-offline\n      env:\n        HUSKY: '0'\n        NODE_ENV: development\n
\n

Picking a good key and restore key can help optimize when the cache is read from and written to:

\n
\n

\"You need to determine a good key and restore key. For pnpm, we use the hash of the lock file in the key, but leave it out of the restore key. So if I update the lock file, it starts from the last cache, updates it, and stores the new cache with the new hash\"

\n
\n

If you'd like a good starting-point for GitHub Actions Caching, Han Verstraete from our team wrote up a good primer for the actuated docs:

\n

Example: GitHub Actions cache

\n

Conclusion

\n

We were able to dramatically speed up caching for GitHub Actions by using a self-hosted S3 service. We used Seaweedfs directly on the server running Firecracker with a fallback to GitHub's cache if the S3 service was unavailable.

\n

\"Brr\"

\n
\n

An Ampere Altra Arm server running parallel VMs using Firecracker. The CPU is going brr. Find a server with our guide

\n
\n

We also tend to recommend that all customers enable a mirror of the Docker Hub to counter restrictive rate-limits. The other reason is to avoid any penalties that you'd see from downloading large base images - or from downloading small to medium sized images when running in high concurrency.

\n

You can find out how to configure a container mirror for the Docker Hub using actuated here: Set up a registry mirror. When testing builds for the Discourse team, there was a 2.5GB container image used for UI testing with various browsers preinstalled within it. We found that we could shave off a few minutes off the build time by using the local mirror. Imagine 10x of those builds running at once, needlessly downloading 250GB of data.

\n

What if you're not an actuated customer? Can you still benefit from a faster cache? You could try out a hosted service like AWS S3 or Google Cloud Storage, provisioned in a region closer to your runners. The speed probably won't quite be as good, but it should still be a lot faster than reaching over the Internet to GitHub's cache.

\n

If you'd like to try out actuated for your team, reach out to us to find out more.

\n

Book 20 mins with me if you think your team could benefit from the below for GitHub Actions:

🚀 Insights into CI usage across your organisation
🚀 Faster x86 builds
🚀 Native Arm builds that can actually finish
🚀 Fixed-costs & less managementhttps://t.co/iTiZsH9pgv

— Alex Ellis (@alexellisuk) May 10, 2023
","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/firecracker-container-lab.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/firecracker-container-lab.json new file mode 100644 index 00000000..3074a895 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/firecracker-container-lab.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","contentHtml":"

When I started learning Firecracker, I ran into frustration after frustration with broken tutorials that were popular in their day, but just hadn't been kept up to date. Almost nothing worked, or was far too complex for the level of interest I had at the time. Most recently, one of the Firecracker maintainers in an effort to make the quickstart better, made it even harder to use. (You can still get a copy of the original Firecracker quickstart in our tutorial on nested virtualisation)

\n

So I wrote a lab that takes a container image and converts it to a microVM. You'll get your hands dirty, you'll run a microVM, you'll be able to use curl and ssh, even expose a HTTP server to the Internet via inlets, if (like me), you find that kind of thing fun.

\n

Why would you want to explore Firecracker? A friend of mine, Ivan Velichko is a prolific writer on containers, and Docker. He is one of the biggest independent evangelists for containers and Kubernetes that I know.

\n

So when he wanted to build an online labs and training environment, why did he pick Firecracker instead? Simply put, he told us that containers don't cut it. He needed something that would mirror the type of machine that you'd encounter in production, when you provision an EC2 instance or a GCP VM. Running Docker, Kubernetes, and performing are hard to do securely within a container, and he knew that was important for his students.

\n

For us - we had very similar reasons for picking Firecracker for a secure CI solution. Too often the security issues around running privileged containers, and the slow speed of Docker In Docker's (DIND) Virtual Filesystem Driver (VFS), are ignored. Heads are put into the sand. We couldn't do that and developed actuated as a result. Since we launched the pilot, we've now run over 110k VMs for customer CI jobs on GitHub Actions, and have a tech preview for GitLab CI where a job can be running within 1 second of pushing a \"commit\".

\n

So let's get that microVM running for you?

\n

How it works 🔬

\n

How to build a microVM from a container

\n

\"/images/2023-09-firecracker-lab/conceptual.png\"

\n
\n

Conceptual archicture of the lab

\n
\n

Here's what we'll be doing:

\n\n

Let's look at why we need a init, instead of just running the entrypoint of a container.

\n

Whilst in theory, you can start a microVM where the first process (PID 1) is your workload, in the same way as Docker, it will leave you with a system which is not properly initialised with things like a /proc/ filesystem, tempfs, hostname, and other things that you'd expect to find in a Linux system.

\n

For that reason, you'll need to either install systemd into the container image you want to use, or build your own basic init system, which sets up the machine, then starts your workload.

\n

We're doing the latter here.

\n

In the below program, you'll see key devices and files mounted, to make a functional system. The hostname is then set by using a syscall, and finally /bin/sh is started. You could also start a specific binary, or build an agent into the init for Remote Procedure Calls (RPC) to start and stop your workload, and to query metrics.

\n

The team at Fly.io built their own init and agent combined, and opened-sourced a very early version: github.com/superfly/init-snapshot.

\n

You'll find my init in: ./init/main.go:

\n
// Copyright Alex Ellis 2023\n\npackage main\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\n\t\"syscall\"\n)\n\nconst paths = \"PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\"\n\n// main starts an init process that can prepare an environment and start a shell\n// after the Kernel has started.\nfunc main() {\n\tfmt.Printf(\"Lab init booting\\nCopyright Alex Ellis 2022, OpenFaaS Ltd\\n\")\n\n\tmount(\"none\", \"/proc\", \"proc\", 0)\n\tmount(\"none\", \"/dev/pts\", \"devpts\", 0)\n\tmount(\"none\", \"/dev/mqueue\", \"mqueue\", 0)\n\tmount(\"none\", \"/dev/shm\", \"tmpfs\", 0)\n\tmount(\"none\", \"/sys\", \"sysfs\", 0)\n\tmount(\"none\", \"/sys/fs/cgroup\", \"cgroup\", 0)\n\n\tsetHostname(\"lab-vm\")\n\n\tfmt.Printf(\"Lab starting /bin/sh\\n\")\n\n\tcmd := exec.Command(\"/bin/sh\")\n\n\tcmd.Env = append(cmd.Env, paths)\n\tcmd.Stdin = os.Stdin\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\n\terr := cmd.Start()\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not start /bin/sh, error: %s\", err))\n\t}\n\n\terr = cmd.Wait()\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not wait for /bin/sh, error: %s\", err))\n\t}\n}\n\nfunc setHostname(hostname string) {\n\terr := syscall.Sethostname([]byte(hostname))\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"cannot set hostname to %s, error: %s\", hostname, err))\n\t}\n}\n\nfunc mount(source, target, filesystemtype string, flags uintptr) {\n\n\tif _, err := os.Stat(target); os.IsNotExist(err) {\n\t\terr := os.MkdirAll(target, 0755)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Sprintf(\"error creating target folder: %s %s\", target, err))\n\t\t}\n\t}\n\n\terr := syscall.Mount(source, target, filesystemtype, flags, \"\")\n\tif err != nil {\n\t\tlog.Printf(\"%s\", fmt.Errorf(\"error mounting %s to %s, error: %s\", source, target, err))\n\t}\n}\n
\n

What you'll need

\n

Firecracker is a Virtual Machine Monitor (VMM) that leans on Linux's KVM functionality to run VMs. Its beauty is in its simplicity, however even though it doesn't need a lot, you will need KVM to be available. If you have a bare-metal machine, like your own PC, or an old server or laptop, you're all set. There's also plenty of options for bare-metal in the cloud - billed either on a per minute/hour basis or per month.

\n

And finally, for quick testing, DigitalOcean, GCP, and Azure all support what is known as \"Nested Virtualization\". That's where you obtain a VM, which itself can start further VMs, it's not as fast as bare-metal, but it's cheap and works.

\n

Finally, whilst Firecracker and actuated (our CI product) both support Arm, and Raspberry Pi, this tutorial is only available for `x86_64`` to keep the instructions simple.

\n

Provision the machine

\n

I'd recommend you use Ubuntu 22.04, so that you can copy and paste instructions from this tutorial.

\n

Install Docker CE

\n
curl -fsSL https://get.docker.com  | sudo sh\n
\n

Docker will be used to fetch an initial Operating System, to build the init system, and to customise the root filesystem.

\n

Install arkade, which gives you an easy way to install Firecracker:

\n
curl -sLS https://get.arkade.dev | sudo sh\n
\n

Install Firecracker:

\n
sudo arkade system install firecracker\n
\n

Clone the lab

\n

Clone the lab onto the machine:

\n
git clone https://github.com/alexellis/firecracker-init-lab --depth=1\n\ncd firecracker-init-lab\n
\n

Update the networking script

\n

Find out what the primary interface is on the machine using ip addr or ifconfig.

\n

Edit ./setup-networking.sh:

\n
IFNAME=enp8s0\n
\n

The script will configure a TAP device which bridges microVMs to your host, then sets up IP forwarding and masquerading so that the microVMs can access the Internet.

\n

Run ./setup-networking.sh to setup the TAP device.

\n

Download the Kernel

\n

Add make via build-essential:

\n
sudo apt update && sudo apt install -y build-essential\n
\n

Run make kernel to download the quickstart Kernel made available by the Firecracker team. Of course, you can build your own, but bear in mind that Firecracker does not have PCI support, so many of the ones you'll find on the Internet will not be appropriate.

\n

This Makefile target will not actually build a new Kernel, but wil download one that the Firecracker team have pre-built and uploaded to S3.

\n

Make the container image

\n

Here's the Dockerfile we'll use to build the init system in a multi-stage build, then derive from Alpine Linux for the runtime, this could of course be anything like Ubuntu 22.04, Python, or Node.

\n

./Dockerfile:

\n
FROM golang:1.20-alpine as build\n\nWORKDIR /go/src/github.com/alexellis/firecracker-init-lab/init\n\nCOPY init .\n\nRUN go build --tags netgo --ldflags '-s -w -extldflags \"-lm -lstdc++ -static\"' -o init main.go\n\nFROM alpine:3.18 as runtime\n\nRUN apk add --no-cache curl ca-certificates htop\n\nCOPY --from=build /go/src/github.com/alexellis/firecracker-init-lab/init/init /init\n
\n

I've added in a few extra packages to play with.

\n

Run make root, and you'll see an image in your library:

\n
docker images | grep alexellis2/custom-init\n\nREPOSITORY                                                          TAG                                                                      IMAGE ID       CREATED         SIZE\nalexellis2/custom-init                                              latest                                                                   f89aa7f3dd27   20 hours ago    13.7MB\n
\n

Build the disk image

\n

Firecracker needs a disk image, or an existing block device as its boot drive. You can make this dynamically as required, run make extract to extract the container image into the local filesystem as rootfs.tar.

\n

This step uses docker create followed by docker export to create a temporary container, and then to save its filesystem contents into a tar file.

\n

Run make extract

\n

If you want to see what a filesystem looks like, you could extract rootfs.tar into /tmp and have a poke around. This is not a required step.

\n

Then run make image.

\n

Here, a loopback file allocated with 5GB, then formatted as ext4, under the name rootfs.img. The script mounts the drive and then extracts the contents of the rootfs.tar file into it before unmounting the file.

\n

Start a Firecracker process

\n

Now, this may feel a little odd or different to Docker users. For each Firecracker VM you want to launch, you'll need to start a process, configure it via curl over a UNIX socket, then issue a boot command.

\n

To run multiple Firecracker microVMs at once, configure a different socket path for each.

\n
make start\n
\n

Boot the microVM

\n

In another window, issue the boot command:

\n
make boot\n
\n

Explore the system

\n

You're now booted into a serial console, this isn't a fully functional TTY, so some things won't work like Control + C. The serial console is really just designed for showing boot-up information, not interactive use. For proper remote administration, you should install an OpenSSH server and then connect to the VM using its IP address.

\n

That said, you can now explore a little.

\n

Add a DNS server to /etc/resolv.conf:

\n
echo \"nameserver 8.8.8.8\" > /etc/resolv.conf\n
\n

Then try to reach the Internet:

\n
ping -c 1 8.8.8.8\n\nping -c 4 google.com\n\ncurl --connect-timeout 1 -4 -i http://captive.apple.com/\n\ncurl --connect-timeout 1 -4 -i https://inlets.dev\n
\n

Check out the system specifications:

\n
free -m\ncat /proc/cpuinfo\nip addr\nip route\n
\n

When you're done, kill the firecracker process with sudo killall firecracker, or type in halt to the serial console.

\n

Wrapping up

\n

I was frustrated by the lack of a simple guide for tinkering with Firecracker, and so that's why I wrote this lab and am keeping it up to date.

\n

For production use, you could use a HTTP client to make the API requests to the UNIX socket, or an SDK, which abstracts away some of the complexity. There's an official SDK for Go and several unofficial ones for Rust. If you look at the sample code for either, you'll see that they are doing the same things we did in the lab, so you should find it relatively easy to convert the lab to use an SDK instead.

\n

Did you enjoy the lab? Have you got a use-case for Firecracker? Let me know on Twitter @alexellisuk

\n

If you'd like to see how we've applied Firecracker to bring fast and secure CI to teams, check out our product actuated.com

\n

Here's a quick demo of our control-plane, scheduler and bare-metal agent in action:

\n","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/github-actions-usage-cli.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/github-actions-usage-cli.json new file mode 100644 index 00000000..79ed64ed --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/github-actions-usage-cli.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","contentHtml":"

Whenever GitHub Actions users get in touch with us to ask about actuated, we ask them a number of questions. What do you build? What pain points have you been running into? What are you currently spending? And then - how many minutes are you using?

\n

That final question is a hard one for many to answer because the GitHub UI and API will only show billable minutes. Why is that a problem? Some teams only use open-source repositories with free runners. Others may have a large free allowance of credit for one reason or another and so also don't really know what they're using. Then you have people who already use some form of self-hosted runners - they are also excluded from what GitHub shows you.

\n

So we built an Open Source CLI tool called actions-usage to generate a report of your total minutes by querying GitHub's REST API.

\n

And over time, we had requests to break-down per day - so for our customers in the Middle East like Kubiya, it's common to see a very busy day on Sunday, and not a lot of action on Friday. Given that some teams use mono-repos, we also added the ability to break-down per repository - so you can see which ones are the most active. And finally, we added the ability to see hot-spots of usage like the longest running repo or the most active day.

\n

You can run the tool in three ways:

\n\n

I'll show you each briefly, but the one I like the most is the third option because it's kind of recursive.

\n

Before we get started, download arkade, and use it to install the tool:

\n
# Move the binary yourself into $PATH\ncurl -sLS https://get.arkade.dev | sh\n\n# Have sudo move it for you\ncurl -sLS https://get.arkade.dev | sudo sh\narkade install actions-usage\n
\n

Or if you prefer - you can add my brew tap, or head over to the arkade releases page.

\n

Later on, I'll also show you how to use the alexellis/arkade-get action to install the tool for CI.

\n

Finding out about your organisation

\n

If you want to find out about your organisation, you can run the tool like this:

\n
actions-usage \\\n    -org $GITHUB_REPOSITORY_OWNER \\\n    -days 28 \\\n    -by-repo \\\n    -punch-card \\\n    -token-file ~/PAT.txt\n
\n

You'll need a Personal Access Token, there are instructions on how to create this in the actions-usage README file

\n

There are many log lines printed to stderr during the scan of repositories and the workflows. You can omit all of this by adding 2> /dev/null to the command.

\n

First off we show the totals:

\n
Fetching last 28 days of data (created>=2023-05-19)\n\nGenerated by: https://github.com/self-actuated/actions-usage\nReport for actuated-samples - last: 28 days.\n\nTotal repos: 24\nTotal private repos: 0\nTotal public repos: 24\n\nTotal workflow runs: 107\nTotal workflow jobs: 488\n\nTotal users: 1\n
\n

Then break down on success/failure and cancelled jobs overall, plus the biggest and average build time:

\n
Success: 369/488\nFailure: 45/488\nCancelled: 73/488\n\nLongest build: 29m32s\nAverage build time: 1m26s\n
\n

Next we have the day by day breakdown. You can see that we try to focus on our families on Sunday at OpenFaaS Ltd, instead of working:

\n
Day            Builds\nMonday         61\nTuesday        50\nWednesday      103\nThursday       110\nFriday         153\nSaturday       10\nSunday         0\nTotal          488\n
\n

Our customers in the Middle East work to a different week, and so you'd see Saturday with no builds or nothing, and Sunday like a normal working day.

\n

Then we have the repo-by-repo breakdown with some much more granular data:

\n
Repo                                      Builds         Success        Failure        Cancelled      Skipped        Total          Average        Longest\nactuated-samples/k3sup-matrix-test        355            273            20             62             0              2h59m1s        30s            1m29s\nactuated-samples/discourse                49             38             7              4              0              6h37m21s       8m7s           20m1s\nactuated-samples/specs                    35             31             1              3              0              10m20s         18s            32s\nactuated-samples/cypress-test             17             4              13             0              0              6m23s          23s            49s\nactuated-samples/cilium-test              9              4              2              3              0              1h10m41s       7m51s          29m32s\nactuated-samples/kernel-builder-linux-6.0 9              9              0              0              0              11m28s         1m16s          1m27s\nactuated-samples/actions-usage-job        8              4              2              1              0              46s            6s             11s\nactuated-samples/faasd-nix                6              6              0              0              0              24m20s         4m3s           10m49s\n
\n

Finally, we have the original value that the tool set out to display:

\n
Total usage: 11h40m20s (700 mins)\n
\n

We display the value in a Go duration for readability and in minutes because that's the number that GitHub uses to talk about usage.

\n

One customer told us that they were running into rate limits when querying for 28 days of data, so they dropped down to 14 days and then multiplied the result by two to get a rough estimate.

\n
-days 14 \n
\n

The team at Todoist got in touch with us to see if actuated could reduce their bill on GitHub Actions. When he tried to run the tool the rate-limit was exhausted even when he changed the flag to -days 1. Why? They were using 550,000 minutes!

\n

So we can see one of the limitations already of this approach. Fortunately, actuated customers have their job stats recorded in a database and can generate reports from the dashboard very quickly.

\n

Finding out about your personal account

\n

Actuated isn't built for personal users, but for teams, so we didn't add this feature initially. Then we saw a few people reach out via Twitter and GitHub and decided to add it for them.

\n

For your personal account, you only have to change one of the input parameters:

\n
actions-usage \\\n    -user alexellis \\\n    -days 28 \\\n    -by-repo \\\n    -punch-card \\\n    -token-file ~/ae-pat.txt 2> /dev/null \n
\n

Now I actually have > 250 repositories and most of them don't even have Actions enabled, so this makes the tool less useful for me personally. So it was great when a community member suggested offering a way to filter repos when you have so many that the tool takes a long time to run or can't complete due to rate-limits.

\n

Being that today I used it to get the same insights from a Github Org where I currently work, which contains 1.4k of repositories.

And this was running for a considerable time. I am only related to only a few repositories within this organization.

— lbrealdeveloper (@lbrealdeveloper) June 15, 2023
\n

I've already created an issue and have found someone who'd like to contribute the change: Offer a way to filter repos for large organisations / users #8

\n

This is the beauty of open source and community. We all get to benefit from each other's ideas and contributions.

\n

Running actions-usage with a GitHub Action

\n

Now this is my favourite way to run the tool. I can run it on a schedule and get a report sent to me via email or Slack.

\n

\"Example

\n
\n

Example output from running the tool as a GitHub Action

\n
\n

Create actions-usage.yaml in your .github/workflows folder:

\n
name: actions-usage\n\non:\n  push:\n    branches:\n      - master\n      - main\n  workflow_dispatch:\n\npermissions:\n  actions: read\n\njobs:\n  actions-usage:\n    name: daily-stats\n    runs-on: actuated-any-1cpu-2gb\n    steps:\n    - uses: alexellis/arkade-get@master\n      with:\n        actions-usage: latest\n        print-summary: false\n    - name: Generate actions-usage report\n      run: |\n       echo \"### Actions Usage report by [actuated.dev](https://actuated.com)\" >> SUMMARY\n       echo \"\\`\\`\\`\" >> SUMMARY\n       actions-usage \\\n        -org $GITHUB_REPOSITORY_OWNER \\\n        -days 1 \\\n        -by-repo \\\n        -punch-card \\\n        -token ${{ secrets.GITHUB_TOKEN }}  2> /dev/null  >> SUMMARY\n       echo \"\\`\\`\\`\" >> SUMMARY\n       cat SUMMARY >> $GITHUB_STEP_SUMMARY\n
\n

Ths is designed to run within an organisation, but you can change the -org flag to -user and then use your own username.

\n

The days are for the past day of activity, but you can change this to any number like 7, 14 or 28 days.

\n

You can learn about the other flags by running actions-usage --help on your own machine.

\n

Wrapping up

\n

actions-usage is a practical tool that we use with customers to get an idea of their usage and how we can help with actuated. That said, it's also a completely free and open source tool for which the community is finding their own set of use-cases.

\n

And there are no worries about privacy, we've gone very low tech here. The output is only printed to the console, and we never receive any of your data unless you specifically copy and paste the output into an email.

\n

Feel free to create an issue if you have a feature request or a question.

\n

Check out self-actuated/actions-usage on GitHub

\n

I wrote an eBook writing CLIs like this in Go and keep it up to date on a regular basis - adding new examples and features of Go.

\n

Why not check out what people are saying about it on Gumroad?

\n

Everyday Go - The Fast Track

","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/gpus-for-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/gpus-for-github-actions.json new file mode 100644 index 00000000..a81a3938 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/gpus-for-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","contentHtml":"

With the surge of interest in AI and machine learning models, it's not hard to think of reasons why people want GPUs in their workstations and production environments. They make building & training, fine-tuning and serving (inference) from a machine learning model just that much quicker than running with a CPU alone.

\n

So if you build and test code for CPUs in CI pipelines like GitHub Actions, why wouldn't you do the same with code built for GPUs? Why exercise only a portion of your codebase?

\n

GPUs for GitHub Actions

\n

One of our earliest customers moved all their GitHub Actions to actuated for a team of around 30 people, but since Firecracker has no support for GPUs, they had to keep a few self-hosted runners around for testing their models. Their second hand Dell servers were racked in their own datacentre, with 8x 3090 GPUs in each machine.

\n

Their request for GPU support in actuated predated the hype around OpenAI, and was the catalyst for us doing this work.

\n

They told us how many issues they had keeping drivers in sync when trying to use self-hosted runners, and the security issues they ran into with mounting Docker sockets or running privileged containers in Kubernetes.

\n

With a microVM and actuated, you'll be able to test out different versions of drivers as you see fit, and know there will never be side-effects between builds. You can read more in our FAQ on how actuated differs from other solutions which rely on the poor isolation afforded by containers. Actuated is the closest you can get to a hosted runner, whilst having full access to your own hardware.

\n

I'll tell you a bit more about it, how to build your own workstation with commodity hardware, or where to rent a powerful bare-metal host with a capable GPU for less than 200 USD / mo that you can use with actuated.

\n

Available in early access

\n

So today, we're announcing early access to actuated for GPUs. Whether your machine has one GPU, two, or ten, you can allocate them directly to a microVM for a CI job, giving strong isolation, and the same ephemeral environment that you're used to with GitHub's hosted runners.

\n

\"Our

\n
\n

Our test rig has 2x Nvidia 3060 GPUs and is available for customer demos and early testing.

\n
\n

We've compiled a list of vendors that provide access to fast, bare-metal compute, but at the moment, there are only a few options for bare-metal with GPUs.

\n\n

We have a full bill of materials available for anyone who wants to build a workstation with 2x Nvidia 3060 graphics cards, giving 24GB of usage RAM at a relatively low maximum power consumption of 170W. It's ideal for CI and end to end testing.

\n

If you'd like to go even more premium, the Nvidia RTX 4000 card comes with 20GB of RAM, so two of those would give you 40GB of RAM available for Large Language Models (LLMs).

\n

For Hetzner, you can get started with an i5 bare-metal host with 14 cores, 64GB RAM and a dedicated Nvidia RTX 4000 for around 184 EUR / mo (less than 200 USD / mo). If that sounds like ridiculously good value, it's because it is.

\n

What does the build look like?

\n

Once you've installed the actuated agent, it's the same process as a regular bare-metal host.

\n

It'll show up on your actuated dashboard, and you can start sending jobs to it immediately.

\n

\"The

\n
\n

The server with 2x GPUs showing up in the dashboard

\n
\n

Here's how we install the Nvidia driver for a consumer-grade card. The process is very similar for the datacenter range of GPUs found in enterprise servers.

\n
name: nvidia-smi\n\njobs:\n    nvidia-smi:\n        name: nvidia-smi\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - name: Download Nvidia install package\n          run: |\n           curl -s -S -L -O https://us.download.nvidia.com/XFree86/Linux-x86_64/525.60.11/NVIDIA-Linux-x86_64-525.60.11.run \\\n              && chmod +x ./NVIDIA-Linux-x86_64-525.60.11.run\n        - name: Install Nvidia driver and Kernel module\n          run: |\n              sudo ./NVIDIA-Linux-x86_64-525.60.11.run \\\n                  --accept-license \\\n                  --ui=none \\\n                  --no-questions \\\n                  --no-x-check \\\n                  --no-check-for-alternate-installs \\\n                  --no-nouveau-check\n        - name: Run nvidia-smi\n          run: |\n            nvidia-smi\n
\n

This is a very similar approach to installing a driver on your own machine, just without any interactive prompts. It took around 38s which is not very long considering how much time AI and ML operations can run for when doing end to end testing. The process installs some binaries like nvidia-smi and compiles a Kernel module to load the graphics driver, these could easily be cached with GitHub Action's built-in caching mechanism.

\n

For convenience, we created a composite action that reduces the duplication if you have lots of workflows with the Nvidia driver installed.

\n
name: gpu-job\n\njobs:\n    gpu-job:\n        name: gpu-job\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Run nvidia-smi\n          run: |\n            nvidia-smi\n
\n

Of course, if you have an AMD graphics card, or even an ML accelerator like a PCIe Google Corale, that can also be passed through into a VM in a dedicated way.

\n

The mechanism being used is called VFIO, and allows a VM to take full, dedicated, isolated control over a PCI device.

\n

A quick example to get started

\n

To show the difference between using a GPU and CPU, I ran OpenAI's Whisper project, which transcribes audio or video to a text file.

\n

With the following demo video of Actuated's SSH gateway, running with the tiny model.

\n\n

That's over 2x quicker, for a 5:34 minute video. If you process a lot of clips, or much longer clips then the difference may be even more marked.

\n

The tiny model is really designed for demos, and in production you'd use the medium or large model which is much more resource intensive.

\n

Here's a screenshot showing what this looks like with the medium model, which is much larger and more accurate:

\n

Medium model running on a GPU via actuated

\n
\n

Medium model running on a GPU via actuated

\n
\n

With a CPU, even with 16 vCPU, all of them get pinned at 100%, and then it takes a significantly longer time to process.

\n

\"You

\n
\n

You can run the medium model on CPU, but would you want to?

\n
\n

With the medium model:

\n\n

The GPU increased the speed by 9x, imagine how much quicker it'd be if you used an Nvidia 3090, 4090, or even an RTX 4000.

\n

If you want to just explore the system, and run commands interactively, you can use actuated's SSH feature to get a shell. Once you know the commands you want to run, you can copy them into your workflow YAML file for GitHub Actions.

\n

We took the SSH debug session for a test-drive. We installed the NVIDIA Container Toolkit, then ran the ollama tool to test out some Large Language Models (LLMs).

\n

Ollama is an open source tool for downloading and testing prepackaged models like Mistral or Llama2.

\n

\"Our

\n
\n

Our experiment with ollama within a GitHub Actions runner

\n
\n

The technical details

\n

Since launch, actuated powered by Firecracker has securely isolated over 220k CI jobs for GitHub Actions users. Whilst it's a complex project to integrate, it has been very reliable in production.

\n

Now in order to bring GPUs to actuated, we needed to add support for a second Virtual Machine Manager (VMM), and we picked cloud-hypervisor.

\n

cloud-hypervisor was originally a fork from Firecracker and shares a significant amount of code. One place it diverged was adding support for PCI devices, such as GPUs. Through VFIO, cloud-hypervisor allows for a GPU to be passed through to a VM in a dedicated way, so it can be used in isolation.

\n

Here's the first demo that I ran when we had everything working, showing the output from nvidia-smi:

\n

\"The

\n
\n

The first run of nvidia-smi

\n
\n

Reach out for more

\n

In a relatively short period of time, we were able to update our codebase to support both Firecracker and cloud-hypervisor, and to enable consumer-grade GPUs to be passed through to VMs in isolation.

\n

You can rent a really powerful and capable machine from Hetzner for under 200 USD / mo, or build your own workstation with dual graphics cards like our demo rig, for less than 2000 USD and then you own that and can use it as much as you want, plugged in under your desk or left in a cabinet in your office.

\n

A quick recap on use-cases

\n

Let's say you want to run end to end tests for an application that uses a GPU? Perhaps it runs on Kubernetes? You can do that.

\n

Do you want to fine-tune, train, or run a batch of inferences on a model? You can do that. GitHub Actions has a 6 hour timeout, which is plenty for many tasks.

\n

Would it make sense to run Stable Diffusion in the background, with different versions, different inputs, across a matrix? GitHub Actions makes that easy, and actuated can manage the GPU allocations for you.

\n

Do you run inference from OpenFaaS functions? We have a tutorial on OpenAI Whisper within a function with GPU acceleration here and a separate one on how to serve Server Sent Events (SSE) from OpenAI or self-hosted models, which is popular for chat-style interfaces to AI models.

\n

If you're interested in GPU support for GitHub Actions, then reach out to talk to us with this form.

","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/how-to-run-multi-arch-builds-natively.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/how-to-run-multi-arch-builds-natively.json new file mode 100644 index 00000000..88c9337f --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/how-to-run-multi-arch-builds-natively.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","contentHtml":"

In two previous articles, we covered huge improvements in performance for the Parca project and VPP (Network Service Mesh) simply by switching to actuated with Arm64 runners instead of using QEMU and hosted runners.

\n

In the first case, using QEMU took over 33 minutes, and bare-metal Arm showed a 22x improvement at only 1 minute 26 seconds. For Network Service Mesh, VPP couldn't even complete a build in 6 hours using QEMU - and I got it down to 9 minutes flat using a bare-metal Ampere Altra server.

\n

What are we going to see and why is it better?

\n

In this article, I'll show you how to run multi-arch builds natively on bare-metal hardware using GitHub Actions and actuated.

\n

Actuated is a SaaS service that we built so that you can Bring Your Own compute to GitHub Actions, and have every build run in an immutable, single-use VM.

\n

\"Comparison

\n
\n

Comparison of splitting out to run in parallel on native hardware and QEMU.

\n
\n

Not every build will see such a dramatic increase as the ones I mentioned in the introduction. Here, with the inlets-operator, we gained 4 minutes on each commit. But I often speak to users who are running past 30 minutes to over an hour because of QEMU.

\n

Three things got us a speed bump here:

\n\n

Only last week an engineer at Calyptia (the team behind fluent-bit) reached out for help after telling me they had to disable and stop publishing open source images for Arm, it was simply timing out at the 6 hour mark.

\n

So how does this thing work, and is QEMU actually \"OK\"?

\n

QEMU can be slow, but it's actually \"OK\"

\n

So if the timings are so bad, why does anyone use QEMU?

\n

Well it's free - as in beer, there's no cost at all to use it. And many builds can complete in a reasonable amount of time using QEMU, even if it's not as fast as native.

\n

That's why we wrote up how we build 80+ multi-arch images for various products like OpenFaaS and Inlets:

\n

The efficient way to publish multi-arch containers from GitHub Actions

\n

Here's what the build looks like with QEMU:

\n
name: split-operator\n\non:\n  push:\n    branches: [ master, qemu ]\n\njobs:\n\n  publish_qemu:\n    concurrency: \n      group: ${{ github.ref }}-qemu\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v2\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/amd64,linux/arm64\n          file: ./Dockerfile\n          context: .\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          provenance: false\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-qemu\n
\n

This is the kind of build that is failing or causing serious delays for projects like Parca, VPP and Fluent Bit.

\n

Let's look at the alternative.

\n

Building on native hardware

\n

Whilst QEMU emulates the architecture you need within a build, it's not the same as running on the real hardware. This is why we see such a big difference in performance.

\n

The downside is that we have to write a bit more CI configuration and run two builds instead of one, but there is some good news - we can now run them in parallel.

\n

In parallel we:

\n
    \n
  1. We publish the x86_64 image - ghcr.io/owner/repo:sha-amd64
    2. \n
  2. We publish the ARM image - ghcr.io/owner/repo:sha-arm64
    3. \n
\n

Then:

\n
    \n
  1. We create a manifest with its own name - ghcr.io/owner/repo:sha
    2. \n
  2. We annotate the manifest with the images we built earlier
    3. \n
  3. We push the manifest
    4. \n
\n

In this way, anyone can pull the image with the name ghcr.io/owner/repo:sha and it will map to either of the two images for Arm64 or Amd64.

\n

\"Parallel

\n
\n

The two builds on the left ran on two separate bare-metal hosts, and the manifest was published using one of GitHub's hosted runners.

\n
\n

Here's a sample for the inlets-operator, a Go binary which connects to the Kubernetes API.

\n

First up, we have the x86 build:

\n
name: split-operator\n\non:\n  push:\n    branches: [ master ]\n\njobs:\n\n  publish_x86:\n    concurrency: \n      group: ${{ github.ref }}-x86\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Setup mirror\n        uses: self-actuated/hub-mirror@master\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/amd64\n          file: ./Dockerfile\n          context: .\n          provenance: false\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64\n
\n

Then we have the arm64 build which is almost identical, but we specify a different value for platforms and the runs-on field.

\n
\n  publish_aarch64:\n    concurrency: \n      group: ${{ github.ref }}-aarch64\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: actuated-aarch64\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Setup mirror\n        uses: self-actuated/hub-mirror@master\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/arm64\n          file: ./Dockerfile\n          context: .\n          provenance: false\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n
\n

Finally, we need to create the manifest. GitHub Actions has a needs variable that we can set to control the execution order:

\n
  publish_manifest:\n    runs-on: ubuntu-latest\n    needs: [publish_x86, publish_aarch64]\n    steps:\n\n    - name: Get Repo Owner\n      id: get_repo_owner\n      run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n    - name: Login to container Registry\n      uses: docker/login-action@v2\n      with:\n        username: ${{ github.repository_owner }}\n        password: ${{ secrets.GITHUB_TOKEN }}\n        registry: ghcr.io\n\n    - name: Create manifest\n      run: |\n        docker manifest create ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} \\\n          --amend ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64 \\\n          --amend ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n        docker manifest annotate --arch amd64 --os linux ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64\n        docker manifest annotate --arch arm64 --os linux ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n        docker manifest inspect ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n\n        docker manifest push ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n
\n

One thing I really dislike about this final stage is how much repetition we get. Fortunately, it's relatively simple to hide this complexity behind a custom GitHub Action.

\n

Note that this is just an example at the moment, but I could make a custom composite action in Bash in about 30 minutes, including testing. So it's not a lot of work and it would make our whole workflow a lot less repetitive.

\n
   uses: self-actuated/compile-manifest@master\n    with:\n      image: ghcr.io/${{ env.REPO_OWNER }}/inlets-operator\n      sha: ${{ github.sha }}\n      platforms: amd64,arm64\n
\n

As a final note, we recently saw that with upgrading from docker/build-push-action@v3 to docker/build-push-action@v4, buildx no longer publishes an image, but a manifest for each architecture. This is because a new \"provenance\" feature is enabled which under the hood is publishing multiple artifacts instead of a single image. We've turned this off with provenance: false and are awaiting a response from Docker on how to enable provenance for multi-arch images built with a split build.

\n

Wrapping up

\n

Yesterday we took a new customer on for actuated who wanted to improve the speed of Arm builds, but on the call we both knew they would need to leave QEMU behind. I put this write-up together to show what would be involved, and I hope it's useful to you.

\n

Where can you run these builds?

\n

Couldn't you just add a low-cost Arm VM from AWS, Oracle Cloud, Azure or Google Cloud?

\n

The answer unfortunately is no.

\n

The self-hosted runner is not suitable for open source / public repositories, the GitHub documentation has a stark warning about this.

\n

The Kubernetes controller that's available has the same issues, because it re-uses the Pods by default, and runs in a dangerous Docker In Docker Mode as a privileged container or by mounting the Docker Socket. I'm not sure which is worse, but both mean that code in CI can take over the host, potentially even the whole cluster.

\n

Hosted runners solve this by creating a fresh VM per job, and destroying it immediately. That's the same approach that we took with actuated, but you get to bring your own metal along, so that you keep costs from growing out of control. Actuated also supports Arm, out of the box.

\n

Want to know more about the security of self-hosted runners? Read more in our FAQ.

\n

Want to talk to us about your CI/CD needs? We're happy to help.

\n","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/is-the-self-hosted-runner-safe-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/is-the-self-hosted-runner-safe-github-actions.json new file mode 100644 index 00000000..d030169e --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/is-the-self-hosted-runner-safe-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","contentHtml":"

First of all, why would someone working on an open source project need a self-hosted runner?

\n

Having contributed to dozens of open source projects, and gotten to know many different maintainers, the primary reason tends to be out of necessity. They face an 18 minute build time upon every commit or Pull Request revision, and want to make the best of what little time they can give over to Open Source.

\n

Having faster builds also lowers friction for contributors, and since many contributors are unpaid and rely on their own internal drive and enthusiasm, a fast build time can be the difference between them fixing a broken test or waiting another few days.

\n

To sum up, there are probably just a few main reasons:

\n
    \n
  1. Faster builds, higher concurrency, more disk space
    2. \n
  2. Needing to build and test Arm binaries or containers on real hardware
    3. \n
  3. Access to services on private networks
    4. \n
\n

The first point is probably one most people can relate to. Simply by provisioning an AMD bare-metal host, or a high spec VM with NVMe, you can probably shave minutes off a build.

\n

For the second case, some projects like Flagger from the CNCF felt their only option to support users deploying to AWS Graviton, was to seek sponsorship for a large Arm server and to install a self-hosted runner on it.

\n

The third option is more nuanced, and specialist. This may or may not be something you can relate to, but it's worth mentioning. VPNs have very limited speed and there may be significant bandwidth costs to transfer data out of a region into GitHub's hosted runner environment. Self-hosted runners eliminate the cost and give full local link bandwidth, even as high as 10GbE. You just won't get anywhere near that with IPSec or Wireguard over the public Internet.

\n

Just a couple of days ago Ed Warnicke, Distinguished Engineer at Cisco reached out to us to pilot actuated. Why?

\n

Ed, who had Network Service Mesh in mind said:

\n
\n

I'd kill for proper Arm support. I'd love to be able to build our many containers for Arm natively, and run our KIND based testing on Arm natively.\nWe want to build for Arm - Arm builds is what brought us to actuated

\n
\n

But are self-hosted runners safe?

\n

The GitHub team has a stark warning for those of us who are tempted to deploy a self-hosted runner and to connect it to a public repository.

\n
\n

Untrusted workflows running on your self-hosted runner pose significant security risks for your machine and network environment, especially if your machine persists its environment between jobs. Some of the risks include:

\n\n
\n

See also: Self-hosted runner security

\n

Now you may be thinking \"I won't approve pull requests from bad actors\", but quite often the workflow goes this way: the contributor gets approval, then you don't need to approve subsequent pull requests after that.

\n

An additional risk is if that user's account is compromised, then the attacker can submit a pull request with malicious code or malware. There is no way in GitHub to enforce Multi-Factor Authentication (MFA) for pull requests, even if you have it enabled on your Open Source Organisation.

\n

Here are a few points to consider:

\n\n

The chances are that if you're running the Flagger or Network Service Mesh project, you are shipping code that enterprise companies will deploy in production with sensitive customer data.

\n

If you are not worried, try explaining the above to them, to see how they may see the risk differently.

\n

Doesn't Kubernetes fix all of this?

\n

Kubernetes is a well known platform built for orchestrating containers. It's especially suited to running microservices, webpages and APIs, but has support for batch-style workloads like CI runners too.

\n

You could make a container image and install the self-hosted runner binary within in, then deploy that as a Pod to a cluster. You could even scale it up with a few replicas.

\n

If you are only building Java code, Python or Node.js, you may find this resolves many of the issues that we covered above, but it's hard to scale, and you still get side-effects as the environment is not immutable.

\n

That's where the community project \"actions-runtime-controller\" or ARC comes in. It's a controller that launches a pool of Pods with the self-hosted runner.

\n
\n

How much work does ARC need?

\n

Some of the teams I have interviewed over the past 3 months told me that ARC took them a lot of time to set up and maintain, whilst others have told us it was a lot easier for them. It may depend on your use-case, and whether you're more of a personal user, or part of a team with 10-30 people committing code several times per day.\nThe first customer for actuated, which I'll mention later in the article was a team of ~ 20 people who were using ARC and had grew tired of the maintenance overhead and certain reliability issues.

\n
\n

Unfortunately, by default ARC uses the same Pod many times as a persistent runner, so side effects still build up, malware can still be introduced and you have to maintain a Docker image with all the software needed for your builds.

\n

You may be happy with those trade-offs, especially if you're only building private repositories.

\n

But those trade-offs gets a lot worse if you use Docker or Kubernetes.

\n

Out of the box, you simply cannot start a Docker container, build a container image or start a Kubernetes cluster.

\n

And to do so, you'll need to resort to what can only be described as dangerous hacks:

\n
    \n
  1. You expose the Docker socket from the host, and mount it into each Pod - any CI job can take over the host, game over.
    2. \n
  2. You run in Docker in Docker (DIND) mode. DIND requires a privileged Pod, which means that any CI job can take over the host, game over.
    3. \n
\n

There is some early work on running Docker In Docker in user-space mode, but this is slow, tricky to set up and complicated. By default, user-space mode uses a non-root account. So you can't install software packages or run commands like apt-get.

\n

See also: Using Docker-in-Docker for your CI or testing environment? Think twice.

\n

Have you heard of Kaniko?

\n

Kaniko is a tool for building container images from a Dockerfile, without the need for a Docker daemon. It's a great option, but it's not a replacement for running containers, it can only build them.

\n

And when it builds them, in nearly every situation it will need root access in order to mount each layer to build up the image.

\n

See also: The easiest way to prove that root inside the container is also root on the host

\n

And what about Kubernetes?

\n

To run a KinD, Minikube or K3s cluster within your CI job, you're going to have to sort to one of the dangerous hacks we mentioned earlier which mean a bad actor could potentially take over the host.

\n

Some of you may be running these Kubernetes Pods in your production cluster, whilst others have taken some due diligence and deployed a separate cluster just for these CI workloads. I think that's a slightly better option, but it's still not ideal and requires even more access control and maintenance.

\n

Ultimately, there is a fine line between overconfidence and negligence. When building code on a public repository, we have to assume that the worst case scenario will happen one day. When using DinD or privileged containers, we're simply making that day come sooner.

\n

Containers are great for running internal microservices and Kubernetes excels here, but there is a reason that AWS insists on hard multi-tenancy with Virtual Machines for their customers.

\n
\n

See also: Firecracker whitepaper

\n
\n

What's the alternative?

\n

When GitHub cautioned us against using self-hosted runners, on public repos, they also said:

\n
\n

This is not an issue with GitHub-hosted runners because each GitHub-hosted runner is always a clean isolated virtual machine, and it is destroyed at the end of the job execution.

\n
\n

So using GitHub's hosted runners are probably the most secure option for Open Source projects and for public repositories - if you are happy with the build speed, and don't need Arm runners.

\n

But that's why I'm writing this post, sometimes we need faster builds, or access to specialist hardware like Arm servers.

\n

The Kubernetes solution is fast, but it uses a Pod which runs many jobs, and in order to make it useful enough to run docker run, docker build or to start a Kubernetes cluster, we have to make our machines vulnerable.

\n

With actuated, we set out to re-build the same user experience as GitHub's hosted runners, but without the downsides of self-hosted runners or using Kubernetes Pods for runners.

\n

Actuated runs each build in a microVM on servers that you alone provision and control.

\n

Its centralised control-plane schedules microVMs to each server using an immutable Operating System that is re-built with automation and kept up to date with the latest security patches.

\n

Once the microVM has launched, it connects to GitHub, receives a job, runs to completion and is completely erased thereafter.

\n

You get all of the upsides of self-hosted runners, with a user experience that is as close to GitHub's hosted runners as possible.

\n

Pictured - an Arm Server with 270 GB of RAM and 80 cores - that's a lot of builds.

\n

\"\"

\n

You get to run the following, without worrying about security or side-effects:

\n\n

Need to test against a dozen different Kubernetes versions?

\n

Not a problem:

\n

\"Testing

\n

What about running the same on Arm servers?

\n

Just change runs-on: actuated to runs-on: actuated-aarch64 and you're good to go. We test and maintain support for Docker and Kubernetes for both Intel and Arm CPU architectures.

\n

Do you need insights for your Open Source Program Office (OSPO) or for the Technical Steering Committee (TSC)?

\n

\"\"

\n

We know that no open source project has a single repository that represents all of its activity. Actuated provides insights across an organisation, including total build time and the time queued - which is a reflection of whether you could do with more or fewer build machines.

\n

And we are only just getting started with compiling insights, there's a lot more to come.

\n

Get involved today

\n

We've already launched 10,000 VMs for customers jobs, and are now ready to open up the platform to the wider community. So if you'd like to try out what we're offering, we'd love to hear from you. As you offer feedback, you'll get hands on support from our engineering team and get to shape the product through collaboration.

\n

So what does it cost? There is a subscription fee which includes - the control plane for your organisation, the agent software, maintenance of the OS images and our support via Slack. But all the plans are flat-rate, so it may even work out cheaper than paying GitHub for the bigger instances that they offer.

\n

Professional Open Source developers like the ones you see at Red Hat, VMware, Google and IBM, that know how to work in community and understand cloud native are highly sought after and paid exceptionally well. So the open source project you work on has professional full-time engineers allocated to it by one or more companies, as is often the case, then using actuated could pay for itself in a short period of time.

\n

If you represent an open source project that has no funding and is purely maintained by volunteers, what we have to offer may not be suited to your current position. And in that case, we'd recommend you stick with the slower GitHub Runners. Who knows? Perhaps one day GitHub may offer sponsored faster runners at no cost for certain projects?

\n

And finally, what if your repositories are private? Well, we've made you aware of the trade-offs with a static self-hosted runner, or running builds within Kubernetes. It's up to you to decide what's best for your team, and your customers. Actuated works just as well with private repositories as it does with public ones.

\n

See microVMs launching in ~ 1s during a matrix build for testing a Custom Resource Definition (CRD) on different Kubernetes versions:

\n\n

Want to know how actuated works? Read the FAQ for more technical details.

\n\n

Follow us on Twitter - selfactuated

","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/kvm-in-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/kvm-in-github-actions.json new file mode 100644 index 00000000..d6cdc227 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/kvm-in-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","contentHtml":"

GitHub's hosted runners do not support nested virtualization. This means some frequently used tools that require KVM like packer, the Android emulator, etc can not be used in GitHub Actions CI pipelines.

\n

We noticed there are quite a few issues for people requesting KVM support for GitHub Actions:

\n\n

As mentioned in some of these issues, an alternative would be to run your own self-hosted runner on a bare metal host. This comes with the downside that builds can conflict and cause side effects to system-level packages. On top if this self-hosted runners are considered insecure for public repositories.

\n

Solutions like the \"actions-runtime-controller\" or ARC that use Kubernetes to orchestrate and run self-hosted runners in Pods are also out of scope if you need to run VMs.

\n

With Actuated we make it possible to launch a Virtual Machine (VM) within a GitHub Action. Jobs are launched in isolated VMs just like GitHub hosted runners but with support for nested virtualization.

\n

Case Study: Githedgehog

\n

One of our customers Sergei Lukianov, founding engineer at Githedgehog told us he needed somewhere to build Docker images and to test them with Kubernetes, he uses KinD for that.

\n

Prior to adopting Actuated, his team used hosted runners which are considerably slower, and paid on a per minute basis. Actuated made his builds both faster, and more secure than using any of the alternatives for self-hosted runners.

\n

It turned out that he also needed to launch VMs in those jobs, and that's something else that hosted runners cannot cater for right now. Actuated’s KVM guest support means he can run all of his workloads on fast hardware.

\n

Some other common use cases that require KVM support on the CI runner:

\n\n

Running VMs in GitHub Actions

\n

In this section we will walk you through a couple of hands-on examples.

\n

Firecracker microVM

\n

In this example we are going to follow the Firecracker quickstart guide to boot up a Firecracker VM but instead of running it on our local machine we will run it from within a GitHub Actions workflow.

\n

The workflow instals Firecracker, configures and boots a guest VM and then waits 20 seconds before shutting down the VM and exiting the workflow. The image below shows the run logs of the workflow. We see the login prompt of the running microVM.

\n

\"Running

\n
\n

Running a firecracker microVM in a GitHub Actions job

\n
\n

Here is the workflow file used by this job:

\n
name: run-vm\n\non: push\njobs:\n  vm-run:\n    runs-on: actuated-4cpu-8gb\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: Install arkade\n        uses: alexellis/setup-arkade@v2\n      - name: Install firecracker\n        run: |\n          sudo arkade system install firecracker\n      - name: Run microVM\n        run: sudo -E ./run-vm.sh\n
\n

The setup-arkade is to install arkade on the runner. Next firecracker is installed from the arkade system apps.

\n

As a last step we run a firecracker microVM. The run-vm.sh script is based on the firecracker quickstart and collects all the steps into a single script that can be run in the CI pipeline.

\n

It script will:

\n\n

The run-vm.sh script:

\n
#!/bin/bash\n\n# Get a kernel and rootfs\narch=`uname -m`\ndest_kernel=\"hello-vmlinux.bin\"\ndest_rootfs=\"hello-rootfs.ext4\"\nimage_bucket_url=\"https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$arch\"\n\nif [ ${arch} = \"x86_64\" ]; then\n    kernel=\"${image_bucket_url}/kernels/vmlinux.bin\"\n    rootfs=\"${image_bucket_url}/rootfs/bionic.rootfs.ext4\"\nelif [ ${arch} = \"aarch64\" ]; then\n    kernel=\"${image_bucket_url}/kernels/vmlinux.bin\"\n    rootfs=\"${image_bucket_url}/rootfs/bionic.rootfs.ext4\"\nelse\n    echo \"Cannot run firecracker on $arch architecture!\"\n    exit 1\nfi\n\necho \"Downloading $kernel...\"\ncurl -fsSL -o $dest_kernel $kernel\n\necho \"Downloading $rootfs...\"\ncurl -fsSL -o $dest_rootfs $rootfs\n\necho \"Saved kernel file to $dest_kernel and root block device to $dest_rootfs.\"\n\n# Start firecracker\necho \"Starting firecracker\"\nfirecracker --api-sock /tmp/firecracker.socket &\nfirecracker_pid=$!\n\n# Set the guest kernel and rootfs\nrch=`uname -m`\nkernel_path=$(pwd)\"/hello-vmlinux.bin\"\n\nif [ ${arch} = \"x86_64\" ]; then\n    curl --unix-socket /tmp/firecracker.socket -i \\\n      -X PUT 'http://localhost/boot-source'   \\\n      -H 'Accept: application/json'           \\\n      -H 'Content-Type: application/json'     \\\n      -d \"{\n            \\\"kernel_image_path\\\": \\\"${kernel_path}\\\",\n            \\\"boot_args\\\": \\\"console=ttyS0 reboot=k panic=1 pci=off\\\"\n       }\"\nelif [ ${arch} = \"aarch64\" ]; then\n    curl --unix-socket /tmp/firecracker.socket -i \\\n      -X PUT 'http://localhost/boot-source'   \\\n      -H 'Accept: application/json'           \\\n      -H 'Content-Type: application/json'     \\\n      -d \"{\n            \\\"kernel_image_path\\\": \\\"${kernel_path}\\\",\n            \\\"boot_args\\\": \\\"keep_bootcon console=ttyS0 reboot=k panic=1 pci=off\\\"\n       }\"\nelse\n    echo \"Cannot run firecracker on $arch architecture!\"\n    exit 1\nfi\n\nrootfs_path=$(pwd)\"/hello-rootfs.ext4\"\ncurl --unix-socket /tmp/firecracker.socket -i \\\n  -X PUT 'http://localhost/drives/rootfs' \\\n  -H 'Accept: application/json'           \\\n  -H 'Content-Type: application/json'     \\\n  -d \"{\n        \\\"drive_id\\\": \\\"rootfs\\\",\n        \\\"path_on_host\\\": \\\"${rootfs_path}\\\",\n        \\\"is_root_device\\\": true,\n        \\\"is_read_only\\\": false\n   }\"\n\n# Start the guest machine\ncurl --unix-socket /tmp/firecracker.socket -i \\\n  -X PUT 'http://localhost/actions'       \\\n  -H  'Accept: application/json'          \\\n  -H  'Content-Type: application/json'    \\\n  -d '{\n      \"action_type\": \"InstanceStart\"\n   }'\n\n# Kill the firecracker process to exit the workflow\nsleep 20\nkill -9 $firecracker_pid\n\n
\n

The full example can be found on GitHub

\n

If you'd like to know more about how Firecracker works and how it compares to traditional VMs and Docker you can watch Alex's webinar on the topic.

\n\n
\n

Join Alex and Richard Case for a cracking time. The pair share what's got them so excited about Firecracker, the kinds of use-cases they see for microVMs, fundamentals of Linux Operating Systems and plenty of demos.

\n
\n

NixOS integration tests

\n

With nix there is the ability to provide a set of declarative configuration to define integration tests that spin up virtual machines using QEMU as the backend. While running these tests in CI without hardware acceleration is supported this is considerably slower.

\n

For a more detailed overview of the test setup and configuration see the original tutorial on nix.dev:

\n\n

The workflow file for running NixOS tests on GitHub Actions:

\n
name: nixos-tests\n\non: push\njobs:\n  nixos-test:\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - uses: actions/setup-python@v3\n        with:\n          python-version: '3.x'\n      - uses: cachix/install-nix-action@v16\n        with:\n          extra_nix_config: \"system-features = nixos-test benchmark big-parallel kvm\"\n      - name: NixOS test\n        run: nix build -L .#checks.x86_64-linux.postgres\n
\n

We just install Nix using the install-nix-action and run the tests in the next step.

\n

The full example is available on GitHub

\n

Other examples of using a VM

\n

In the previous section we showed you some brief examples for the kind of workflows you can run. Here are some other resources and tutorials that should be easy to adapt and run in CI.

\n\n

Conclusion

\n

Hosted runners do not support nested virtualization. That makes them unsuitable for running CI jobs that require KVM support.

\n

For Actuated runners we provide a custom Kernel that enables KVM support. This will allow you to run Virtual Machines within your CI jobs.

\n

At time of writing there is no support for aarch64 runners. Only Intel and AMD CPUs support nested virtualisation.

\n

While it is possible to deploy your own self-hosted runners to run jobs that need KVM support, this is not recommended:

\n\n

Want to see a demo or talk to our team? Contact us here

\n

Just want to try it out instead? Register your GitHub Organisation and set-up a subscription

","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/local-caching-for-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/local-caching-for-github-actions.json new file mode 100644 index 00000000..792c5f74 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/local-caching-for-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","contentHtml":"

We heard from the Discourse project last year because they were looking to speed up their builds. After trying out a couple of solutions that automated self-hosted runners, they found out that whilst faster CPUs were nice, reliability was a problem and the cache hosted on GitHub's network became the new bottleneck. We ran some tests to compare the hosted cache with hosted runners, to self-hosted with a local cache running with S3. This post will cover what we found.

\n\"Discourse\n
\n

Discourse is the online home for your community. We offer a 100% open source community platform to those who want complete control over how and where their site is run.

\n
\n

Set up a local cache

\n

Hosted runners are placed close to the cache which means the latency is very low. Self-hosted runners can also make good use of this cache but the added latency can negate the advantage of switching to these faster runners. Running a local S3 cache with Minio or Seaweedfs on the self hosted runner or in the same region/network can solve this problem.

\n

For this test we ran the cache on the runner host. Instructions to set up a local S3 cache with Seaweedfs can be found in our docs.

\n

The Discourse repo is already using the actions/cachein their tests workflow which makes it easy to switch out the official actions/cache with tespkg/actions-cache.

\n

The S3 cache is not directly compatible with the official actions/cache and some changes to the workflows are required to start using the cache.

\n

The tespkg/actions-cache supports the same properties as the actions cache and only requires some additional parameters to configure the S3 connection.

\n
 - name: Bundler cache\n-  uses: actions/cache@v3\n+  uses: tespkg/actions-cache@v1\n   with:\n+    endpoint: \"192.168.128.1\"\n+    port: 8333\n+    insecure: true\n+    accessKey: ${{ secrets.ACTIONS_CACHE_ACCESS_KEY }}\n+    secretKey: ${{ secrets.ACTIONS_CACHE_SECRET_KEY }}\n+    bucket: actuated-runners\n+    region: local\n+    use-fallback: false\n\n     path: vendor/bundle\n     key: ${{ runner.os }}-${{ matrix.ruby }}-gem-${{ hashFiles('**/Gemfile.lock') }}-cachev2\n
\n

The endpoint could also be a HTTPS URL to a S3 server hosted within the same network as the self-hosted runners.

\n

If you are relying on the built-in cache support that is included in some actions like setup-node and setup-go you will need to add an additional caching step to your workflow as they are not directly compatible with the self-hosted S3 cache.

\n

The impact of switching to a local cache

\n

The Tests workflow from the Discourse repository was used to test the impact of switching to a local cache. We ran the workflow on a self-hosted Actuated runner, both with the S3 local cache and with the GitHub cache.

\n

Next we looked at the time required to restore the caches in our two environments and compared it with the times we saw on GitHub hosted runners:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Bundler cache
(±273MB)		Yarn cache
(±433MB)		Plugins gems cache
(±51MB)		App state cache
(±1MB)
Actuated with local cache5s11s1s0s
Actuated with hosted cache13s19s3s2s
Runner & cache hosted on GitHub6s11s3s2s
\n

While the GitHub runner and the self-hosted runner with a local cache perform very similarly, cache restores on the self-hosted runner that uses the GitHub cache take a bit longer.

\n

If we take a look at the yarn cache, which is the biggest cache, we can see that switching to the local S3 cache saved 8s for the cache size in this test vs using GitHub's cache from a self-hosted runner. This is a 42% improvement.

\n

Depending on your workflow and the cache size this can add up quickly. If a pipeline has multiple steps or when you are running matrix builds a cache step may need to run multiple times. In the case of the Discourse repo this cache step runs nine times which adds up to 1m12s that can be saved per workflow run.

\n

When Discourse approached us, we found that they had around a dozen jobs running for each pull request, all with varying sizes of caches. At busy times of the day, their global team could have 10 or more of those pull requests running, so these savings could add up to a significant amount.

\n

What if you also cached git checkout

\n

If your repository is a monorepo or has lots of large artifacts, you may get a speed boost caching the git checkout step too. Depending on where your runners are hosted, pulling from GitHub can take some time vs. restoring the same files from a local cache.

\n

We demonstrated what impact that had for Settlemint's CTO in this case study. They saw a cached checkout using a GitHub's hosted cache from from 2m40s to 11s.

\n

How we improved testpkg's custom action

\n

During our testing we noticed that every cache restore took a minimum of 10 seconds regardless of the cache size. It turned out to be an issue with timeouts in the tespkg/actions-cache action when listing objects in S3. We reported it and sent them a pull request with a fix.

\n

With the fix in place restoring small caches from the local cache dropped from 10s to sub 1s.

\n

The impact of switching to faster runners

\n

The Discourse repo uses the larger GitHub hosted runners to run tests. The jobs we are going to compare are part of the Tests workflow. They are using runners with 8 CPUs and 32GB of ram so we replaced the runs-on label with an actuated label actuated-8cpu-24gb to run the jobs on similar sized microVMs.

\n

All jobs ran on the same Hetzner AX102 bare metal host.

\n

This table compares the time it took to run each job on the hosted runner and on our Actuated runner.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
JobGitHub hosted runnerActuated runnerSpeedup
core annotations3m23s1m22s59%
core backend7m11s6m0s16%
plugins backend7m42s5m54s23%
plugins frontend5m28s4m3s26%
themes frontend4m20s2m46s36%
chat system9m37s6m33s32%
core system7m12s5m24s25%
plugin system5m32s3m56s29%
themes system4m32s2m4141%
\n

The first thing we notice is that all jobs completed faster on the Actuated runner. On average we see an improvement of around 1m40s seconds for each individual job.

\n

Conclusion

\n

While switching to faster self-hosted runners is the most obvious way to speed up your builds, the cache hosted on GitHub's network can become a new bottleneck if you use caching in your actions. After switching to a local S3 cache we saw a very significant improvement in the cache latency. Depending on how heavily the cache is used in your workflow and the size of your cache artifacts, switching to a local S3 cache might even have a bigger impact on build times.

\n

Both Seaweedfs and Minio were tested in our setup and they performed in a very similar way. Both have different open source licenses, so we'd recommend reading those before picking one or the other. Of course you could also use AWS S3, Google Cloud Storage, or another S3 compatible hosted service.

\n

In addition to the reduced latency, switching to a self hosted cache has a couple of other benefits.

\n\n

GitHub's caching action does not yet support using a custom S3 server, so we had to make some minor adjustments to the Discourse's workflow files. For this reason, if you use something like setup-go or setup-node, you won't be able to just set cache: true. Instead you'll need an independent caching step with the testpkg/actions-cache action.

\n

If you'd like to reach out to us and see if we can advise you on how to optmise your builds, you can set up a call with us here..

\n

If you want to learn more about caching for GitHub Actions checkout some of our other blog posts:

\n

You may also like:

\n","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/managing-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/managing-github-actions.json new file mode 100644 index 00000000..ffaf98d5 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/managing-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","contentHtml":"

Over the past few months, we've launched over 20,000 VMs for customers and have handled over 60,000 webhook messages from the GitHub API. We've learned a lot from every customer PoC and from our own usage in OpenFaaS.

\n

First of all - what is it we're offering? And how is it different to managed runners and the self-hosted runner that GitHub offers?

\n

Actuated replicates the hosted experience you get from paying for hosted runners, and brings it to hardware under your own control. That could be a bare-metal Arm server, or a regular Intel/AMD cloud VM that has nested virtualisation enabled.

\n

Just like managed runners - every time actuated starts up a runner, it's within a single-tenant virtual machine (VM), with an immutable filesystem.

\n

\"The

\n
\n

Asahi Linux running on my lab of two M1 Mac Minis - used for building the Arm64 base images and Kernels.

\n
\n

Can't you just use a self-hosted runner on a VM? Yes, of course you can. But it's actually more nuanced than that. The self-hosted runner isn't safe for OSS or public repos. And whether you run it directly on the host, or in Kubernetes - it's subject to side-effects, poor isolation, malware and in some cases uses very high privileges that could result in taking over a host completely.

\n

You can learn more in the actuated announcement and FAQ.

\n

how does it work?

\n

We run a SaaS - a managed control-plane which is installed onto your organisation as a GitHub App. At that point, we'll receive webhooks about jobs in a queued state.

\n

\"Conceptual

\n

As you can see in the diagram above, when a webhook is received, and we determine it's for your organisation, we'll schedule a Firecracker MicroVM on one of your servers.

\n

We have no access to your code or build secrets. We just obtain a registration token and send the runner a bit of metadata. Then we get out the way and let the self-hosted runner do its thing - in an isolated Kernel, with an immutable filesystem and its own Docker daemon.

\n

Onboarding doesn't take very long - you can use your own servers or get them from a cloud provider. We've got a detailed guide, but can also recommend an option on a discovery call.

\n

Want to learn more about how Firecracker compares to VMs and containers? Watch my webinar on YouTube

\n

Lesson 1 - GitHub's images are big and beautiful

\n

The first thing we noticed when building our actuated VM images was that the GitHub ones are huge.

\n

And if you've ever tried to find out how they're built, or hoped to find a nice little Dockerfile, you can may be disappointed. The images for Linux, Windows and MacOS are built through a set of bespoke scripts, and are hard to adapt for your own use.

\n

Don't get me wrong. The scripts are very clever and they work well. GitHub have been tuning these runner images for years, and they cover a variety of different use-cases.

\n

The first challenge for actuated before launching a pilot was getting enough of the most common packages installed through a Dockerfile. Most of our own internal software is built with Docker, so we can get by with quite a spartan environment.

\n

We also had to adapt the sample Kernel configuration provided by the Firecracker team so that it could launch Docker and so it had everything it required to launch Kubernetes.

\n

\"M1s

\n
\n

Two M1 Mac Minis running Asahi Linux and four separate versions of K3s

\n
\n

So by following the 80/20 principle, and focusing on the most common use-cases, we were able to launch quite quickly and cover 80% of the use-cases.

\n

I don't know if you realised, things like Node.js are pre-installed in the environment, but many Node developers also add the \"setup-node\" action which guess what? Downloads and installs Node.js again. The same is true for many other languages and tools. We do ship Node.js and Python in the image, but the chances are that we could probably remove them at some point.

\n

With one of our earliest pilots, a customer wanted to use a terraform action. It failed and I felt a bit embarrassed by the reason. We were missing unzip in our images.

\n

The cure? Go and add unzip to the Dockerfile, and hit publish on our builder repository. In 3 minutes the problem was solved.

\n

But GitHub Actions is also incredibly versatile and it means even if something is missing, we don't necessary have to publish a new image for you to continue your work. Just add a step to your workflow to install the missing package.

\n
- name: Add unzip\n  run: sudo apt-get install -qy unzip\n
\n

With every customer pilot we've done, there's tended to be one or two packages like this that they expected to see. For another customer it was \"libpq\". As a rule, if something is available in the hosted runner, we'll strongly consider adding it to ours.

\n

Lesson 2 - It’s not GitHub, it can’t be GitHub. It was GitHub

\n

Since actuated is a control-plane, a SaaS, a full-service - supported product, we are always asking first - is it us? Is it our code? Is it our infrastructure? Is it our network? Is it our hardware?

\n

If you open up the GitHub status page, you'll notice an outage almost every week - at times on consecutive days, or every few days on GitHub Actions or a service that affects them indirectly - like the package registry, Pages or Pull Requests.

\n

\"Outage

\n
\n

The second outage this week that unfortunately affected actuated customers.

\n
\n

I'm not bashing on GitHub here, we're paying a high R&D cost to build on their platform. We want them to do well.

\n

But this is getting embarrassing. On a recent call, a customer told us: \"it's not your solution, it looks great for us, it's the reliability of GitHub, we're worried about adopting it\"

\n

What can you say to that? I can't tell them that their concerns are misplaced, because they're not.

\n

I reached out to Martin Woodward - Director of DevRel at GitHub. He told me that \"leadership are taking this very seriously. We're doing better than we were 12 months ago.\"

\n

GitHub is too big to fail. Let's hope they smooth out these bumps.

\n

Lesson 3 - Webhooks can be unreliable

\n

There's no good API to collect this historical data at the moment but we do have an open-source tool (self-actuated/actions-usage) we give to customers to get a summary of their builds before they start out with us.

\n

So we mirror a summary of job events from GitHub into our database, so that we can show customers trends in behaviour, and identify hot-spots on specific repos - long build times, or spikes in failure rates.

\n

\"Insights

\n
\n

Insights chart from the actuated dashboard

\n
\n

We noticed that from time to time, jobs would show in our database as \"queued\" or \"in_progress\" and we couldn't work out why. A VM had been scheduled, the job had run, and completed.

\n

In some circumstances, GitHub forgot to send us an In Progress event, or they never sent us a queued event.

\n

Or they sent us queued, in progress, then completed, but in the reverse order.

\n

It took us longer than I'm comfortable with to track down this issue, but we've now adapted our API to handle these edge-cases.

\n

Some deeper digging showed that people have also had issues with Stripe webhooks coming out of order. We saw this issue only very recently, after handling 60k webhooks - so perhaps it was a change in the system being used at GitHub?

\n

Lesson 4 - Autoscaling is hard and the API is sometimes wrong

\n

We launch a VM on your servers for every time we receive a queued event. But we have no good way of saying that a particular VM can only run for a certain job.

\n

If there were five jobs queued up, then GitHub would send us five queued events, and we'd launch five VMs. But if the first job was cancelled, we'd still have all of those VMs running.

\n

Why? Why can't we delete the 5th?

\n

Because there is no determinism. It'd be a great improvement for user experience if we could tell GitHub's API - \"great, we see you queued build X, it must run on a runner with label Y\". But we can't do that today.

\n

So we developed a \"reaper\" - a background task that tracks launched VMs and can delete them after a period of inactivity. We did have an initial issue where GitHub was taking over a minute to send a job to a ready runner, which we fixed by increasing the idle timeout value. Right now it's working really well.

\n

There is still one remaining quirk where GitHub's API reports that an active runner where a job is running as idle. This happens surprisingly often - but it's not a big deal, the VM deletion call gets rejected by the GitHub API.

\n

Lesson 5 - We can launch in under one second, but what about GitHub?

\n

The way we have things tuned today, the delay from you hitting commit in GitHub, to the job executing is similar to that of hosted runners. But sometimes, GitHub lags a little - especially during an outage or when they're under heavy load.

\n

\"VM

\n
\n

Grafana Cloud showing a gauge of microVMs per managed host

\n
\n

There could be a delay between when you commit, and when GitHub delivers the \"queued\" webhook.

\n

Scoring and placing a VM on your servers is very quick, then the boot time of the microVM is generally less than 1 second including starting up a dedicated Docker daemon inside the VM.

\n

Then the runner has to run a configuration script to register itself on the API

\n

Finally, the runner connects to a queue, and GitHub has to send it a payload to start the job.

\n

On those last two steps - we see a high success rate, but occasionally, GitHub's API will fail on either of those two operations. We receive an alert via Grafana Cloud and Discord - then investigate. In the worst case, we re-queue via our API the job and the new VM will pick up the pending job.

\n

Want to watch a demo?

\n\n

Lesson 6 - Sometimes we need to debug a runner

\n

When I announced actuated, I heard a lot of people asking for CircleCI's debug experience, so I built something similar and it's proved to be really useful for us in building actuated.

\n

Only yesterday, Ivan Subotic from Dasch Swiss messaged me and said:

\n
\n

\"How cool!!! you don’t know how many hours I have lost on GitHub Actions without this.\"

\n
\n

Recently there were two cases where we needed to debug a runner with an SSH shell.

\n

The first was for a machine on Hetzner, where the Docker Daemon was unable to pull images due to a DNS failure. I added steps to print out /etc/resolv.conf and that would be my first port of call. Debugging is great, but it's slow, if an extra step in the workflow can help us diagnose the problem, it's worth it.

\n

In the end, it took me about a day and a half to work out that Hetzner was blocking outgoing traffic on port 53 to Google and Cloudflare. What was worse - was that it was an intermittent problem.

\n

When we did other customer PoCs on Hetzner, we did not run into this issue. I even launched a \"cloud\" VM in the same region and performed a couple of nslookups - they worked as expected for me.

\n

So I developed a custom GitHub Action to unblock the customer:

\n
steps:\n    - uses: self-actuated/hetzner-dns-action@v1\n
\n

Was this environmental issue with Hetzner our responsibility? Arguably not, but our customers pay us to provide a \"like managed\" solution, and we are currently able to help them be successful.

\n

In the second case, Ivan needed to launch headless Chrome, and was using one of the many setup-X actions from the marketplace.

\n

I opened a debug session on one of our own runners, then worked backwards:

\n
curl -sLS -O https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n\ndpkg -i ./google-chrome-stable_current_amd64.deb\n
\n

This reported that some packages were missing, I got which packages by running apt-get --fix-missing --no-recommends and provided an example of how to add them.

\n
jobs:\n    chrome:\n        name: chrome\n        runs-on: actuated\n        steps:\n        - name: Add extra packages for Chrome\n          run: |\n            sudo apt install -qyyy --no-install-recommends adwaita-icon-theme fontconfig fontconfig-config fonts-liberation gtk-update-icon-cache hicolor-icon-theme humanity-icon-theme libatk-bridge2.0-0 libatk1.0-0 libatk1.0-data libatspi2.0-0 ...\n        - uses: browser-actions/setup-chrome@v1\n        - run: chrome --version\n
\n

We could also add these to the base image by editing the Dockerfile that we maintain.

\n

Lesson 7 - Docker Hub Rate Limits are a pain

\n

Docker Hub rate limits are more of a pain on self-hosted runners than they are on GitHub's own runners.

\n

I ran into this problem whilst trying to rebuild around 20 OpenFaaS Pro repositories to upgrade a base image. So after a very short period of time, all code ground to a halt and every build failed.

\n

GitHub has a deal to pay Docker Inc so that you don't run into rate limits. At time of writing, you'll find a valid Docker Hub credential in the $HOME/.docker/config.json file on any hosted runner.

\n

Actuated customers would need to login at the top of every one of their builds that used Docker, and create an organisation-level secret with a pull token from the Docker Hub.

\n

We found a way to automate this, and speed up subsequent jobs by caching images directly on the customer's server.

\n

All they need to add to their builds is:

\n
- uses: self-actuated/hub-mirror@master\n
\n

Wrapping up

\n

I hope that you've enjoyed hearing a bit about our journey so far. With every new pilot customer we learn something new, and improve the offering.

\n

Whilst there was a significant amount of very technical work at the beginning of actuated, most of our time now is spent on customer support, education, and improving the onboarding experience.

\n

If you'd like to know how actuated compares to hosted runners or managing the self-hosted runner on your own, we'd encourage checking out the blog and FAQ.

\n

Are your builds slowing the team down? Do you need better organisation-level insights and reporting? Or do you need Arm support? Are you frustrated with managing self-hosted runners?

\n

Reach out to us take part in the pilot

","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/millions-of-cncf-minutes.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/millions-of-cncf-minutes.json new file mode 100644 index 00000000..9312cafa --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/millions-of-cncf-minutes.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","contentHtml":"

In this post I'll cover:

\n\n

What's Actuated?

\n
\n

Actuated is the only solution that gives a managed experience for self-hosted runners, on your own hardware through immutable microVMs.

\n
\n

The immutability is key to both security and reliability. Each build is run in its own ephemeral and immutable microVM, meaning side effects cannot be left behind by prior builds. In addition, our team manages reliability for your servers and the integration with GitHub, for minimal disruption during GitHub outages.

\n

Why did the CNCF want Actuated for its projects?

\n

We've been working with Ampere and Equinix Metal to provide CI via GitHub Actions for Cloud Native Computing (CNCF) projects. Ampere manufacture Arm-based CPUs with a focus on efficiency and high core density. Equinix Metal provide access to the Ampere Altra in their datacenters around the world.

\n

Last December, we met with Chris Aniszczyck - CTO Linux Foundation/CNCF, Ed Vielmetti - Open Source Manager Equinix, Dave Neary - Director of Developer Relations at Ampere and myself to discuss the program and what impact it was having so far.

\n

Watch the recap on YouTube: The Ampere Developer Impact: CNCF Pilot Discussion

\n

Past articles on the blog include:

\n\n

Before and after the program

\n

Before we started the program, CNCF projects could be divided into three buckets:

\n
    \n
  1. They were running Arm-based CI jobs on static servers that they managed
    2. \n
\n

In this case, etcd for instance had a team of half a dozen maintainers who were responsible for setting up, maintaining, and upgrading statically provisioned CI servers for the project. This was a significant overhead for the project maintainers, and the servers were often underutilized. The risk of side-effects being left behind between builds also posed a serious supply chain risk since etcd is consumed in virtually every Kubernetes deployment.

\n
    \n
  1. They were running Arm-based CI using emulation (QEMU)
    2. \n
\n

QEMU can be combined with Docker's buildx for a quick and convenient way to build container images for x86 and Arm architectures. In the best case, it's a small change and may add a few minutes of extra overhead. In the worst case, we saw that jobs that ran in ~ 5 minutes, took over 6 hours to complete using QEMU and hosted runners. A prime example was fluentd, read their case-study here: Scaling ARM builds with Actuated

\n
    \n
  1. They had no Arm CI at all
    2. \n
\n

In the third case, we saw projects like OpenTelemetry which had no support for Arm at all, but demand from their community to bring it up to on par with x86 builds. The need to self-manage insecure CI servers meant that Arm was a blocker for them.

\n

After the program

\n

After the program was live, teams who had been maintaining their own servers got to remove lengthy documentation on server configuration and maintenance, and relied on our team to manage a pool of servers used for scheduling microVMs.

\n

As demand grew, we saw OpenTelemetry and etcd starve the shared pool of resources through very high usage patterns. This is a classic and known problem called \"Tragedy of the Commons\" - when a shared resource is overused by a subset of users, it can lead to a degradation of service for all users. To combat the problem, we added code to provision self-destructing servers for a period of 24-48 hours as need arose, and prevented the higher usage projects from running on at least on of the permanent servers through scheduling rules. One other issue we saw with OpenTelemetry in particular was that the various Go proxies that offer up Go modules appeared to be rejecting requests when too many jobs were running concurrently. As a workaround, we added a private Go proxy for them into the private network space where the CNCF servers run, this also massively reduced the bandwidth costs for the shared infrastructure.

\n

Teams like fluent moved from flakey builds that couldn't finish in 6 hours, to builds that finished in 5-10 minutes. This meant they could expand on their suite of tests.

\n

Where teams such as Cilium, Falco, or OpenTelemetry had no Arm CI support, we saw them quickly ramp up to running thousands of builds per month.

\n

Here's a quote from Federico Di Pierro, Senior Open Source Engineer @ Sysdig and maintainer of Falco:

\n
\n

Falco really needed arm64 GitHub runners to elevate its support for the architecture and enlarge its userbase.\nActuated was the perfect solution for us because it was easy to leverage and relieved any burden for the maintainers. This way, we as maintainers, can focus on what really matters for the project, instead of fighting with maintaining and deploying self-hosted infrastructure.\nNow we are building, testing and releasing artifacts for arm64 leveraging Actuated for many of our projects, and it works flawlessly.\nSupport from Alex's team is always on point, and new kernel features are coming through super quickly!

\n
\n

Akihiro Suda, Software Engineer at NTT Corp, and maintainer of several open source projects including: runc, containerd and lima had this to say:

\n
\n

Huge thanks to Actuated for enabling us to run ARM64 tests without any mess.\nIt is very important for the runc project to run the tests on ARM64, as runc depends on several architecture-dependent components such as seccomp and criu.\nIt is also so nice that the Arm instance specification can be adjusted in a single line in the GitHub Actions workflow file.

\n
\n

Wei Fu, a maintainer for containerd said:

\n
\n

The containerd project was able to test each pull request for the Linux arm64 platform with the support of Actuated.\nIt's a significant step for containerd to mark the Linux arm64 platform as a top-tier supported platform, similar to amd64, since containerd has been widely used in the Arm world.

\n

Thanks to Actuated, we, the containerd community, were able to test container features (like mount-idmapping) on the new kernel without significant maintenance overhead for the test infrastructure.\nWith Actuated, we can focus on open-source deployment to cover more use case scenarios.

\n
\n

Maintainers have direct access to discuss issues and improvements with us via a private Slack community. One of the things we've done in addition to adding burst capacity to the pool, was to provide a tool to help teams right-size VMs for jobs and to add support for eBPF technologies like BTF in the Kernel.

\n

Numbers at a glance

\n

In our last update, 3 months ago, we'd run just under 400k build minutes for the CNCF. That number has now increased to 1.52M minutes, which is a ~ 300x increase in demand in a short period of time.

\n

Here's a breakdown of the top 9 projects by total minutes run, bearing in mind that this only includes jobs that ran to completion, there are thousands of minutes which ran, but were cancelled mid-way or by automation.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationTotal minsTotal JobsFirst job
1open-telemetry593726400932024-02-15
2etcd-io372080213472023-10-24
3cri-o163927111312023-11-27
4falcosecurity138469132742023-12-06
5fluent89856106582023-06-07
6containerd87007111922023-12-02
7cilium7340662522023-10-31
8opencontainers37164582023-12-15
9argoproj187122024-01-30
(all)(Total)1520464116217
\n

Most organisations build for several projects or repositories. In the case of etcd, the numbers also include the boltdb project, and for cilium, tetragon, and the Go bindings for ebpf are also included. Open Telemetry is mainly focused around the collectors and SDKs.

\n

runc which is within the opencontainers organisation is technically an Open Container Initiative (OCI) project under the LinuxFoundation, rather than a CNCF project, but we gave them access since it is a key dependency for containerd and cri-o.

\n

What's next?

\n

With the exception of Argo, all of the projects are now relatively heavy users of the platform, with demand growing month on month, as you can see from the uptick from 389k minutes in March to a record high of 1.52 million minutes by the end of May of the same year. In the case of Argo, if you're a contributor or have done previous open source enablement, perhaps you could help them expand their Arm support via a series of Pull Requests to enable unit/e2e tests to run on Arm64?

\n

We're continuing to improve the platform to support users during peak demand, outages on GitHub, and to provide a reliable way for CNCF projects to run their CI on real Arm hardware, at full speed.

\n

For instance, last month we just released a new 6.1 Kernel for the Ampere Altra, which means projects like Cilium and Falco can make use of new eBPF features introduced in recent Kernel versions, and will bring support for newer Kernels as the Firecracker team make them available. The runc and container teams also benefit from the newer Kernel and have been able to enable further tests for (Checkpoint/Restore In Userspace) CRIU and User namespaces for containerd.

\n

You can watch the interview I mentioned earlier with Chris, Ed, Dave and myself on YouTube:

\n\n

We could help you too

\n

Actuated can manage x86 and Arm64 servers for GitHub Actions and self-managed GitLab CI. If you'd like to speak to us about how we can speed up your jobs, reduce your maintenance efforts and lower your CI costs, reach out via this page.

","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/multi-arch-docker-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/multi-arch-docker-github-actions.json new file mode 100644 index 00000000..f58439a0 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/multi-arch-docker-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","contentHtml":"

In 2017, I wrote an article on multi-stage builds with Docker, and it's now part of the Docker Documentation. In my opinion, multi-arch builds were the proceeding step in the evolution of container images.

\n

What's multi-arch and why should you care?

\n

If you want users to be able to use your containers on different types of computer, then you'll often need to build different versions of your binaries and containers.

\n

The faas-cli tool is how users interact with OpenFaaS.

\n

It's distributed in binary format for users, with builds for Windows, MacOS and Linux.

\n\n

But why are there six different binaries for three Operating Systems? With the advent of Raspberry Pi, M1 Macs (Apple Silicon) and AWS Graviton servers, we have had to start building binaries for more than just Intel systems.

\n

If you're curious how to build multi-arch binaries with Go, you can check out the release process for the open source arkade tool here, which is a simpler example than faas-cli: arkade Makefile and GitHub Actions publish job

\n

So if we have to support at least six different binaries for Open Source CLIs, what about container images?

\n

Do we need multi-arch containers too?

\n

Until recently, it was common to hear people say: \"I can't find any containers that work for Arm\". This was because the majority of container images were built only for Intel. Docker Inc has done a sterling job of making their \"official\" images work on different platforms, that's why you can now run docker run -t -i ubuntu /bin/bash on a Raspberry Pi, M1 Mac and your regular PC.

\n

Many open source projects have also caught on to the need for multi-arch images, but there are still a few like Bitnami, haven't yet seen value. I think that is OK, this kind work does take time and effort. Ultimately, it's up to the project maintainers to listen to their users and decide if they have enough interest to add support for Arm.

\n

A multi-arch image is a container that will work on two or more different combinations of operating system and CPU architecture.

\n

Typically, this would be:

\n\n

So multi-arch is really about catering for the needs of Arm users. Arm hardware platforms like the Ampere Altra come with 80 efficient CPU cores, have a very low TDP compared to traditional Intel hardware, and are available from various cloud providers.

\n

How do we build multi-arch containers work?

\n

There are a few tools and tricks that we can combine together to take a single Dockerfile and output an image that anyone can pull, which will be right for their machine.

\n

Let's take the: ghcr.io/inlets-operator:latest image from inlets.

\n

When a user types in docker pull, or deploys a Pod to Kubernetes, their local containerd daemon will fetch the manifest file and inspect it to see what SHA reference to use for to download the required layers for the image.

\n

\"How

\n
\n

How manifests work

\n
\n

Let's look at a manifest file with the crane tool. I'm going to use arkade to install crane:

\n
arkade get crane\n\ncrane manifest ghcr.io/inlets/inlets-operator:latest\n
\n

You'll see a manifests array, with a platform section for each image:

\n
{\n  \"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\",\n  \"manifests\": [\n    {\n      \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n      \"digest\": \"sha256:bae8025e080d05f1db0e337daae54016ada179152e44613bf3f8c4243ad939df\",\n      \"platform\": {\n        \"architecture\": \"amd64\",\n        \"os\": \"linux\"\n      }\n    },\n    {\n      \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n      \"digest\": \"sha256:3ddc045e2655f06653fc36ac88d1d85e0f077c111a3d1abf01d05e6bbc79c89f\",\n      \"platform\": {\n        \"architecture\": \"arm64\",\n        \"os\": \"linux\"\n      }\n    }\n  ]\n}\n
\n

How do we convert a Dockerfile to multi-arch?

\n

Instead of using the classic version of Docker, we can enable the buildx and Buildkit plugins which provide a way to build multi-arch images.

\n

We'll continue with the Dockerfile from the open source inlets-operator project.

\n

Within the Dockerfile, we need to make a couple of changes.

\n
- FROM golang:1.18 as builder\n+ FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.18 as builder\n\n+ ARG TARGETPLATFORM\n+ ARG BUILDPLATFORM\n+ ARG TARGETOS\n+ ARG TARGETARCH\n
\n

The BUILDPLATFORM variable is the native architecture and platform of the machine performing the build, this is usually amd64.

\n

The TARGETPLATFORM is important for the final step of the build, and will normally be injected based upon one each of the platforms you have specified for the build command.

\n

For Go specifically, we also updated the go build command to tell Go to use cross-compilation based upon the TARGETOS and TARGETARCH environment variables, which are populated by Docker.

\n
- go build -o inlets-operator\n+ GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o inlets-operator\n
\n

Here's the full example:

\n
FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.18 as builder\n\nARG TARGETPLATFORM\nARG BUILDPLATFORM\nARG TARGETOS\nARG TARGETARCH\n\nARG Version\nARG GitCommit\n\nENV CGO_ENABLED=0\nENV GO111MODULE=on\n\nWORKDIR /go/src/github.com/inlets/inlets-operator\n\n# Cache the download before continuing\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY .  .\n\nRUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \\\n  go test -v ./...\n\nRUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \\\n  go build -ldflags \"-s -w -X github.com/inlets/inlets-operator/pkg/version.Release=${Version} -X github.com/inlets/inlets-operator/pkg/version.SHA=${GitCommit}\" \\\n  -a -installsuffix cgo -o /usr/bin/inlets-operator .\n\nFROM --platform=${BUILDPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot\n\nLABEL org.opencontainers.image.source=https://github.com/inlets/inlets-operator\n\nWORKDIR /\nCOPY --from=builder /usr/bin/inlets-operator /\nUSER nonroot:nonroot\n\nCMD [\"/inlets-operator\"]\n
\n

How to do you configure GitHub Actions to publish multi-arch images?

\n

Now that the Dockerfile has been configured, it's time to start working on the GitHub Action.

\n

This example is taken from the Open Source inlets-operator. It builds a container image containing a Go binary and uses a Dockerfile in the root of the repository.

\n

View publish.yaml, adapted for actuated:

\n
name: publish\n\non:\n  push:\n    tags:\n      - '*'\n\njobs:\n  publish:\n+    permissions:\n+      packages: write\n\n-   runs-on: ubuntu-latest\n+   runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n\n+     - name: Setup mirror\n+       uses: self-actuated/hub-mirror@master\n      - name: Get TAG\n        id: get_tag\n        run: echo TAG=${GITHUB_REF#refs/tags/} >> $GITHUB_ENV\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n+     - name: Set up QEMU\n+       uses: docker/setup-qemu-action@v2\n+     - name: Set up Docker Buildx\n+       uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          provenance: false\n+         platforms: linux/amd64,linux/arm/v6,linux/arm64\n          build-args: |\n            Version=${{  env.TAG }}\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ env.TAG }}\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:latest\n
\n

All of the images and corresponding manifest are published to GitHub's Container Registry (GHCR). The action itself is able to authenticate to GHCR using a built-in, short-lived token. This is dependent on the \"permissions\" section and \"packages: write\" being set.

\n

You'll see that we added a Setup mirror step, this explained in the Registry Mirror example and is not required for Hosted Runners.

\n

The docker/setup-qemu-action@v2 step is responsible for setting up QEMU, which is used to emulate the different CPU architectures.

\n

The docker/build-push-action@v4 step is responsible for passing in a number of platform combinations such as: linux/amd64 for cloud, linux/arm64 for Arm servers and linux/arm/v6 for Raspberry Pi.

\n

What if you're not using GitHub Actions?

\n

The various GitHub Actions published by the Docker team are a great way to get started, but if you look under the hood, they're just syntactic sugar for the Docker CLI.

\n
export DOCKER_CLI_EXPERIMENTAL=enabled\n\n# Have Docker download the latest buildx plugin\ndocker buildx install\n\n# Create a buildkit daemon with the name \"multiarch\"\ndocker buildx create \\\n    --use \\\n    --name=multiarch \\\n    --node=multiarch\n\n# Install QEMU\ndocker run --rm --privileged \\\n    multiarch/qemu-user-static --reset -p yes\n\n# Run a build for the different platforms\ndocker buildx build \\\n    --platform=linux/arm64,linux/amd64 \\\n    --output=type=registry,push=true --tag image:tag .\n
\n

For OpenFaaS users, we do all of the above any time you type in faas-cli publish and the faas-cli build command just runs a regular Docker build, without any of the multi-arch steps.

\n

If you're interested, you can checkout the code here: publish.go.

\n

Putting it all together

\n\n

In our experience with OpenFaaS, inlets and actuated, once you have converted one or two projects to build multi-arch images, it becomes a lot easier to do it again, and make all software available for Arm servers.

\n

You can learn more about Multi-platform images in the Docker Documentation.

\n

Want more multi-arch examples?

\n

OpenFaaS uses multi-arch Dockerfiles for all of its templates, and the examples are freely available on GitHub including Python, Node, Java and Go.

\n

See also: OpenFaaS templates

\n

A word of caution

\n

QEMU can be incredibly slow at times when using a hosted runner, where a build takes takes 1-2 minutes can extend to over half an hour. If you do run into that, one option is to check out actuated or another solution, which can build directly on an Arm server with a securely isolated Virtual Machine.

\n

In How to make GitHub Actions 22x faster with bare-metal Arm, we showed how we decreased the build time of an open-source Go project from 30.5 mins to 1.5 mins. If this is the direction you go in, you can use a matrix-build instead of a QEMU-based multi-arch build.

\n

See also: Recommended bare-metal Arm servers

","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/native-arm64-for-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/native-arm64-for-github-actions.json new file mode 100644 index 00000000..7324fa4a --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/native-arm64-for-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","contentHtml":"

GitHub Actions is a modern, fast and efficient way to build and test software, with free runners available. We use the free runners for various open source projects and are generally very pleased with them, after all, who can argue with good enough and free? But one of the main caveats is that GitHub's hosted runners don't yet support the Arm architecture.

\n

So many people turn to software-based emulation using QEMU. QEMU is tricky to set up, and requires specific code and tricks if you want to use software in a standard way, without modifying it. But QEMU is great when it runs with hardware acceleration. Unfortunately, the hosted runners on GitHub do not have KVM available, so builds tend to be incredibly slow, and I mean so slow that it's going to distract you and your team from your work.

\n

This was even more evident when Frederic Branczyk tweeted about his experience with QEMU on GitHub Actions for his open source observability project named Parca.

\n

Does anyone have a @github actions self-hosted runner manifest for me to throw at a @kubernetesio cluster? I'm tired of waiting for emulated arm64 CI runs taking ages.

— Frederic 🧊 Branczyk @brancz@hachyderm.io (@fredbrancz) October 19, 2022
\n

I checked out his build and expected \"ages\" to mean 3 minutes, in fact, it meant 33.5 minutes. I know because I forked his project and ran a test build.

\n

After migrating it to actuated and one of our build agents, the time dropped to 1 minute and 26 seconds, a 22x improvement for zero effort.

\n

This morning @fredbrancz said that his ARM64 build was taking 33 minutes using QEMU in a GitHub Action and a hosted runner.

I ran it on @selfactuated using an ARM64 machine and a microVM.

That took the time down to 1m 26s!! About a 22x speed increase. https://t.co/zwF3j08vEV pic.twitter.com/ps21An7B9B

— Alex Ellis (@alexellisuk) October 20, 2022
\n

You can see the results here:

\n

\"Results

\n

As a general rule, the download speed is going to be roughly the same with a hosted runner, it may even be slightly faster due to the connection speed of Azure's network.

\n

But the compilation times speak for themselves - in the Parca build, go test was being run with QEMU. Moving it to run on the ARM64 host directly, resulted in the marked increase in speed. In fact, the team had introduced lots of complicated code to try and set up a Docker container to use QEMU, all that could be stripped out, replacing it with a very standard looking test step:

\n
  - name: Run the go tests\n    run: go test ./...\n
\n

Can't I just install the self-hosted runner on an Arm VM?

\n

There are relatively cheap Arm VMs available from Oracle OCI, Google and Azure based upon the Ampere Altra CPU. AWS have their own Arm VMs available in the Graviton line.

\n

So why shouldn't you just go ahead and install the runner and add them to your repos?

\n

The moment you do that you run into three issues:

\n\n

Chasing your tail with package updates, faulty builds due to caching and conflicts is not fun, you may feel like you're saving money, but you are paying with your time and if you have a team, you're paying with their time too.

\n

Most importantly, GitHub say that it cannot be used safely with a public repository. There's no security isolation, and state can be left over from one build to the next, including harmful code left intentionally by bad actors, or accidentally from malware.

\n

So how do we get to a safer, more efficient Arm runner?

\n

The answer is to get us as close as possible to a hosted runner, but with the benefits of a self-hosted runner.

\n

That's where actuated comes in.

\n

We run a SaaS that manages bare-metal for you, and talks to GitHub upon your behalf to schedule jobs efficiently.

\n\n

microVMs on Arm require a bare-metal server, and we have tested all the options available to us. Note that the Arm VMs discussed above do not currently support KVM or nested virtualisation.

\n\n

If you're already an AWS customer, the a1.metal is a good place to start. If you need expert support, networking and a high speed uplink, you can't beat Equinix Metal (we have access to hardware there and can help you get started) - you can even pay per minute and provision machines via API. The Mac Mini <1 has a really fast NVMe and we're running one of these with Asahi Linux for our own Kernel builds for actuated. The RX Line from Hetzner has serious power and is really quite affordable, but just be aware that you're limited to a 1Gbps connection, a setup fee and monthly commitment, unless you pay significantly more.

\n

I even tried Frederic's Parca job on my 8GB Raspberry Pi with a USB NVMe. Why even bother, do I hear you say? Well for a one-time payment of 80 USD, it was 26m30s quicker than a hosted runner with QEMU!

\n

Learn how to connect an NVMe over USB-C to your Raspberry Pi 4

\n

What does an Arm job look like?

\n

Since I first started trying to build code for Arm in 2015, I noticed a group of people who had a passion for this efficient CPU and platform. They would show up on GitHub issue trackers, ready to send patches, get access to hardware and test out new features on Arm chips. It was a tough time, and we should all be grateful for their efforts which go largely unrecognised.

\n
\n

If you're looking to make your software compatible with Arm, feel free to reach out to me via Twitter.

\n
\n

In 2020 when Apple released their M1 chip, Arm went mainstream, and projects that had been putting off Arm support like KinD and Minikube, finally had that extra push to get it done.

\n

I've had several calls with teams who use Docker on their M1/M2 Macs exclusively, meaning they build only Arm binaries and use only Arm images from the Docker Hub. Some of them even ship to project using Arm images, but I think we're still a little behind the curve there.

\n

That means Kubernetes - KinD/Minikube/K3s and Docker - including Buildkit, compose etc, all work out of the box.

\n

I'm going to use the arkade CLI to download KinD and kubectl, however you can absolutely find the download links and do all this manually. I don't recommend it!

\n
name: e2e-kind-test\n\non: push\njobs:\n  start-kind:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: get arkade\n        uses: alexellis/setup-arkade@v1\n      - name: get kubectl and kubectl\n        uses: alexellis/arkade-get@master\n        with:\n          kubectl: latest\n          kind: latest\n      - name: Create a KinD cluster\n        run: |\n          mkdir -p $HOME/.kube/\n          kind create cluster --wait 300s\n      - name: Wait until CoreDNS is ready\n        run: |\n          kubectl rollout status deploy/coredns -n kube-system --timeout=300s\n      - name: Explore nodes\n        run: kubectl get nodes -o wide\n      - name: Explore pods\n        run: kubectl get pod -A -o wide\n
\n

That's our x86_64 build, or Intel/AMD build that will run on a hosted runner, but will be kind of slow.

\n

Let's convert it to run on an actuated ARM64 runner:

\n
jobs:\n  start-kind:\n-    runs-on: ubuntu-latest\n+    runs-on: actuated-aarch64\n
\n

That's it, we've changed the runner type and we're ready to go.

\n

\"In

\n
\n

An in progress build on the dashboard

\n
\n

Behind the scenes, actuated, the SaaS schedules the build on a bare-metal ARM64 server, the boot up takes less than 1 second, and then the standard GitHub Actions Runner talks securely to GitHub to run the build. The build is isolated from other builds, and the runner is destroyed after the build is complete.

\n

\"Setting

\n
\n

Setting up an Arm KinD cluster took about 49s

\n
\n

Setting up an Arm KinD cluster took about 49s, then it's over to you to test your Arm images, or binaries.

\n

If I were setting up CI and needed to test software on both Arm and x86_64, then I'd probably create two separate builds, one for each architecture, with a runs-on label of actuated and actuated-aarch64 respectively.

\n

Do you need to test multiple versions of Kubernetes? Let's face it, it changes so often, that who doesn't need to do that. You can use the matrix feature to test multiple versions of Kubernetes on Arm and x86_64.

\n

I show 5x clusters being launched in parallel in the video below:

\n

Demo - Actuated - secure, isolated CI for containers and Kubernetes

\n

What about Docker?

\n

Docker comes pre-installed in the actuated OS images, so you can simply use docker build, without any need to install extra tools like Buildx, or to have to worry about multi-arch Dockerfiles. Although these are always good to have, and are available out of the box in OpenFaaS, if you're curious what a multi-arch Dockerfile looks like.

\n

Wrapping up

\n

Building on bare-metal Arm hosts is more secure because side effects cannot be left over between builds, even if malware is installed by a bad actor. It's more efficient because you can run multiple builds at once, and you can use the latest software with our automated Operating System image. Enabling actuated on a build is as simple as changing the runner type.

\n

And as you've seen from the example with the OSS Parca project, moving to a native Arm server can improve speed by 22x, shaving off a massive 34 minutes per build.

\n

Who wouldn't want that?

\n

Parca isn't a one-off, I was also told by Connor Hicks from Suborbital that they have an Arm build that takes a good 45 minutes due to using QEMU.

\n

Just a couple of days ago Ed Warnicke, Distinguished Engineer at Cisco reached out to us to pilot actuated. Why?

\n

Ed, who had Network Service Mesh in mind said:

\n
\n

I'd kill for proper Arm support. I'd love to be able to build our many containers for Arm natively, and run our KIND based testing on Arm natively.\nWe want to build for Arm - Arm builds is what brought us to actuated

\n
\n

So if that sounds like where you are, reach out to us and we'll get you set up.

\n\n

Additional links:

\n","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/oidc-proxy-for-openfaas.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/oidc-proxy-for-openfaas.json new file mode 100644 index 00000000..9cb783ac --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/oidc-proxy-for-openfaas.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","contentHtml":"

In 2021, GitHub released OpenID Connect (OIDC) support for CI jobs running under GitHub Actions. This was a huge step forward for security meaning that any GitHub Action could mint an OIDC token and use it to securely federate into another system without having to store long-lived credentials in the repository.

\n

I wrote a prototype for OpenFaaS shortly after the announcement and a deep dive explaining how it works. I used inlets to set up a HTTPS tunnel, and send the token to my machine for inspection. Various individuals and technical teams have used my content as a reference guide when working with GitHub Actions and OIDC.

\n

See the article: Deploy without credentials with GitHub Actions and OIDC

\n

Since then, custom actions for GCP, AWS and Azure have been created which allow an OIDC token from a GitHub Action to be exchanged for a short-lived access token for their API - meaning you can manage cloud resources securely. For example, see: Configuring OpenID Connect in Amazon Web Services - we have actuated customers who use this approach to deploy to ECR from their self-hosted runners without having to store long-lived credentials in their repositories.

\n

Why OIDC is important for OpenFaaS customers

\n

Before we talk about the new OIDC proxy for OpenFaaS, I should say that OpenFaaS Enterprise also has an IAM feature which includes OIDC support for the CLI, dashboard and API. It supports any trusted OIDC provider, not just GitHub Actions. Rather than acting as a proxy, it actually implements a full fine-grained authorization and permissions policy language that resembles the one you'll be used to from AWS.

\n

However, not everyone needs this level of granularity.

\n

Shaked, the CTO of Kubiya.ai is an OpenFaaS & inlets customer. His team at Kubiya is building a conversational AI for DevOps - if you're ever tried ChatGPT, imagine that it was hooked up to your infrastructure and had superpowers. On a recent call, he told me that their team now has 30 different repositories which deploy OpenFaaS functions to their various AWS EKS clusters. That means that a secret has to be maintained at the organisation level and then consumed via faas-cli login in each job.

\n

It gets a little worse for them - because different branches deploy to different OpenFaaS gateways and to different EKS clusters.

\n

In addition to managing various credentials for each cluster they add - they were uncomfortable with exposing all of their functions on the Internet.

\n

So today the team working on actuated is releasing a new OIDC proxy which can be deployed to any OpenFaaS cluster to avoid the need to manage and share long-lived credentials with GitHub.

\n

\"Conceptual

\n
\n

Conceptual design of the OIDC proxy for OpenFaaS

\n
\n

About the OIDC proxy for OpenFaaS

\n\n

Best of all, unlike OpenFaaS Enterprise, it's free for all actuated customers - whether they're using OpenFaaS CE, Standard or Enterprise.

\n

Here's what Shaked had to say about the new proxy:

\n
\n

That's great - thank you! Looking forward to it as it will simplify our usage of the openfaas templates and will speed up our development process\nShaked, CTO, Kubiya.ai

\n
\n

How to deploy the proxy for OpenFaaS

\n

Here's what you need to do:

\n\n
\n

My cluster is not publicly exposed on the Internet, so I'm using an inlets tunnel to expose the OIDC Proxy from my local KinD cluster. I'll be using the domain minty.exit.o6s.io but you'd create something more like oidc-proxy.example.com for your own cluster.

\n
\n

First Set up your values.yaml for Helm:

\n
# The public URL to access the proxy\npublicURL: https://oidc-proxy.example.com\n\n# Comma separated list of repository owners for which short-lived OIDC tokens are authorized.\n# For example: alexellis,self-actuated\nrepositoryOwners: 'alexellis,self-actuated'\ningress:\n    host: oidc-proxy.example.com\n    issuer: letsencrypt-prod\n
\n

The chart will create an Ingress record for you using an existing issuer. If you want to use something else like Inlets or Istio to expose the OIDC proxy, then simply set enabled: false under the ingress: section.

\n

Create a secret for the actuated subscription key:

\n
kubectl create secret generic actuated-license \\\n  -n openfaas \\\n  --from-file=actuated-license=$HOME/.actuated/LICENSE\n
\n

Then run:

\n
helm repo add actuated https://self-actuated.github.io/charts/\nhelm repo update\n\nhelm upgrade --install actuated/openfaas-oidc-proxy \\\n    -f ./values.yaml\n
\n

For the full setup - see the README for the Helm chart

\n

You can now go to one of your repositories and update the workflow to authenticate to the REST API via an OIDC token.

\n

In order to get an OIDC token within a build, add the id_token: write permission to the permissions list.

\n
name: keyless_deploy\n\non:\n  workflow_dispatch:\n  push:\n    branches:\n    - '*'\njobs:\n  keyless_deploy:\n    permissions:\n      contents: 'read'\n      id-token: 'write'\n
\n

Then set runs-on to actuated to use your faster actuated servers:

\n
-   runs-on: ubuntu-latest\n+   runs-on: actuated\n
\n

Then in the workflow, install the OpenFaaS CLI:

\n
steps:\n    - uses: actions/checkout@master\n    with:\n        fetch-depth: 1\n    - name: Install faas-cli\n    run: curl -sLS https://cli.openfaas.com | sudo sh\n
\n

Then get a token:

\n
- name: Get token and use the CLI\n    run: |\n        OPENFAAS_URL=https://minty.exit.o6s.io\n        OIDC_TOKEN=$(curl -sLS \"${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=$OPENFAAS_URL\" -H \"User-Agent: actions/oidc-client\" -H \"Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN\")\n        JWT=$(echo $OIDC_TOKEN | jq -j '.value')\n
\n

Finally, use the token whenever you need it by passing in the --token flag to any of the faas-cli commands:

\n
faas-cli list -n openfaas-fn --token \"$JWT\"\nfaas-cli ns --token \"$JWT\"\nfaas-cli store deploy printer --name p1 --token \"$JWT\"\n\nfaas-cli describe p1 --token \"$JWT\"\n
\n

Since we have a lot of experience with GitHub Actions, we decided to make the above simpler by creating a custom Composite Action. If you check out the code for self-actuated/openfaas-oidc you'll see that it obtains a token, then writes it into an openfaaas config file, so that the --token flag isn't required.

\n

Here's how it changes:

\n
- uses: self-actuated/openfaas-oidc@v1\n  with: \n    gateway: https://minty.exit.o6s.io\n- name: Check OpenFaaS version\n  run: |\n    OPENFAAS_CONFIG=$HOME/.openfaas/\n    faas-cli version\n
\n

Here's the complete example:

\n
name: federate\n\non:\n  workflow_dispatch:\n  push:\n    branches:\n    - '*'\njobs:\n  auth:\n    # Add \"id-token\" with the intended permissions.\n    permissions:\n      contents: 'read'\n      id-token: 'write'\n\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: Install faas-cli\n        run: curl -sLS https://cli.openfaas.com | sudo sh\n      - uses: self-actuated/openfaas-oidc@v1\n        with:\n          gateway: https://minty.exit.o6s.io\n\n      - name: Get token and use the CLI\n        run: |\n          export OPENFAAS_URL=https://minty.exit.o6s.io\n          faas-cli store deploy env --name http-header-printer\n          faas-cli list\n
\n

How can we be sure that our functions cannot be invoked over the proxy?

\n

Just add an extra line to test it out:

\n
      - name: Get token and use the CLI\n        run: |\n          export OPENFAAS_URL=https://minty.exit.o6s.io\n          faas-cli store deploy env --name http-header-printer\n          sleep 5\n\n          echo | faas-cli invoke http-header-printer\n
\n

\"A

\n
\n

A failed invocation over the proxy

\n
\n

Best of all, now that you're using OIDC, you can now go and delete any of those long lived basic auth credentials from your secrets!

\n

Wrapping up

\n

The new OIDC proxy for OpenFaaS is available for all actuated customers and works with OpenFaaS CE, Standard and Enterprise. You can use it on as many clusters as you like, whilst you have an active subscription for actuated at no extra cost.

\n

In a short period of time, you can set up the Helm chart for the OIDC proxy and no longer have to worry about storing various secrets in GitHub Actions for all your clusters, simply obtain a token and use it to deploy to any cluster - securely. There's no risk that your functions will be exposed on the Internet, because the OIDC proxy only works for the /system endpoints of the OpenFaaS REST API.

\n

An alternative for those who need it

\n

OpenFaaS Enterprise has its own OIDC integration with much more fine-grained permissions implemented. It means that team members using the CLI, Dashboard or API do not need to memorise or share basic authentication credentials with each other, or worry about getting the right password for the right cluster.

\n

An OpenFaaS Enterprise policy can restrict all the way down to read/write permissions on a number of namespaces, and also integrates with OIDC.

\n

See an example:

\n","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/ollama-in-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/ollama-in-github-actions.json new file mode 100644 index 00000000..16be8e7f --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/ollama-in-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","contentHtml":"

That means you can run real end to end tests in CI with the same models you may use in dev and production. And if you use OpenAI or AWS SageMaker extensively, you could perhaps swap out what can be a very expensive API endpoint for your CI or testing environments to save money.

\n

If you'd like to learn more about how and why you'd want access to GPUs in CI, read my past update: Accelerate GitHub Actions with dedicated GPUs.

\n

We'll first cover what ollama is, why it's so popular, how to get it, what kinds of fun things you can do with it, then how to access it from actuated using a real GPU.

\n

\"ollama

\n
\n

ollama can now run in CI with isolated GPU acceleration using actuated

\n
\n

What's ollama?

\n

ollama is an open source project that aims to do for AI models, what Docker did for Linux containers. Whilst Docker created a user experience to share and run containers using container images in the Open Container Initiative (OCI) format, ollama bundles well-known AI models and makes it easy to run them without having to think about Python versions or Nvidia CUDA libraries.

\n

The project packages and runs various models, but seems to take its name from Meta's popular llama2 model, which whilst not released under an open source license, allows for a generous amount of free usage for most types of users.

\n

The ollama project can be run directly on a Linux, MacOS or Windows host, or within a container. There's a server component, and a CLI that acts as a client to pre-trained models. The main use-case today is that of inference - exercising the model with input data. A more recent feature means that you can create embeddings, if you pull a model that supports them.

\n

On Linux, ollama can be installed using a utility script:

\n
curl -fsSL https://ollama.com/install.sh | sh\n
\n

This provides the ollama CLI command.

\n

A quick tour of ollama

\n

After the initial installation, you can start a server:

\n
ollama serve\n
\n

By default, its REST API will listen on port 11434 on 127.0.0.1.

\n

You can find the reference for ollama's REST API here: API endpoints - which includes things like: creating a chat completion, pulling a model, or generating embeddings.

\n

You can then browse available models on the official website, which resembles the Docker Hub. This set currently includes: gemma (built upon Google's DeepMind), mistral (an LLM), codellama (for generating Code), phi (from Microsoft research), vicuna (for chat, based upon llama2), llava (a vision encoder), and many more.

\n

Most models will download with a default parameter size that's small enough to run on most CPUs or GPUs, but if you need to access it, there are larger models for higher accuracy.

\n

For instance, the llama2 model by Meta will default to the 7b model which needs around 8GB of RAM.

\n
# Pull the default model size:\n\nollama pull llama2\n\n# Override the parameter size\n\nollama pull llama2:13b\n
\n

Once you have a model, you can then either \"run\" it, where you'll be able to ask it questions and interact with it like you would with ChatGPT, or you can send it API requests from your own applications using REST and HTTP.

\n

For an interactive prompt, give no parameters:

\n
ollama run llama2\n
\n

To get an immediate response for use in i.e. scripts:

\n
ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n
\n

And you can use the REST API via curl, or your own codebase:

\n
curl -s http://localhost:11434/api/generate -d '{\n    \"model\": \"llama2\",\n    \"stream\": false,\n    \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n}' | jq\n
\n

We are just scratching the surface with what ollama can do, with a focus on testing and pulling pre-built models, but you can also create and share models using a Modelfile, which is another homage to the Docker experience by the ollama developers.

\n

Access ollama from Python code

\n

Here's how to access the API via Python, the stream parameter will emit JSON progressively when set to True, block until done if set to False. With Node.js, Python, Java, C#, etc the code will be very similar, but using your own preferred HTTP client. For Golang (Go) users, ollama founder Jeffrey Morgan maintains a higher-level Go SDK.

\n
import requests\nimport json\n\nurl = \"http://localhost:11434/api/generate\"\npayload = {\n    \"model\": \"llama2\",\n    \"stream\": False,\n    \"prompt\": \"What are the risks of running privileged Docker containers for CI workloads?\"\n}\nheaders = {\n    \"Content-Type\": \"application/json\"\n}\n\nresponse = requests.post(url, data=json.dumps(payload), headers=headers)\n\n# Parse the JSON response\nresponse_json = response.json()\n\n# Pretty print the JSON response\nprint(json.dumps(response_json, indent=4))\n
\n

When you're constructing a request by API, make sure you include any tags in the model name, if you've used one. I.e. \"model\": \"llama2:13b\".

\n

I hear from so many organisations who have gone to lengths to get SOC2 compliance, doing CVE scanning, or who are running Open Policy Agent or Kyverno to enforce strict Pod admission policies in Kubernetes, but then are happy to run their CI in Pods in privileged mode. So I asked the model why that may not be a smart idea. You can run the sample for yourself or see the response here. We also go into detail in the actuated FAQ, the security situation around self-hosted runners and containers is the main reason we built the solution.

\n

Putting it together for a GitHub Action

\n

The following GitHub Action will run on for customers who are enrolled for GPU support for actuated. If you'd like to gain access, contact us via the form on the Pricing page.

\n

The self-actuated/nvidia-run installs either the consumer or datacenter driver for Nvidia, depending on what you have in your system. This only takes about 30 seconds and could be cached if you like. The ollama models could also be cached using a local S3 bucket.

\n

Then, we simply run the equivalent bash commands from the previous section to:

\n\n
name: ollama-e2e\n\non:\n    workflow_dispatch:\n\njobs:\n    ollama-e2e:\n        name: ollama-e2e\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Install Ollama\n          run: |\n            curl -fsSL https://ollama.com/install.sh | sudo -E sh\n        - name: Start serving\n          run: |\n              # Run the background, there is no way to daemonise at the moment\n              ollama serve &\n\n              # A short pause is required before the HTTP port is opened\n              sleep 5\n\n              # This endpoint blocks until ready\n              time curl -i http://localhost:11434\n\n        - name: Pull llama2\n          run: |\n              ollama pull llama2\n\n        - name: Invoke via the CLI\n          run: |\n              ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n\n        - name: Invoke via API\n          run: |\n            curl -s http://localhost:11434/api/generate -d '{\n              \"model\": \"llama2\",\n              \"stream\": false,\n              \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n            }' | jq\n
\n

There is no built-in way to daemonise the ollama server, so for now we run it in the background using bash. The readiness endpoint can then be accessed which blocks until the server has completed its initialisation.

\n

Interactive access with SSH

\n

By modifying your CI job, you can drop into a remote SSH session and run interactive commands at any point in the workflow.

\n

That's how I came up with the commands for the Nvidia driver installation, and for the various ollama commands I shared.

\n

Find out more about SSH for GitHub Actions in the actuated docs.

\n

\"Pulling

\n
\n

Pulling one of the larger llama2 models interactively in an SSH session, directly to the runner VM

\n
\n

Wrapping up

\n

Within a very short period of time ollama helped us pull a popular AI model that can be used for chat and completions. We were then able to take what we learned and run it on a GPU at an accelerated speed and accuracy by using actuated's new GPU support for GitHub Actions and GitLab CI. Most hosted CI systems provide a relatively small amount of disk space for jobs, with actuated you can customise this and that may be important if you're going to be downloading large AI models. You can also easily customise the amount of RAM and vCPU using the runs-on label to any combination you need.

\n

ollama isn't the only way to find, download and run AI models, just like Docker wasn't the only way to download and install Nginx or Postgresql, but it provides a useful and convenient interface for those of us who are still learning about AI, and are not as concerned with the internal workings of the models.

\n

Over on the OpenFaaS blog, in the tutorial Stream OpenAI responses from functions using Server Sent Events, we covered how to stream a response from a model to a function, and then back to a user. There, we used the llama-api open source project, which is a single-purpose HTTP API for simulating llama2.

\n

One of the benefits of ollama is the detailed range of examples in the docs, and the ability to run other models that may include computer vision such as with the LLaVA: Large Language and Vision Assistant model or generating code with Code Llama.

\n

Right now, many of us are running and tuning models in development, some of us are using OpenAI's API or self-hosted models in production, but there's very little talk about doing thorough end to end testing or exercising models in CI. That's where actuated can help.

\n

Feel free to reach out for early access, or to see if we can help your team with your CI needs.

","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/right-sizing-vms-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/right-sizing-vms-github-actions.json similarity index 56% rename from _next/data/qP6XrePfh_ktdNhbSnok_/blog/right-sizing-vms-github-actions.json rename to _next/data/j64IiYEEq_eJG_mFKDqPk/blog/right-sizing-vms-github-actions.json index 5b850787..1d750441 100644 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/right-sizing-vms-github-actions.json +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/right-sizing-vms-github-actions.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","contentHtml":"

When we onboarded the etcd project from the CNCF, they'd previously been using a self-hosted runner for their repositories on a bare-metal host. There are several drawbacks to this approach, including potential security issues, especially when using Docker.

\n

actuated VM sizes can be configured by a label, and you can pick any combination of vCPU and RAM, there's no need to pick a pre-defined size.

\n

At the same time, it can be hard to know what size to pick, and if you make the VM size too large, then you won't be able to run as many jobs at once.

\n

There's three main things to consider:

\n\n

We wrote a tool called vmmeter which takes samples of resource consumption over the duration of a build, and will then report back with the peak and average values.

\n

vmmeter is written in Go, and is available to use as a pre-built binary. We may consider open-sourcing it in the future. The information you gather still needs to be carefully considered and some experimentation will be required to get the right balance between VM size and performance.

\n

The tool can be run in an action by adding some YAML, however, it can also be run on any Linux system using bash, or potentially within a different CI/CD system. See the note at the end if you're interested in trying that out.

\n

Running vmmeter inside GitHub Actions

\n

This action will work with a Linux VM environment, so with a hosted runner or with actuated. It may not work when used within the containers: section of a workflow, or with a Kubernetes-based runner.

\n

Add to the top of your GitHub action:

\n
\nsteps:\n\n# vmmeter start\n        - uses: alexellis/setup-arkade@master\n        - name: Install vmmeter\n          run: |\n            sudo -E arkade oci install ghcr.io/openfaasltd/vmmeter:latest --path /usr/local/bin/\n        - uses: self-actuated/vmmeter-action@master\n# vmmeter end\n
\n

The first set installs arkade, which we then use to extract vmmeter from a container image to the host.

\n

Then self-actuated/vmmeter-action is used to run the tool in the background, and also runs a post-setup setup to stop the measurements, and upload the results to the workflow run.

\n

To show you how the tool works, I ran a simple build of the Linux Kernel without any additional modules or options added in.

\n

Here's the summary text that was uploaded to the workflow run:

\n
Total RAM: 61.2GB\nTotal vCPU: 32\nLoad averages:\nMax 1 min: 5.63 (17.59%)\nMax 5 min: 1.25 (3.91%)\nMax 15 min: 0.41 (1.28%)\n\nRAM usage (10 samples):\nMax RAM usage: 2.528GB\n\nMax 10s avg RAM usage: 1.73GB\nMax 1 min avg RAM usage: 1.208GB\nMax 5 min avg RAM usage: 1.208GB\n\nDisk read: 374.2MB\nDisk write: 458.2MB\nMax disk I/O inflight: 0\nFree: 45.57GB\tUsed: 4.249GB\t(Total: 52.52GB)\n\nEgress adapter RX: 271.4MB\nEgress adapter TX: 1.535MB\n\nEntropy min: 256\nEntropy max: 256\n\nMax open connections: 125\nMax open files: 1696\nProcesses since boot: 18081\n\nRun time: 45s\n
\n

The main thing to look for is the peak load on the system. This roughly corresponds to the amount of vCPUs used at peak. If the number is close to the amount you allocated, then try allocating more and measuring the effect in build time and peak usage.

\n

We've found that some jobs are RAM hungry, and others use a lot of CPU. So if you find that the RAM requested is much higher than the peak or average usage, the chances are that you can safely reduce it.

\n

Disk usage is self-explanatory, if you've allocated around 30GB per VM, and a job is getting close to that limit, it may need increasing to avoid future failures.

\n

Disk, network read/write and open files are potential indicators of I/O contention. if a job reads or writes a large amount of data over the network interface, then that may become a bottleneck. Caching is one of the ways to work around that, whether you set up your workflow to use GitHub's hosted cache, or one running in the same datacenter or region as your CI servers.

\n

Wrapping up

\n

In one case, a build on the etcd-io project was specified with 16 vCPU and 32GB of RAM, but when running vmmeter, they found that less than 2 vCPU was used at peak and less than 3GB of RAM. That's a significant difference.

\n

Toolpath is a commercial customer, and we were able to help them reduce their wall time per pull request from 6 hours to 60 minutes. Or from 6x 1 hour jobs to 6x 15-20 minute jobs running in parallel. Jason Gray told me during a product market fit interview that \"the level of expertise and support pays for itself\". We'd noticed that his teams jobs were requesting far too much CPU, but not enough RAM and were able to make recommendations. We then saw that disk space was running dangerously low, and were able to reconfigure their dedicated build servers for them, remotely, without them having to even think about it.

\n

If you'd like to try out vmmeter, it's free to use on GitHub's hosted runners and on actuated runners. We wouldn't recommend making it a permanent fixture in your workflow, because if it were to fail or exit early for any reason, it may mark the whole build as a failure.

\n

Instead, we recommend you use it learn and explore, and fine-tune your VM sizes. Getting the numbers closer to a right-size could reduce your costs with hosted runners and your efficiency with actuated runners.

\n

The source-code for the action is available here: self-actuated/vmmeter-action.

\n\n

What if you're not using GitHub Actions?

\n

You can run vmmeter with bash on your own system, and may also able to use vmmeter in GitLab CI or Jenkins. We've got manual instructions for vmmeter here. You can even just start it up right now, do some work and then call the collect endpoint to see what was used over that period of time, a bit like a generic profiler.

","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","contentHtml":"

When we onboarded the etcd project from the CNCF, they'd previously been using a self-hosted runner for their repositories on a bare-metal host. There are several drawbacks to this approach, including potential security issues, especially when using Docker.

\n

actuated VM sizes can be configured by a label, and you can pick any combination of vCPU and RAM, there's no need to pick a pre-defined size.

\n

At the same time, it can be hard to know what size to pick, and if you make the VM size too large, then you won't be able to run as many jobs at once.

\n

There's three main things to consider:

\n\n

We wrote a tool called vmmeter which takes samples of resource consumption over the duration of a build, and will then report back with the peak and average values.

\n

vmmeter is written in Go, and is available to use as a pre-built binary. We may consider open-sourcing it in the future. The information you gather still needs to be carefully considered and some experimentation will be required to get the right balance between VM size and performance.

\n

The tool can be run in an action by adding some YAML, however, it can also be run on any Linux system using bash, or potentially within a different CI/CD system. See the note at the end if you're interested in trying that out.

\n

Running vmmeter inside GitHub Actions

\n

This action will work with a Linux VM environment, so with a hosted runner or with actuated. It may not work when used within the containers: section of a workflow, or with a Kubernetes-based runner.

\n

Add to the top of your GitHub action:

\n
\nsteps:\n\n# vmmeter start\n        - uses: alexellis/setup-arkade@master\n        - name: Install vmmeter\n          run: |\n            sudo -E arkade oci install ghcr.io/openfaasltd/vmmeter:latest --path /usr/local/bin/\n        - uses: self-actuated/vmmeter-action@master\n# vmmeter end\n
\n

The first set installs arkade, which we then use to extract vmmeter from a container image to the host.

\n

Then self-actuated/vmmeter-action is used to run the tool in the background, and also runs a post-setup setup to stop the measurements, and upload the results to the workflow run.

\n

To show you how the tool works, I ran a simple build of the Linux Kernel without any additional modules or options added in.

\n

Here's the summary text that was uploaded to the workflow run:

\n
Total RAM: 61.2GB\nTotal vCPU: 32\nLoad averages:\nMax 1 min: 5.63 (17.59%)\nMax 5 min: 1.25 (3.91%)\nMax 15 min: 0.41 (1.28%)\n\nRAM usage (10 samples):\nMax RAM usage: 2.528GB\n\nMax 10s avg RAM usage: 1.73GB\nMax 1 min avg RAM usage: 1.208GB\nMax 5 min avg RAM usage: 1.208GB\n\nDisk read: 374.2MB\nDisk write: 458.2MB\nMax disk I/O inflight: 0\nFree: 45.57GB\tUsed: 4.249GB\t(Total: 52.52GB)\n\nEgress adapter RX: 271.4MB\nEgress adapter TX: 1.535MB\n\nEntropy min: 256\nEntropy max: 256\n\nMax open connections: 125\nMax open files: 1696\nProcesses since boot: 18081\n\nRun time: 45s\n
\n

The main thing to look for is the peak load on the system. This roughly corresponds to the amount of vCPUs used at peak. If the number is close to the amount you allocated, then try allocating more and measuring the effect in build time and peak usage.

\n

We've found that some jobs are RAM hungry, and others use a lot of CPU. So if you find that the RAM requested is much higher than the peak or average usage, the chances are that you can safely reduce it.

\n

Disk usage is self-explanatory, if you've allocated around 30GB per VM, and a job is getting close to that limit, it may need increasing to avoid future failures.

\n

Disk, network read/write and open files are potential indicators of I/O contention. if a job reads or writes a large amount of data over the network interface, then that may become a bottleneck. Caching is one of the ways to work around that, whether you set up your workflow to use GitHub's hosted cache, or one running in the same datacenter or region as your CI servers.

\n

Wrapping up

\n

In one case, a build on the etcd-io project was specified with 16 vCPU and 32GB of RAM, but when running vmmeter, they found that less than 2 vCPU was used at peak and less than 3GB of RAM. That's a significant difference.

\n

Toolpath is a commercial customer, and we were able to help them reduce their wall time per pull request from 6 hours to 60 minutes. Or from 6x 1 hour jobs to 6x 15-20 minute jobs running in parallel. Jason Gray told me during a product market fit interview that \"the level of expertise and support pays for itself\". We'd noticed that his teams jobs were requesting far too much CPU, but not enough RAM and were able to make recommendations. We then saw that disk space was running dangerously low, and were able to reconfigure their dedicated build servers for them, remotely, without them having to even think about it.

\n

If you'd like to try out vmmeter, it's free to use on GitHub's hosted runners and on actuated runners. We wouldn't recommend making it a permanent fixture in your workflow, because if it were to fail or exit early for any reason, it may mark the whole build as a failure.

\n

Instead, we recommend you use it learn and explore, and fine-tune your VM sizes. Getting the numbers closer to a right-size could reduce your costs with hosted runners and your efficiency with actuated runners.

\n

The source-code for the action is available here: self-actuated/vmmeter-action.

\n\n

What if you're not using GitHub Actions?

\n

You can run vmmeter with bash on your own system, and may also able to use vmmeter in GitLab CI or Jenkins. We've got manual instructions for vmmeter here. You can even just start it up right now, do some work and then call the collect endpoint to see what was used over that period of time, a bit like a generic profiler.

","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/sbom-in-github-actions.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/sbom-in-github-actions.json new file mode 100644 index 00000000..6dc1595f --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/sbom-in-github-actions.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","contentHtml":"

What is a Software Bill of Materials (SBOM)?

\n

In April 2022 Justin Cormack, CTO of Docker announced that Docker was adding support to generate a Software Bill of Materials (SBOM) for container images.

\n

An SBOM is an inventory of the components that make up a software application. It is a list of the components that make up a software application including the version of each component. The version is important because it can be cross-reference with a vulnerability database to determine if the component has any known vulnerabilities.

\n

Many organisations are also required to company with certain Open Source Software (OSS) licenses. So if SBOMs are included in the software they purchase or consume from vendors, then it can be used to determine if the software is compliant with their specific license requirements, lowering legal and compliance risk.

\n

Docker's enhancements to Docker Desktop and their open source Buildkit tool were the result of a collaboration with Anchore, a company that provides a commercial SBOM solution.

\n

Check out an SBOM for yourself

\n

Anchore provides commercial solutions for creating, managing and inspecting SBOMs, however they also have two very useful open source tools that we can try out for free.

\n\n

OpenFaaS Community Edition (CE) is a popular open source serverless platform for Kubernetes. It's maintained by open source developers, and is free to use.

\n

Let's pick a container image from the Community Edition of OpenFaaS like the container image for the OpenFaaS gateway.

\n

We can browse the GitHub UI to find the latest revision, or we can use Google's crane tool:

\n
crane ls ghcr.io/openfaas/gateway | tail -n 5\n0.26.0\n8e1c34e222d6c194302c649270737c516fe33edf\n0.26.1\nc26ec5221e453071216f5e15c3409168446fd563\n0.26.2\n
\n

Now we can introduce one of those tags to syft:

\n
syft ghcr.io/openfaas/gateway:0.26.2\n ✔ Pulled image            \n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [39 packages]\nNAME                                              VERSION                               TYPE      \nalpine-baselayout                                 3.4.0-r0                              apk        \nalpine-baselayout-data                            3.4.0-r0                              apk        \nalpine-keys                                       2.4-r1                                apk        \napk-tools                                         2.12.10-r1                            apk        \nbusybox                                           1.35.0                                binary     \nbusybox                                           1.35.0-r29                            apk        \nbusybox-binsh                                     1.35.0-r29                            apk        \nca-certificates                                   20220614-r4                           apk        \nca-certificates-bundle                            20220614-r4                           apk        \ngithub.com/beorn7/perks                           v1.0.1                                go-module  \ngithub.com/cespare/xxhash/v2                      v2.1.2                                go-module  \ngithub.com/docker/distribution                    v2.8.1+incompatible                   go-module  \ngithub.com/gogo/protobuf                          v1.3.2                                go-module  \ngithub.com/golang/protobuf                        v1.5.2                                go-module  \ngithub.com/gorilla/mux                            v1.8.0                                go-module  \ngithub.com/matttproud/golang_protobuf_extensions  v1.0.1                                go-module  \ngithub.com/nats-io/nats.go                        v1.22.1                               go-module  \ngithub.com/nats-io/nkeys                          v0.3.0                                go-module  \ngithub.com/nats-io/nuid                           v1.0.1                                go-module  \ngithub.com/nats-io/stan.go                        v0.10.4                               go-module  \ngithub.com/openfaas/faas-provider                 v0.19.1                               go-module  \ngithub.com/openfaas/faas/gateway                  (devel)                               go-module  \ngithub.com/openfaas/nats-queue-worker             v0.0.0-20230117214128-3615ccb286cc    go-module  \ngithub.com/prometheus/client_golang               v1.13.0                               go-module  \ngithub.com/prometheus/client_model                v0.2.0                                go-module  \ngithub.com/prometheus/common                      v0.37.0                               go-module  \ngithub.com/prometheus/procfs                      v0.8.0                                go-module  \ngolang.org/x/crypto                               v0.5.0                                go-module  \ngolang.org/x/sync                                 v0.1.0                                go-module  \ngolang.org/x/sys                                  v0.4.1-0.20230105183443-b8be2fde2a9e  go-module  \ngoogle.golang.org/protobuf                        v1.28.1                               go-module  \nlibc-utils                                        0.7.2-r3                              apk        \nlibcrypto3                                        3.0.7-r2                              apk        \nlibssl3                                           3.0.7-r2                              apk        \nmusl                                              1.2.3-r4                              apk        \nmusl-utils                                        1.2.3-r4                              apk        \nscanelf                                           1.3.5-r1                              apk        \nssl_client                                        1.35.0-r29                            apk        \nzlib                                              1.2.13-r0                             apk  \n
\n

These are all the components that syft found in the container image. We can see that it found 39 packages, including the OpenFaaS gateway itself.

\n

Some of the packages are Go modules, others are packages that have been installed with apk (Alpine Linux's package manager).

\n

Checking for vulnerabilities

\n

Now that we have an SBOM, we can use grype to check for vulnerabilities.

\n
grype ghcr.io/openfaas/gateway:0.26.2\n ✔ Vulnerability DB        [no update available]\n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [39 packages]\n ✔ Scanned image           [2 vulnerabilities]\nNAME                        INSTALLED  FIXED-IN  TYPE       VULNERABILITY   SEVERITY \ngoogle.golang.org/protobuf  v1.28.1              go-module  CVE-2015-5237   High      \ngoogle.golang.org/protobuf  v1.28.1              go-module  CVE-2021-22570  Medium  \n
\n

In this instance, we can see there are only two vulnerabilities, both of which are in the google.golang.org/protobuf Go module, and neither of them have been fixed yet.

\n\n

With this scenario, I wanted to show that different people care about the supply chain, and have different responsibilities for it.

\n

Generate an SBOM from within GitHub Actions

\n

The examples above were all run locally, but we can also generate an SBOM from within a GitHub Actions workflow. In this way, the SBOM is shipped with the container image and is made available without having to scan the image each time.

\n

Imagine you have the following Dockerfile:

\n
FROM alpine:3.17.0\n\nRUN apk add --no-cache curl ca-certificates\n\nCMD [\"curl\", \"https://www.google.com\"]\n
\n

I know that there's a vulnerability in alpine 3.17.0 in the OpenSSL library. How do I know that? I recently updated every OpenFaaS Pro component to use 3.17.1 to fix a specific vulnerability.

\n

Now a typical workflow for this Dockerfile would look like the below:

\n
name: build\n\non:\n  push:\n    branches: [ master, main ]\n  pull_request:\n    branches: [ master, main ]\n\npermissions:\n  actions: read\n  checks: write\n  contents: read\n  packages: write\n\njobs:\n  publish:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n\n      - name: Login to Docker Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Publish image\n        uses: docker/build-push-action@v4\n        with:\n          build-args: |\n            GitCommit=${{ github.sha }}\n          outputs: \"type=registry,push=true\"\n          provenance: false\n          tags: |\n            ghcr.io/alexellis/gha-sbom:${{ github.sha }}\n
\n

Upon each commit, an image is published to GitHub's Container Registry with the image name of: ghcr.io/alexellis/gha-sbom:SHA.

\n

To generate an SBOM, we just need to update the docker/build-push-action to use the sbom flag:

\n
      - name: Local build\n        id: local_build\n        uses: docker/build-push-action@v4\n        with:\n          sbom: true\n          provenance: false\n
\n

By checking the logs from the action, we can see that the image has been published with an SBOM:

\n
#16 [linux/amd64] generating sbom using docker.io/docker/buildkit-syft-scanner:stable-1\n#0 0.120 time=\"2023-01-25T15:35:19Z\" level=info msg=\"starting syft scanner for buildkit v1.0.0\"\n#16 DONE 1.0s\n
\n

The SBOM can be viewed as before:

\n
syft ghcr.io/alexellis/gha-sbom:46bc16cb4033364233fad3caf8f3a255b5b4d10d@sha256:7229e15004d8899f5446a40ebdd072db6ff9c651311d86e0c8fd8f999a32a61a\n\ngrype ghcr.io/alexellis/gha-sbom:46bc16cb4033364233fad3caf8f3a255b5b4d10d@sha256:7229e15004d8899f5446a40ebdd072db6ff9c651311d86e0c8fd8f999a32a61a\n ✔ Vulnerability DB        [updated]\n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [21 packages]\n ✔ Scanned image           [2 vulnerabilities]\nNAME        INSTALLED  FIXED-IN  TYPE  VULNERABILITY  SEVERITY \nlibcrypto3  3.0.7-r0   3.0.7-r2  apk   CVE-2022-3996  High      \nlibssl3     3.0.7-r0   3.0.7-r2  apk   CVE-2022-3996  High  \n
\n

The image: alpine:3.17.0 contains two High vulnerabilities, and from reading the notes, we can see that both have been fixed.

\n

We can resolve the issue by changing the Dockerfile to use alpine:3.17.1 instead, and re-running the build.

\n
grype ghcr.io/alexellis/gha-sbom:63c6952d1ded1f53b1afa3f8addbba9efa37b52b\n ✔ Vulnerability DB        [no update available]\n ✔ Pulled image            \n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [21 packages]\n ✔ Scanned image           [0 vulnerabilities]\nNo vulnerabilities found\n
\n

Wrapping up

\n

There is a lot written on the topic of supply chain security, so I wanted to give you a quick overview, and how to get started wth it.

\n

We looked at Anchore's two open source tools: Syft and Grype, and how they can be used to generate an SBOM and scan for vulnerabilities.

\n

We then produced an SBOM for a pre-existing Dockerfile and GitHub Action, introducing a vulnerability by using an older base image, and then fixing it by upgrading it. We did this by adding additional flags to the docker/build-push-action. We added the sbom flag, and set the provenance flag to false. Provenance is a separate but related topic, which is explained well in an article by Justin Chadwell of Docker (linked below).

\n

I maintain an Open Source alternative to brew for developer-focused CLIs called arkade. This already includes Google's crane project, and there's a Pull Request coming shortly to add Syft and Grype to the project.

\n

It can be a convenient way to install these tools on MacOS, Windows or Linux:

\n
# Available now\narkade get crane syft\n\n# Coming shortly\narkade get grype\n
\n

In the beginning of the article we mentioned license compliance. SBOMs generated by syft do not seem to include license information, but in my experience, corporations which take this risk seriously tend to run their own scanning infrastructure with commercial tools like Blackduck or Twistlock.

\n

Tools like Twistlock, and certain registries like JFrog Artifactory and the CNCF's Harbor, can be configured to scan images. GitHub has a free, built-in service called Dependabot that won't just scan, but will also send Pull Requests to fix issues.

\n

But with the SBOM approach, the responsibility is rebalanced, with the supplier taking on an active role in security. The consumer can then use the supplier's SBOMs, or run their own scanning infrastructure - or perhaps both.

\n

See also:

\n","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/secure-microvm-ci-gitlab.json b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/secure-microvm-ci-gitlab.json new file mode 100644 index 00000000..28679689 --- /dev/null +++ b/_next/data/j64IiYEEq_eJG_mFKDqPk/blog/secure-microvm-ci-gitlab.json @@ -0,0 +1 @@ +{"pageProps":{"post":{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","contentHtml":"

We started building actuated for GitHub Actions because we at OpenFaaS Ltd had a need for: unmetered CI minutes, faster & more powerful x86 machines, native Arm builds and low maintenance CI builds.

\n

And most importantly, we needed it to be low-maintenance, and securely isolated.

\n

None of the solutions at the time could satisfy all of those requirements, and even today with GitHub adopting the community-based Kubernetes controller to run CI in Pods, there is still a lot lacking.

\n

As we've gained more experience with customers who largely had the same needs as we did for GitHub Actions, we started to hear more and more from GitLab CI users. From large enterprise companies who are concerned about the security risks of running CI with privileged Docker containers, Docker socket binding (from the host!) or the flakey nature and slow speed of VFS with Docker In Docker (DIND).

\n
\n

The GitLab docs have a stark warning about using both of these approaches. It was no surprise that when a consultant at Thoughtworks reached out to me, he listed off the pain points and concerns that we'd set out to solve for GitHub Actions.

\n

At KubeCon, I also spoke to several developers who worked at Deutsche Telekom who had numerous frustrations with the user-experience and management overhead of the Kubernetes executor.

\n
\n

So with growing interest from customers, we built a solution for GitLab CI - just like we did for GitHub Actions. We're excited to share it with you today in tech preview.

\n

\"actuated

\n

For every build that requires a runner, we will schedule and boot a complete system with Firecracker using Linux KVM for secure isolation. After the job is completed, the VM will be destroyed and removed from the GitLab instance.

\n

actuated for GitLab is for self-hosted GitLab instances, whether hosted on-premises or on the public cloud.

\n

If you'd like to use it or find out more, please apply here: Sign-up for the Actuated pilot

\n

Secure CI with Firecracker microVMs

\n

Firecracker is the open-source technology that provides isolation between tenants on certain AWS products like Lambda and Fargate. There's a growing number of cloud native solutions evolving around Firecracker, and we believe that it's the only way to run CI/CD securely.

\n

Firecracker is a virtual machine monitor (VMM) that uses the Linux Kernel-based Virtual Machine (KVM) to create and manage microVMs. It's lightweight, fast, and most importantly, provides proper isolation, which anything based upon Docker cannot.

\n

There are no horrible Kernel tricks or workarounds to be able to use user namespaces, no need to change your tooling from what developers love - Docker, to Kaninko or Buildah or similar.

\n

You'll get sudo, plus a fresh Docker engine in every VM, booted up with systemd, so things like Kubernetes work out of the box, if you need them for end to end testing (as so many of us do these days).

\n

You can learn the differences between VMs, containers and microVMs like Firecracker in my video from Cloud Native Rejekts at KubeCon Amsterdam:

\n\n

Many people have also told me that they learned how to use Firecracker from my webinar last year with Richard Case: A cracking time: Exploring Firecracker & MicroVMs.

\n

Let's see it then

\n

Here's a video demo of the tech preview we have available for customers today.

\n\n

You'll see that when I create a commit in our self-hosted copy of GitLab Enterprise, within 1 second a microVM is booting up and running the CI job.

\n

Shortly after that the VM is destroyed which means there are absolutely no side-effects or any chance of leakage between jobs.

\n

Here's a later demo of three jobs within a single pipeline, all set to run in parallel.

\n

Here's 3x @GitLab CI jobs running in parallel within the same Pipeline demoed by @alexellisuk

All in their own ephemeral VM powered by Firecracker 🔥#cicd #secure #isolation #microvm #baremetal pic.twitter.com/fe5HaxMsGB

— actuated (@selfactuated) June 13, 2023
\n

Everything's completed before I have a chance to even open the logs in the UI of GitLab.

\n

Wrapping up

\n

actuated for GitLab is for self-hosted GitLab instances, whether hosted on-premises or on the public cloud.

\n

Here's what we bring to the table:

\n\n

Runners are registered and running a job in a dedicated VM within less than one second. Our scheduler can pack in jobs across a fleet of servers, they just need to have KVM available.

\n

If you think your automation for runners could be improved, or work with customers who need faster builds, better isolation or Arm support, get in touch with us.

\n\n

You can follow @selfactuated on Twitter, or find me there too to keep an eye on what we're building.

","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog.json deleted file mode 100644 index 7fdf3055..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"posts":[{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"},{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"},{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"},{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"},{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"},{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"},{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"},{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"},{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"},{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"},{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"},{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"},{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"},{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"},{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"},{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"},{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"},{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"},{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"},{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"},{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"},{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"},{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"},{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"},{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"},{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"},{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"},{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"},{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}]},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/arm-ci-cncf-ampere.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/arm-ci-cncf-ampere.json deleted file mode 100644 index 5faa3e17..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/arm-ci-cncf-ampere.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","contentHtml":"

In this post, we'll cover why Ampere Computing and The Cloud Native Computing Foundation (CNCF) are sponsoring a pilot of actuated for open source projects, how you can get involved.

\n

We'll also give you a quick one year recap on actuated, if you haven't checked in with us for a while.

\n

Managed Arm CI for CNCF projects

\n

At KubeCon EU, I spoke to Chris Aniszczyk, CTO at the Cloud Native Computing Foundation (CNCF), and told him about some of the results we'd been seeing with actuated customers, including Fluent Bit, which is a CNCF project. Chris told me that many teams were either putting off Arm support all together, were suffering with the slow builds that come from using QEMU, or were managing their own infrastructure which was underutilized.

\n

Equinix provides a generous amount of credits to the CNCF under CNCF Community Infrastructure Lab (CIL), including access to powerful Ampere Q80 Arm servers (c3.large.arm64), that may at times be required by Equinix customers for their own Arm workloads.

\n

You can find out more about Ampere's Altra platform here, which is being branded as a \"Cloud Native\" CPU, due to its low power consumption, high core count, and ubiquitous availability across Google Cloud, Oracle Cloud Platform, Azure, Equinix Metal, Hetzner Cloud, and Alibaba Cloud.

\n

As you can imagine, over time, different projects have deployed 1-3 of their own runner servers, each with 256GB of RAM and 80 Cores, which remain idle most of the time, and are not available to other projects or Equinix customers when they may need them suddenly. So, if actuated can reduce this number, whilst also improving the experience for maintainers, then that's a win-win.

\n

Around the same time as speaking to Chris, Ampere reached out and asked how they could help secure actuated for a number of CNCF projects.

\n

Together, Ampere and the CNCF are now sponsoring an initial 1-year pilot of managed Arm CI provided by actuated, for CNCF projects, with the view to expand it, if the pilot is a success.

\n

Ed Vielmetti, Developer Partner Manager at Equinix said:

\n
\n

I'm really happy to see this all come together. If all goes according to plan, we'll have better runner isolation, faster builds, and a smaller overall machine footprint.

\n
\n

Dave Neary, Director of Developer Relations at Ampere Computing added:

\n
\n

Actuated offers a faster, more secure way for projects to run 64-bit Arm builds, and will also more efficiently use the Ampere Altra-based servers being used by the projects.

\n

We're happy to support CNCF projects running their CI on Ampere Computing's Cloud Native Processors, hosted by Equinix.

\n
\n

One year recap on actuated

\n

In case you are hearing about actuated for the first time, I wanted to give you a quick one year recap.

\n

Just over 12 months ago, we announced the work we'd been doing with actuated to improve self-hosted runner security and management. We were pleasantly surprised with the amount of people that responded who'd had a common experience with slow builds, running out of RAM, limited disk space, and a lack of an easy and secure way to run self-hosted runners.

\n

Fast forward to today, and we have run over 140,000 individual Firecracker VMs for customers on their own hardware. Rather than the fully managed service that GitHub offers, we believe that you should be able to bring your own hardware, and pay a flat-rate fee for the service, rather than being charged per-minute.

\n

The CNCF project brings about 64-bit Arm support, but we see a good mix of x86_64 and Arm builds from customers, with both closed and open-source repositories being used.

\n

The main benefits are having access to bigger, faster and more specialist hardware.

\n\n

Vendors and consumers are becoming increasingly aware of the importance of the supply chain, GitHub's self-hosted runner is not recommended for open source repos. Why? Due to the way side-effects can be left over between builds. Actuated uses a fresh, immutable, Firecracker VM for every build which boots up in less than 1 second and is destroyed after the build completes, which removes this risk.

\n

If you're wanting to know more about why we think microVMs are the only tool that makes sense for secure CI, then I'd recommend my talk from Cloud Native Rejekts earlier in the year: Face off: VMs vs. Containers vs Firecracker.

\n

What are maintainers saying?

\n

Ellie Huxtable is the maintainer of Atuin, a popular open-source tool to sync, search and backup shell history. Her Rust build for the CLI took 90 minutes with QEMU, but was reduced to just 3 minutes with actuated, and a native Arm server.

\n

Thanks to @selfactuated, Atuin now has very speedy ARM docker builds in our GitHub actions! Thank you @alexellisuk 🙏

Docker builds on QEMU: nearly 90 mins
Docker builds on ARM with Actuated: ~3 mins

— Ellie Huxtable (@ellie_huxtable) October 20, 2023
\n

For Fluent Bit, one of their Arm builds was taking over 6 hours, which meant it always failed with a timed-out on a hosted runner. Patrick Stephens, Tech Lead of Infrastructure at Calyptia reached out to work with us. We got the time down to 5 minutes by changing runs-on: ubuntu-latest to runs-on: actuated-arm64-4cpu-16gb, and if you need more or less RAM/CPU, you can tune those numbers as you wish.

\n

Patrick shares about the experience on the Calyptia blog, including the benefits to their x86_64 builds for the commercial Calyptia product: Scaling ARM builds with Actuated.

\n

A number of CNCF maintainers and community leaders such as Davanum Srinivas (Dims), Principal Engineer at AWS have come forward with project suggestions, and we're starting to work through them, with the first two being Fluent Bit and etcd.

\n

Fluent Bit describes itself as:

\n
\n

..a super fast, lightweight, and highly scalable logging and metrics processor and forwarder. It is the preferred choice for cloud and containerized environments.

\n
\n

etcd is a core component of almost every Kubernetes installation and is responsible for storing the state of the cluster.

\n
\n

A distributed, reliable key-value store for the most critical data of a distributed system

\n
\n

In the case of etcd, there were two servers being maintained by five individual maintainers, all of that work goes away by adopting actuated.

\n

We even sent etcd a minimal Pull Request to make the process smoother.

\n

James Blair, Specialist Solution Architect at Red Hat, commented:

\n
\n

I believe managed on demand arm64 CI hosts will definitely be a big win for the project. Keen to trial this.

\n
\n

Another maintainer also commented that they will no longer need to worry about \"leaky containers\".

\n

\"One

\n
\n

One of the first nightly workflows running within 4x separate isolated Firecracker VMs, one per job

\n
\n

Prior to adopting actuated, the two servers were only configured to run one job at a time, afterwards, the jobs are scheduled by the control-plane, according to the amount of available RAM and CPU in the target servers.

\n

How do we get access?

\n

If you are working on a CNCF project and would like access, please contact us via this form. If your project gets selected for the pilot, there are a couple of things you may need to do.

\n
    \n
  1. If you are already using the GitHub Actions runner hosted on Equinix, then change the runs-on: self-hosted label to: runs-on: actuated-arm64-8cpu-16gb. Then after you've seen a build or two pass, delete the old runner.
    2. \n
  2. If you are using QEMU, the best next step is to \"split\" the build into two jobs which can run on x86_64 and arm64 natively, that's what Ellie did and it only took her a few minutes.
    3. \n
\n

The label for runs-on: allows for dynamic configuration of vCPU and GBs of RAM, just edit the label to match your needs, for etcd, the team asked for 8vCPU and 32GB of RAM, so they used runs-on: actuated-arm64-8cpu-32gb.

\n

I had to split the docker build so that the ARM half would build on ARM, and x86 on x86, and then a step to combine the two - overall this works out to be a very significant improvementhttps://t.co/69cIxjYRcW

— Ellie Huxtable (@ellie_huxtable) October 20, 2023
\n

We have full instructions for 2, in the following tutorial: How to split up multi-arch Docker builds to run natively.

\n

Is there access for AMD64?

\n

This program is limited to CNCF projects and Arm CI only. That said, most actuated customers run AMD64 builds with us.

\n

GitHub already provides access to AMD64 runners for free for open source projects, that should cover most OSS project's needs.

\n

So why would you want dedicated AMD64 support from actuated? Firstly, our recommended provider makes builds up to 3x quicker, secondly, you can run on private repos if required, without accuring a large bill.

\n

What are all the combinations of CPU and RAM?

\n

We get this question very often, but have tried to be as clear as possible in this blog post and in the docs. There are no set combinations. You can come up with what you need.

\n

That helps us make best use of the hardware, you can even have just a couple of cores, and max out to 256GB of RAM, if that's what your build needs.

\n

What if the sponsored program is full?

\n

The program has been very popular and there is a limit to the budget and number of projects that Ampere and the CNCF agreed to pay for. If you contact us and we tell you the limit has been reached, then your employer could sponsor the subscription, and we'll give you a special discount - you could get started immediately. Or you'll need to contact Chris Aniszczyk and tell him why it would be of value to the OSS project you represent to have native Arm CI. If you get in touch with us, we can introduce you to him via email if needed.

\n

Learn more about actuated

\n

We're initially offering access to managed Arm CI for CNCF projects, but if you're working for a company that is experiencing friction with CI, please reach out to us to talk using this form.

\n

Ampere who are co-sponsoring our service with the CNCF have their own announcement here: Ampere Computing and CNCF Supporting Arm Native CI for CNCF Projects.

\n\n
\n

Did you know? Actuated for GitLab CI is now in technical preview, watch a demo here.

\n
","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/automate-packer-qemu-image-builds.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/automate-packer-qemu-image-builds.json deleted file mode 100644 index 8f9b1910..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/automate-packer-qemu-image-builds.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","contentHtml":"

One of the most popular tools for creating images for virtual machines is Packer by Hashicorp. Packer automates the process of building images for a variety of platforms from a single source configuration. Different builders can be used to create machines and generate images from those machines.

\n

In this tutorial we will use the QEMU builder to create a KVM virtual machine image.

\n

We will see how the Packer build can be completely automated by integrating Packer into a continuous integration (CI) pipeline with GitHub Actions. The workflow will automatically trigger image builds on changes and publish the resulting images as GitHub release artifacts.

\n

Actuated supports nested virtualsation where a VM can make use of KVM to launch additional VMs within a GitHub Action. This makes it possible to run the Packer QEMU builder in GitHub Action workflows. Something that is not possible with GitHub's default hosted runners.

\n
\n

See also: How to run KVM guests in your GitHub Actions

\n
\n

Create the Packer template

\n

We will be starting from a Ubuntu Cloud Image and modify it to suit our needs. If you need total control of what goes into the image you can start from scratch using the ISO.

\n

Variables are used in the packer template to set the iso_url and iso_checksum. In addition to these we also use variables to configure the disk_size, ram, cpu, ssh_password and ssh_username:

\n
variable \"cpu\" {\n  type    = string\n  default = \"2\"\n}\n\nvariable \"disk_size\" {\n  type    = string\n  default = \"40000\"\n}\n\nvariable \"headless\" {\n  type    = string\n  default = \"true\"\n}\n\nvariable \"iso_checksum\" {\n  type    = string\n  default = \"sha256:d699ae158ec028db69fd850824ee6e14c073b02ad696b4efb8c59d37c8025aaa\"\n}\n\nvariable \"iso_url\" {\n  type    = string\n  default = \"https://cloud-images.ubuntu.com/jammy/20230719/jammy-server-cloudimg-amd64.img\"\n}\n\nvariable \"name\" {\n  type    = string\n  default = \"jammy\"\n}\n\nvariable \"ram\" {\n  type    = string\n  default = \"2048\"\n}\n\nvariable \"ssh_password\" {\n  type    = string\n  default = \"ubuntu\"\n}\n\nvariable \"ssh_username\" {\n  type    = string\n  default = \"ubuntu\"\n}\n\nvariable \"version\" {\n  type    = string\n  default = \"\"\n}\n\nvariable \"format\" {\n  type    = string\n  default = \"qcow2\"\n}\n
\n

The Packer source configuration:

\n
source \"qemu\" \"jammy\" {\n  accelerator      = \"kvm\"\n  boot_command     = []\n  disk_compression = true\n  disk_interface   = \"virtio\"\n  disk_image       = true\n  disk_size        = var.disk_size\n  format           = var.format\n  headless         = var.headless\n  iso_checksum     = var.iso_checksum\n  iso_url          = var.iso_url\n  net_device       = \"virtio-net\"\n  output_directory = \"artifacts/qemu/${var.name}${var.version}\"\n  qemuargs = [\n    [\"-m\", \"${var.ram}M\"],\n    [\"-smp\", \"${var.cpu}\"],\n    [\"-cdrom\", \"cidata.iso\"]\n  ]\n  communicator           = \"ssh\"\n  shutdown_command       = \"echo '${var.ssh_password}' | sudo -S shutdown -P now\"\n  ssh_password           = var.ssh_password\n  ssh_username           = var.ssh_username\n  ssh_timeout            = \"10m\"\n}\n
\n

Some notable settings in the source configuration:

\n\n

In the next section we will see how cloud-init is used to setup user account with the correct password that Packer needs for provisioning.

\n

The full example of the packer file is available on GitHub.

\n

Create the user-data file

\n

Cloud images provided by Canonical do not have users by default. The Ubuntu images use cloud-init to pre-configure the system during boot.

\n

Packer uses provisioners to install and configure the machine image after booting. To run these provisioners Packer needs to be able to communicate with the machine. By default this happens by establishing an ssh connection to the machine.

\n

Create a user-data file that sets the password of the default user so that it can be used by Packer to connect over ssh:

\n
#cloud-config\npassword: ubuntu\nssh_pwauth: true\nchpasswd:\n  expire: false\n
\n

Next create an ISO that can be referenced by our Packer template and presented to the VM:

\n
genisoimage -output cidata.iso -input-charset utf-8 -volid cidata -joliet -r \\\n
\n

The ISO can be mounted by QEMU to provide the configuration data to cloud-init while the VM boots.

\n

The -cdrom flag is used in the qemuargs field to mount the cidata.iso file:

\n
  qemuargs = [\n    [\"-m\", \"${var.ram}M\"],\n    [\"-smp\", \"${var.cpu}\"],\n    [\"-cdrom\", \"cidata.iso\"]\n  ]\n
\n

Provision the image

\n

The build section of the Packer template is used to define provisioners that can run scripts and commands to install software and configure the machine.

\n

In this example we are installing python3 but you can run any script you want or use tools like Ansible to automate the configuration.

\n
build {\n  sources = [\"source.qemu.jammy\"]\n\n  provisioner \"shell\" {\n    execute_command = \"{{ .Vars }} sudo -E bash '{{ .Path }}'\"\n    inline          = [\"sudo apt update\", \"sudo apt install python3\"]\n  }\n\n  post-processor \"shell-local\" {\n    environment_vars = [\"IMAGE_NAME=${var.name}\", \"IMAGE_VERSION=${var.version}\", \"IMAGE_FORMAT=${var.format}\"]\n    script           = \"scripts/prepare-image.sh\"\n  }\n}\n
\n

Prepare the image for publishing.

\n

Packer supports post-processors. They only run after Packer saves an instance as an image. Post-processors are commonly used to compress artifacts, upload them into a cloud, etc. See the Packer docs for more use-cases and examples.

\n

We will add a post processing step to the packer template to run the prepare-image.sh script. This script renames the image artifacts and calculates the shasum to prepare them to be uploaded as release artifacts on GitHub.

\n
  post-processor \"shell-local\" {\n    environment_vars = [\"IMAGE_NAME=${var.name}\", \"IMAGE_VERSION=${var.version}\", \"IMAGE_FORMAT=${var.format}\"]\n    script           = \"scripts/prepare-image.sh\"\n  }\n
\n

Launch the build locally

\n

If your local system is setup correctly, it has the packer binary and qemu installed, you can build with just:

\n
packer build .\n
\n

The artifacts folder will contain the resulting machine image and shasum file after the build completes.

\n
artifacts\n└── qemu\n    └── jammy\n        ├── jammy.qcow2\n        └── jammy.qcow2.sha256sum\n
\n

Automate image releases with GitHub Actions.

\n

For the QEMU builder to run at peak performance it requires hardware acceleration. This is not always possible in CI runners. GitHub's hosted runners do not support nested virtualization. With Actuated we added support for launching Virtual Machines in GitHub Action pipelines. This makes it possible to run the Packer QEMU builder in your workflows.

\n

Support for KVM is not enabled by default on Actuated and there are some prerequisites:

\n\n

To configure your Actuated Agent for KVM support follow the instructions in the docs.

\n

The GitHub actions workflow

\n

The default GitHub hosted runners come with Packer pre-installed. On self-hosted runners you will need a step to install the Packer binary. The official [setup-packer][https://github.com/hashicorp/setup-packer] action can be used for this.

\n

We set runs-on to actuated so that the build workflow will run on an Actuated runner:

\n
name: Build\n\non:\n  push:\n    tags: [\"v[0-9].[0-9]+.[0-9]+\"]\n    branches:\n      - \"main\"\njobs:\n  build-image:\n    name: Build\n    runs-on: actuated\n    ##...\n
\n

The build job runs the following steps:

\n
    \n
  1. \n

    Retrieve the Packer configuration by checking out the GitHub repository.

    \n
    - name: Checkout Repository\n  uses: actions/checkout@v3\n
    \n
    2. \n
  2. \n

    Install QEMU to ensure Packer is able to launch kvm/qemu virtual machines.

    \n
    - name: Install qemu\n  run: sudo apt-get update && sudo apt-get install qemu-system -y\n
    \n
    3. \n
  3. \n

    Setup packet to ensure the binary is available in the path.

    \n
    - name: Setup packer\n  uses: hashicorp/setup-packer@main\n
    \n
    4. \n
  4. \n

    Initialize the packer template and install all plugins referenced by the template.

    \n
    - name: Packer Init\n  run: packer init .\n
    \n
    5. \n
  5. \n

    Build the images defined in the root directory. Before we run the packer build command we make /dev/kvm world read-writable so that the QEMU builder can use it.

    \n
    - name: Packer Build\n  run: |\n    sudo chmod o+rw /dev/kvm\n    packer build .\n
    \n
    6. \n
  6. \n

    Upload the images as GitHub release artifacts. This job only runs for tagged commits.

    \n
    - name: Upload images and their SHA to Github Release\n  if: startsWith(github.ref, 'refs/tags/v')\n  uses: alexellis/upload-assets@0.4.0\n  env:\n    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n  with:\n    asset_paths: '[\"./artifacts/qemu/*/*\"]'\n
    \n
    7. \n
\n

Taking it further

\n

We created a GitHub actions workflow that can run a Packer build with QEMU to create a custom Ubuntu image. The resulting qcow2 image is automatically uploaded to the GitHub release assets on each release.

\n

The released image can be downloaded and used to spin up a VM instance on your private hardware or on different cloud providers.

\n

We exported the image in qcow2 format but you might need a different image format. The QEMU builder also supports outputting images in raw format. In our Packer template the output format can be changed by setting the format variable.

\n

Additional tools like the qemu disk image utility can also be used to convert images between different formats. A post-processor would be the ideal place for these kinds of extra processing steps.

\n

AWS also supports importing VM images and converting them to an AMI so they can be used to launch EC2 instances. See: Create an AMI from a VM image

\n

If you'd like to know more about nested virtualisation support, check out: How to run KVM guests in your GitHub Actions

","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/blazing-fast-ci-with-microvms.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/blazing-fast-ci-with-microvms.json deleted file mode 100644 index e3e9a49f..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/blazing-fast-ci-with-microvms.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","contentHtml":"

Around 6-8 months ago I started exploring MicroVMs out of curiosity. Around the same time, I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.

\n

There's three parts to this post:

\n
    \n
  1. A quick debrief on Firecracker and MicroVMs vs legacy solutions
    2. \n
  2. Exploring friction with GitHub Actions from a hosted and self-hosted perspective
    3. \n
  3. Blazing fast CI with Actuated, and additional materials for learning more about Firecracker
    4. \n
\n
\n

We're looking for customers who want to solve the problems explored in this post.\nRegister for the pilot

\n
\n

1) A quick debrief on Firecracker 🔥

\n
\n

Firecracker is an open source virtualization technology that is purpose-built for creating and managing secure, multi-tenant container and function-based services.

\n
\n

I learned about Firecracker mostly by experimentation, building bigger and more useful prototypes. This helped me see what the experience was going to be like for users and the engineers working on a solution. I met others in the community and shared notes with them. Several people asked \"Are microVMs the next thing that will replace containers?\" I don't think they are, but they are an important tool where hard isolation is necessary.

\n

Over time, one thing became obvious:

\n
\n

MicroVMs fill a need that legacy VMs and containers can't.

\n
\n

If you'd like to know more about how Firecracker works and how it compares to traditional VMs and Docker, you can replay my deep dive session with Richard Case, Principal Engineer (previously Weaveworks, now at SUSE).

\n\n
\n

Join Alex and Richard Case for a cracking time. The pair share what's got them so excited about Firecracker, the kinds of use-cases they see for microVMs, fundamentals of Linux Operating Systems and plenty of demos.

\n
\n

2) So what's wrong with GitHub Actions?

\n

First let me say that I think GitHub Actions is a far better experience than Travis ever was, and we have moved all our CI for OpenFaaS, inlets and actuated to Actions for public and private repos. We've built up a good working knowledge in the community and the company.

\n

I'll split this part into two halves.

\n

What's wrong with hosted runners?

\n

Hosted runners are constrained

\n

Hosted runners are incredibly convenient, and for most of us, that's all we'll ever need, especially for public repositories with fast CI builds.

\n

Friction starts when the 7GB of RAM and 2 cores allocated causes issues for us - like when we're launching a KinD cluster, or trying to run E2E tests and need more power. Running out of disk space is also a common problem when using Docker images.

\n

GitHub recently launched new paid plans to get faster runners, however the costs add up, the more you use them.

\n

What if you could pay a flat fee, or bring your own hardware?

\n

They cannot be used with public repos

\n

From GitHub.com:

\n
\n

We recommend that you only use self-hosted runners with private repositories. This is because forks of your public repository can potentially run dangerous code on your self-hosted runner machine by creating a pull request that executes the code in a workflow.

\n
\n
\n

This is not an issue with GitHub-hosted runners because each GitHub-hosted runner is always a clean isolated virtual machine, and it is destroyed at the end of the job execution.

\n
\n
\n

Untrusted workflows running on your self-hosted runner pose significant security risks for your machine and network environment, especially if your machine persists its environment between jobs.

\n
\n

Read more about the risks: Self-hosted runner security

\n

Despite a stern warning from GitHub, at least one notable CNCF project runs self-hosted ARM64 runners on public repositories.

\n

On one hand, I don't blame that team, they have no other option if they want to do open source, it means a public repo, which means risking everything knowingly.

\n

Is there another way we can help them?

\n

I spoke to the GitHub Actions engineering team, who told me that using an ephemeral VM and an immutable OS image would solve the concerns.

\n

There's no access to ARM runners

\n

Building with QEMU is incredibly slow as Frederic Branczyk, Co-founder, Polar Signals found out when his Parca project was taking 33m5s to build.

\n

I forked it and changed a line: runs-on: actuated-aarch64 and reduced the total build time to 1m26s.

\n

This morning @fredbrancz said that his ARM64 build was taking 33 minutes using QEMU in a GitHub Action and a hosted runner.

I ran it on @selfactuated using an ARM64 machine and a microVM.

That took the time down to 1m 26s!! About a 22x speed increase. https://t.co/zwF3j08vEV pic.twitter.com/ps21An7B9B

— Alex Ellis (@alexellisuk) October 20, 2022
\n

They limit maximum concurrency

\n

On the free plan, you can only launch 20 hosted runners at once, this increases as you pay GitHub more money.

\n

Builds on private repos are billed per minute

\n

I think this is a fair arrangement. GitHub donates Azure VMs to open source users or any public repo for that matter, and if you want to build closed-source software, you can do so by renting VMs per hour.

\n

There's a free allowance for free users, then Pro users like myself get a few more build minutes included. However, These are on the standard, 2 Core 7GB RAM machines.

\n

What if you didn't have to pay per minute of build time?

\n

What's wrong with self-hosted runners?

\n

It's challenging to get all the packages right as per a hosted runner

\n

I spent several days running and re-running builds to get all the software required on a self-hosted runner for the private repos for OpenFaaS Pro. Guess what?

\n

I didn't want to touch that machine again afterwards, and even if I built up a list of apt packages, it'd be wrong in a few weeks. I then had a long period of tweaking the odd missing package and generating random container image names to prevent Docker and KinD from conflicting and causing side-effects.

\n

What if we could get an image that had everything we needed and was always up to date, and we didn't have to maintain that?

\n

Self-hosted runners cause weird bugs due to caching

\n

If your job installs software like apt packages, the first run will be different from the second. The system is mutable, rather than immutable and the first problem I faced was things clashing like container names or KinD cluster names.

\n

You get limited to one job per machine at a time

\n

The default setup is for a self-hosted Actions Runner to only run one job at a time to avoid the issues I mentioned above.

\n

What if you could schedule as many builds as made sense for the amount of RAM and core the host has?

\n

Docker isn't isolated at all

\n

If you install Docker, then the runner can take over that machine since Docker runs at root on the host. If you try user-namespaces, many things break in weird and frustrating aways like Kubernetes.

\n

Container images and caches can cause conflicts between builds.

\n

Kubernetes isn't a safe alternative

\n

Adding a single large machine isn't a good option because of the dirty cache, weird stateful errors you can run into, and side-effects left over on the host.

\n

So what do teams do?

\n

They turn to a controller called Actions Runtime Controller (ARC).

\n

ARC is non trivial to set up and requires you to create a GitHub App or PAT (please don't do that), then to provision, monitor, maintain and upgrade a bunch of infrastructure.

\n

This controller starts a number of re-usable (not one-shot) Pods and has them register as a runner for your jobs. Unfortunately, they still need to use Docker or need to run Kubernetes which leads us to two awful options:

\n
    \n
  1. Sharing a Docker Socket (easy to become root on the host)
    2. \n
  2. Running Docker In Docker (requires a privileged container, root on the host)
    3. \n
\n

There is a third option which is to use a non-root container, but that means you can't use sudo in your builds. You've now crippled your CI.

\n

What if you don't need to use Docker build/run, Kaniko or Kubernetes in CI at all? Well ARC may be a good solution for you, until the day you do need to ship a container image.

\n

3) Can we fix it? Yes we can.

\n

Actuated (\"cause (a machine or device) to operate.\") is a semi-managed solution that we're building at OpenFaaS Ltd.

\n

\"A

\n
\n

A semi-managed solution, where you provide hosts and we do the rest.

\n
\n

You provide your own hosts to run jobs, we schedule to them and maintain a VM image with everything you need.

\n

You install our GitHub App, then change runs-on: ubuntu-latest to runs-on: actuated or runs-on: actuated-aarch64 for ARM.

\n

Then, provision one or more VMs with nested virtualisation enabled on GCP, DigitalOcean or Azure, or a bare-metal host, and install our agent. That's it.

\n

If you need ARM support for your project, the a1.metal from AWS is ideal with 16 cores and 32GB RAM, or an Ampere Altra machine like the c3.large.arm64 from Equinix Metal with 80 Cores and 256GB RAM if you really need to push things. The 2020 M1 Mac Mini also works well with Asahi Linux, and can be maxed out at 16GB RAM / 8 Cores. I even tried Frederic's Parca job on my Raspberry Pi and it was 26m30s quicker than a hosted runner!

\n

Whenever a build is triggered by a repo in your organisation, the control plane will schedule a microVM on one of your own servers, then GitHub takes over from there. When the GitHub runner exits, we forcibly delete the VM.

\n

You get:

\n\n

It's capable of running Docker and Kubernetes (KinD, kubeadm, K3s) with full isolation. You'll find some examples in the docs, but anything that works on a hosted runner we expect to work with actuated also.

\n

Here's what it looks like:

\n\n

Want the deeply technical information and comparisons? Check out the FAQ

\n

You may also be interested in a debug experience that we're building for GitHub Actions. It can be used to launch a shell session over SSH with hosted and self-hosted runners: Debug GitHub Actions with SSH and launch a cloud shell

\n

Wrapping up

\n

We're piloting actuated with customers today. If you're interested in faster, more isolated CI without compromising on security, we would like to hear from you.

\n

Register for the pilot

\n

We're looking for customers to participate in our pilot.

\n

Register for the pilot 📝

\n

Actuated is live in pilot and we've already run thousands of VMs for our customers, but we're only just getting started here.

\n

\"VM

\n
\n

Pictured: VM launch events over the past several days

\n
\n

Other links:

\n\n

What about GitLab?

\n

We're focusing on GitHub Actions users for the pilot, but have a prototype for GitLab. If you'd like to know more, reach out using the Apply for the pilot form.

\n

Just want to play with Firecracker or learn more about microVMs vs legacy VMs and containers?

\n\n

What are people saying about actuated?

\n
\n

\"We've been piloting Actuated recently. It only took 30s create 5x isolated VMs, run the jobs and tear them down again inside our on-prem environment (no Docker socket mounting shenanigans)! Pretty impressive stuff.\"

\n

Addison van den Hoeven - DevOps Lead, Riskfuel

\n
\n
\n

\"Actuated looks super cool, interested to see where you take it!\"

\n

Guillermo Rauch, CEO Vercel

\n
\n
\n

\"This is great, perfect for jobs that take forever on normal GitHub runners. I love what Alex is doing here.\"

\n

Richard Case, Principal Engineer, SUSE

\n
\n
\n

\"Thank you. I think actuated is amazing.\"

\n

Alan Sill, NSF Cloud and Autonomic Computing (CAC) Industry-University Cooperative Research Center

\n
\n
\n

\"Nice work, security aspects alone with shared/stale envs on self-hosted runners.\"

\n

Matt Johnson, Palo Alto Networks

\n
\n
\n

\"Is there a way to pay github for runners that suck less?\"

\n

Darren Shepherd, Acorn Labs

\n
\n
\n

\"Excited to try out actuated! We use custom actions runners and I think there's something here 🔥\"

\n

Nick Gerace, System Initiative

\n
\n
\n

It is awesome to see the work of Alex Ellis with Firecracker VMs. They are provisioning and running Github Actions in isolated VMs in seconds (vs minutes).\"

\n

Rinat Abdullin, ML & Innovation at Trustbit

\n
\n
\n

This is awesome!\" (After reducing Parca build time from 33.5 minutes to 1 minute 26s)

\n

Frederic Branczyk, Co-founder, Polar Signals

\n
","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/caching-in-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/caching-in-github-actions.json deleted file mode 100644 index ab485719..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/caching-in-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","contentHtml":"

GitHub provides a cache action that allows caching dependencies and build outputs to improve workflow execution time.

\n

A common use case would be to cache packages and dependencies from tools such as npm, pip, Gradle, ... . If you are using Go, caching go modules and the build cache can save you a significant amount of build time as we will see in the next section.

\n

Caching can be configured manually, but a lot of setup actions already use the actions/cache under the hood and provide a configuration option to enable caching.

\n

We use the actions cache to speed up workflows for building the Actuated base images. As part of those workflows we build a kernel and then a rootfs. Since the kernel’s configuration is changed infrequently it makes sense to cache that output.

\n

\"Build

\n
\n

Comparing workflow execution times with and without caching.

\n
\n

Building the kernel takes around 1m20s on our aarch-64 Actuated runner and 4m10s for the x86-64 build so we get some significant time improvements by caching the kernel.

\n

The output of the cache action can also be used to do something based on whether there was a cache hit or miss. We use this to skip the kernel publishing step when there was a cache hit.

\n
- if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}\n  name: Publish Kernel\n  run: make publish-kernel-x86-64\n
\n

Caching Go dependency files and build outputs

\n

In this minimal example we are going to setup caching for Go dependency files and build outputs. As an example we will be building alexellis/registry-creds. This is a Kubernetes operator that can be used to replicate Kubernetes ImagePullSecrets to all namespaces.

\n

It has the K8s API as a dependency which is quite large so we expect to save some time by cashing the Go mod download. By also caching the Go build cache it should be possible to speed up the workflow even more.

\n

Configure caching manually

\n

We will first create the workflow and run it without any caching.

\n
name: ci\n\non: push\n\njobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: \"alexellis/registry-creds\"\n      - name: Setup Golang\n        uses: actions/setup-go@v3\n        with:\n          go-version: ~1.19\n      - name: Build\n        run: |\n          CGO_ENABLED=0 GO111MODULE=on \\\n          go build -ldflags \"-s -w -X main.Release=dev -X main.SHA=dev\" -o controller\n
\n

The checkout action is used to check out the registry-creds repo so the workflow can access it. The next step sets up Go using the setup-go action and as a last step we run go build.

\n

\"No

\n

When triggering this workflow we see that each run takes around 1m20s.

\n

Modify the workflow and add an additional step to configure the caches using the cache action:

\n
steps:\n  - name: Setup Golang\n    uses: actions/setup-go@v3\n    with:\n      go-version: ~1.19\n  - name: Setup Golang caches\n    uses: actions/cache@v3\n    with:\n      path: |\n        ~/.cache/go-build\n        ~/go/pkg/mod\n      key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}\n      restore-keys: |\n        ${{ runner.os }}-golang-\n
\n

The path parameter is used to set the paths on the runner to cache or restore. The key parameter sets the key used when saving the cache. A hash of the go.sum file is used as part of the cache key.

\n

Optionally the restore-keys are used to find and restore a cache if there was no hit for the key. In this case we always restore the cache even if there was no specific hit for the go.sum file.

\n

The first time this workflow is run the cache is not populated so we see a similar execution time as without any cache of around 1m20s.

\n

\"Comparing

\n

Running the workflow again we can see that it now completes in just 18s.

\n

Use setup-go built-in caching

\n

The V3 edition of the setup-go action has support for caching built-in. Under the hood it also uses the actions/cache with a similar configuration as in the example above.

\n

The advantage of using the built-in functionality is that it requires less configuration settings. Caching can be enabled by adding a single line to the workflow configuration:

\n
name: ci\n\non: push\n\njobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: \"alexellis/registry-creds\"\n      - name: Setup Golang\n        uses: actions/setup-go@v3\n        with:\n          go-version: ~1.19\n+         cache: true\n      - name: Build\n        run: |\n          CGO_ENABLED=0 GO111MODULE=on \\\n          go build -ldflags \"-s -w -X main.Release=dev -X main.SHA=dev\" -o controller\n
\n

Triggering the workflow with the build-in cache yields similar time gains as with the manual cache configuration.

\n

Conclusion

\n

We walked you through a short example to show you how to set up caching for a Go project and managed to build the project 4x faster.

\n

If you are building with Docker you can use Docker layer caching to make your builds faster. Buildkit automatically caches the build results and allows exporting the cache to an external location. It has support for uploading the build cache to GitHub Actions cache

\n

See also: GitHub: Caching dependencies in Workflows

\n

Keep in mind that there are some limitations to the GitHub Actions cache. Cache entries that have not been accessed in over 7 days will be removed. There is also a limit on the total cache size of 10 GB per repository.

\n

Some points to take away:

\n\n
\n

Want to learn more about Go and GitHub Actions?

\n

Alex's eBook Everyday Golang has a chapter dedicated to building Go programs with Docker and GitHub Actions.

\n
","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/calyptia-case-study-arm.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/calyptia-case-study-arm.json deleted file mode 100644 index c216413f..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/calyptia-case-study-arm.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","contentHtml":"

This is a case-study, and guest article by Patrick Stephens, Tech Lead of Infrastructure at Calyptia.

\n

Introduction

\n

Different architecture builds can be slow using the Github Actions hosted runners due to emulation of the non-native architecture for the build. This blog shows a simple way to make use of self-hosted runners for dedicated builds but in a secure and easy to maintain fashion.

\n

Calyptia maintains the OSS and Cloud Native Computing Foundation (CNCF) graduated Fluent projects including Fluent Bit. We then add value to the open-source core by providing commercial services and enterprise-level features.

\n
\n

Fluent Bit is a Fast and Lightweight Telemetry Agent for Logs, Metrics, and Traces for Linux, macOS, Windows, and BSD family operating systems. It has been made with a strong focus on performance to allow the collection and processing of telemetry data from different sources without complexity.

\n

It was originally created by Eduardo Silva and is now an independent project.

\n
\n

To learn about Fluent Bit, the Open Source telemetry agent that Calyptia maintains, check out their docs.

\n

The Problem

\n

One of the best things about Fluent Bit is that we provide native packages (RPMs and DEBs) for a myriad of supported targets (various Linux, macOS and Windows), however to do this is also one of the hardest things to support due to the complexity of building and testing across all these targets.

\n

When PRs are provided we would like to ensure they function across the targets but doing so can take a very long time (hours) and consume a lot of resources (that must be paid for). This means that these long running jobs are only done via exception (manually labelling a PR or on full builds for releases) leading to issues only discovered when a full build & test is done, e.g. during the release process so blocking the release until it is fixed.

\n

The long build time problem came to a head when we discovered we could no longer build for Amazon Linux 2023 (AL2023) because the build time exceeded the 6 hour limit for a single job on Github. We had to disable the AL2023 target for releases which means users cannot then update to the latest release leading to missing features or security problems: See the issue here

\n

In addition to challenges in the OSS, there are also challenges on the commercial side. Here, we are seeing issues with extended build times for ARM64 targets because our CI is based on Github Actions and currently only AMD64 (also called x86-64 or x64) runners are provided for builds. This slows down development and can mean bugs are not caught as early as possible.

\n

Why not use self-hosted runners?

\n

One way to speed up builds is to provide self-hosted ARM64 runners.

\n

Unfortunately, runners pose security implications, particularly for public repositories. In fact, Github recommends against using self-hosted runners: About self-hosted runners - GitHub Docs

\n

In addition to security concerns, there are also infrastructure implications for using self-hosted runners. We have to provide the infrastructure around deploying and managing the self-hosted runners, installing an agent, configuring it for jobs, etc. From a perspective of OSS we want anything we do to be simple and easy for maintenance purposes.

\n

Any change we make needs to be compatible with downstream forks as well. We do not want to break builds for existing users, particularly for those who are contributors as well to the open source project. Therefore we need a solution that does not impact them.

\n

There are various tools that can help with managing self-hosted runners, https://jonico.github.io/awesome-runners/ provides a good curated list. I performed an evaluation of some of the recommended tools but the solution would be non-trivial and require some effort to maintain.

\n

Our considerations

\n

We have the following high level goals in a rough priority order:

\n\n

The solution

\n

At Kubecon EU 2023 I met up with Alex Ellis from Actuated (and of OpenFaaS fame) in-person and we wanted to put Alex and his technology to the test, to see if the Actuated technology could fix the problems we see with our build process.

\n

To understand what Actuated is then it is best to refer to their documentation with this specific blog post being a good overview of why we considered adopting it. We're not the only CNCF project that Alex's team was able to help. He describes how he helped Parca and Network Service Mesh to slash their build teams by using native Arm hardware.

\n

A quick TLDR; though would be that Actuated provides an agent you install which then automatically creates ephemeral VMs on the host for each build job. Actuated seemed to tick the various boxes (see the considerations above) we had for it but never trust a vendor until you’ve tried it yourself!

\n

Quote from Alex:

\n
\n

\"Actuated aims to give teams the closest possible experience to managed runners, but with native arm support flat rate billing, and secure VM-level isolation. Since Calyptia adopted actuated, we’ve also shipped an SSH debug experience (like you’d find with CircleCI) and detailed reports and insights on usage across repos, users and organisations.\"

\n
\n

To use Actuated, you have to provision a machine with the Actuated agent, which is trivial and well documented: https://docs.actuated.dev/install-agent/.

\n

We deployed an Ampere Altra Q80 server with 256GB of RAM and 80 cores ARM64 machine via Equinix (Equinix donates resources to the CNCF which we use for Fluent Bit so this satisfies the cost side of things) and installed the Actuated agent on it per the Actuated docs.

\n

The update required to start using Actuated in OSS Fluent Bit is a one-liner. (Thanks in part to my excellent work refactoring the CI workflows, or so I like to think. You can see the actual PR here for the change: https://github.com/fluent/fluent-bit/pull/7527.)

\n

The following is the code required to start using Actuated:

\n
-    runs-on: ubuntu-latest\n+    runs-on: ${{ (contains(matrix.distro, 'arm' ) & 'actuated-arm64') || 'ubuntu-latest' }}\n
\n

For most people, the change will be much simpler:

\n
-    runs-on: ubuntu-latest\n+    runs-on: actuated\n
\n

In Github Actions parlance, the code above translates to “if we are doing an ARM build, then use the Actuated runner; otherwise, use the default Github Hosted (AMD64) Ubuntu runner”.

\n

In the real code, I added an extra check so that we only use Actuated runners for the official source repo which means any forks will also carry on running as before on the Github Hosted runner.

\n

With this very simple change, all the ARM64 builds that used to take hours to complete now finish in minutes. In addition, we can actually build the AL2023 ARM64 target to satisfy those users too. A simple change gave us a massive boost to performance and also provided a missing target.

\n

To demonstrate this is not specific to Equinix hosts or in some fashion difficult to manage in heterogeneous infrastructure (e.g. various hosts/VMs from different providers), we also replicated this for all our commercial offerings using a bare-metal Hetzner host. The process was identical: install the agent and make the runs-on code change as above to use Actuated. Massive improvements in build time were seen again as expected.

\n

The usage of bare-metal (or cloud) hosts providers is invisible and only a choice of which provider you want to put the agent on. In our case we have a mixed set up with no difference in usage or maintenance.

\n

Challenges building containers

\n

The native package (RPM/DEB) building described above was quite simple to integrate via the existing workflows we had.

\n

Building the native packages is done via a process that runs a target-specific container for each of the builds, e.g. we run a CentOS container to build for that target. This allows a complete build to be run on any Linux-compatible machine with a container runtime either in CI or locally. For ARM builds, we were using QEMU emulation for ARM builds hence the slowdown as this has to emulate instructions between architectures.

\n

Container builds are the primary commercial area for improvement as we provide a SAAS solution running on K8S. Container builds were also a trickier proposition for OSS as we were using a single job to build all architectures using the docker/build-push-action. The builds were incredibly slow for ARM and also atomic, which means if you received a transient issue in one of the architecture builds, you would have to repeat the whole lot.

\n

As an example: https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-images.yaml

\n
      - name: Build the production images\n        id: build_push\n        uses: docker/build-push-action@v4\n        with:\n          file: ./dockerfiles/Dockerfile\n          context: .\n          tags: ${{ steps.meta.outputs.tags }}\n          labels: ${{ steps.meta.outputs.labels }}\n          platforms: linux/amd64, linux/arm64, linux/arm/v7\n          target: production\n          # Must be disabled to provide legacy format images from the registry\n          provenance: false\n          push: true\n          load: false\n          build-args: |\n            FLB_NIGHTLY_BUILD=${{ inputs.unstable }}\n            RELEASE_VERSION=${{ inputs.version }}\n
\n

The build step above is a bit more complex to tease out into separate components: we need to run single architecture builds for each target then provide a multi-arch manifest that links them together.

\n

We reached out to Alex on a good way to modify this to work within a split build per architecture approach. The Actuated team has been very responsive on these types of questions along with proactive monitoring of our build queue and runners.

\n

Within Calyptia we have followed the approach Docker provided here and suggested by the Actuated team: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners

\n

Based on what we learned, we recommend the following process is followed:

\n

Build each architecture and push by digest in a set of parallel matrix jobs.\nCapture the output digest of each build.\nCreate the multi-arch manifest made up of each digest we have pushed in step 1 using the artefact from step 2.

\n

This approach provides two key benefits. First, it allows us to run on dedicated runners per-arch. Second, if a job fails we only need to repeat the single job, instead of having to rebuild all architectures.

\n

The new approach reduced the time for the release process for the Calyptia Core K8S Operator from more than an hour to minutes. Additionally, because we can do this so quickly, we now build all architectures for every change rather than just on release. This helps developers who are running ARM locally for development as they have containers always available.

\n

The example time speed up for the Calyptia Core K8S operator process was replicated across all the other components. A very good bang for your buck!

\n

For us, the actuated subscription fee has been of great value. Initially we tested the waters on the Basic Plan, but soon upgraded when we saw more areas where we could use it. The cost for us has been offset against a massive improvement in CI time and development time plus reducing the infrastructure costs of managing the self-hosted runners.

\n

Lessons learned

\n

The package updates were seamless really, however we did encounter some issues with the ecosystem (not with actuated), when refactoring and updating our container builds. The issues with the container builds are covered below to help anyone else with the same problems.

\n

Provenance is now enabled by default

\n

We were using v3 of Docker’s docker/build-push-action, but they made a breaking change which caused us a headache. They changed the default in v4 to create the various extra artifacts for provenance (e.g. SBOMs) which did have a few extra side effects both at the time and even now.

\n

If you do not disable this then it will push manifest lists rather than images so you will subsequently get an error message when you try to create a manifest list of another manifest list.

\n

Separately this also causes issues for older docker clients or organisations that need the legacy Docker schema format from a registry: using it means only OCI format schemas are pushed. This impacted both OSS and our commercial offerings: https://github.com/fluent/fluent-bit/issues/7748.

\n

It meant people on older OS’s or with requirements on only consuming Docker schema (e.g. maybe an internal mirror registry only supports that) could not pull the images.

\n

Invalid timestamps for gcr.io with manifests

\n

A funny problem found with our cloud-run deployments for Calyptia Core SAAS offering was that pushing the manifests to (Google Container Registry) gcr.io meant they ended up with a zero-epoch timestamp. This messed up some internal automation for us when we tried to get the latest version.

\n

To resolve this we just switched back to doing a single architecture build as we do not need multi-arch manifests for cloud-run. Internally we still have multi-arch images in ghcr.io for internal use anyway, this is purely the promotion to gcr.io.

\n

Manifests cannot use sub-paths

\n

This was a fun one: when specifying images to make up your manifest they must be in the same registry of course!

\n

Now, we tend to use sub-paths a lot to handle specific use cases for ghcr.io but unfortunately you cannot use them when trying to construct a manifest.

\n

OK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/product:tag-amd64\nNOK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/amd64/product:tag

\n

As with all good failures, the tooling let me make a broken manifest at build time but unfortunately trying to pull it meant a failure at runtime.

\n

Actuated container registry mirror

\n

All Github hosted runners provide default credentials to authenticate with docker.io for pulling public images. When running on a self-hosted runner you need to authenticate for this otherwise you will hit rate limits and builds may fail as they cannot download required base images.

\n

Actuated provide a registry mirror and Github Action to simplify this so make sure you set it up: https://docs.actuated.dev/tasks/registry-mirror/

\n

As part of this, ensure it is set up for anything that uses images (e.g. we run integration tests on KIND that failed as the cluster could not download its images) and that it is done after any buildx config as it creates a dedicated buildx builder for the mirror usage.

\n

Actuated support

\n

The Actuated team helped us in two ways: the first was that we were able to enable Arm builds for our OSS projects and our commercial products, when they timed out with hosted runners. The second way was where our costs were getting out of hand on GitHub’s larger hosted runners: Actuated not only reduced the build time, but the billing model is flat-rate, meaning our costs are now fixed, rather than growing.

\n

As we made suggestions or collaborated with the Actuated team, they updated the documentation, including our suggestions on smoothing out the onboarding of new build servers and new features for the CLI.

\n

The more improvements we’ve made, the more we’ve seen. Next on our list is getting the runtime of a Go release down from 26 minutes by bringing it over to actuated.

\n

Conclusion

\n

Alex Ellis: We've learned a lot working with Patrick and Calyptia and are pleased to see that they were able to save money, whilst getting much quicker, and safer Open Source and commercial builds.

\n

We value getting feedback and suggestions from customers, and Patrick continues to provide plenty of them.

\n

If you'd like to learn more about actuated, reach out to speak to our team by clicking \"Sign-up\" and filling out the form. We'll be in touch to arrange a call.

","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/case-study-bring-your-own-bare-metal-to-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/case-study-bring-your-own-bare-metal-to-actions.json deleted file mode 100644 index cf3c3c43..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/case-study-bring-your-own-bare-metal-to-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","contentHtml":"

I'm going to show you how both a regular x86_64 build and an Arm build were made dramatically faster by using Bring Your Own (BYO) bare-metal servers.

\n

At the early stage of a project, GitHub's standard runners with 2x cores, 8GB RAM, and a little free disk space are perfect because they're free for public repos. For private repos they come in at a modest cost, if you keep your usage low.

\n

What's not to love?

\n

Well, Ed Warnicke, Distinguished Engineer at Cisco contacted me a few weeks ago and told me about the VPP project, and some of the problems he was running into trying to build it with hosted runners.

\n
\n

The Fast Data Project (FD.io) is an open-source project aimed at providing the world's fastest and most secure networking data plane through Vector Packet Processing (VPP).

\n
\n

Whilst VPP can be used as a stand-alone project, it is also a key component in the Cloud Computing Foundation's (CNCF's) Open Source Network Service Mesh project.

\n

There were two issues:

\n
    \n
  1. \n

    The x86_64 build was taking 1 hour 25 minutes on a standard runner.

    \n

    Why is that a problem? CI is meant to both validate against regression, but to build binaries for releases. If that process can take 50 minutes before failing, it's incredibly frustrating. For an open source project, it's actively hostile to contributors.

    \n
    2. \n
  2. \n

    The Arm build was hitting the 6 hour limit for GitHub Actions then failing

    \n

    Why? Well it was using QEMU, and I've spoken about this in the past - QEMU is a brilliant, zero cost way to build Arm binaries on a regular machine, but it's slow. And you'll see just how slow in the examples below, including where my Raspberry Pi beat a GitHub runner.

    \n
    3. \n
\n

We explain how to use QEMU in Docker Actions in the following blog post:

\n

The efficient way to publish multi-arch containers from GitHub Actions

\n

Rubbing some bare-metal on it

\n

So GitHub does actually have a beta going for \"larger runners\", and if Ed wanted to try that out, he'd have to apply to a beta waitlist, upgrade to a Team or Enterprise Plan, and then pick a new runner size.

\n

But that wouldn't have covered him for the Arm build, GitHub don't have any support there right now. I'm sure it will come one, day, but here we are unable to release binaries for our Arm users.

\n

With actuated, we have no interest in competing with GitHub's business model of selling compute on demand. We want to do something more unique than that - we want to enable you to bring your own (BYO) devices and then use them as runners, with VM-level isolation and one-shot runners.

\n
\n

What does Bring Your Own (BYO) mean?

\n

\"Your Own\" does not have to mean physical ownership. You do not need to own a datacenter, or to send off a dozen Mac Minis to a Colo.\nYou can provision bare-metal servers on AWS or with Equinix Metal as quickly as you can get an EC2 instance.\nActually, bare-metal isn't strictly needed at all, and even DigitalOcean's and Azure's VMs will work with actuated because they support KVM, which we use to launch Firecracker.

\n
\n

And who is behind actuated? We are a nimble team, but have a pedigree with Cloud Native and self-hosted software going back 6-7 years from OpenFaaS. OpenFaaS is a well known serverless platform which is used widely in production by commercial companies including Fortune 500s.

\n

Actuated uses a Bring Your Own (BYO) server model, but there's very little for you to do once you've installed the actuated agent.

\n

Here's how to set up the agent software: Actuated Docs: Install the Agent.

\n

You then get detailed stats about each runner, the build queue and insights across your whole GitHub organisation, in one place:

\n

Actuated now aggregates usage data at the organisation level, so you can get insights and spot changes in behaviour.

This peak of 57 jobs was when I was quashing CVEs for @openfaas Pro customers in Alpine Linux and a bunch of Go https://t.co/a84wLNYYjo… https://t.co/URaxgMoQGW pic.twitter.com/IuPQUjyiAY

— Alex Ellis (@alexellisuk) March 7, 2023
\n

First up - x86_64

\n

I forked Ed's repo into the \"actuated-samples\" repo, and edited the \"runs-on:\" field from \"ubuntu-latest\" to \"actuated\".

\n

The build which previously took 1 hour 25 minutes now took 18 minutes 58 seconds. That's a 4.4x improvement.

\n

\"Improvements

\n

4.4x doesn't sound like a big number, but look at the actual number.

\n

It used to take well over an hour to get feedback, now you get it in less than 20 minutes.

\n

And for context, this x86_64 build took 17 minutes to build on Ed's laptop, with some existing caches in place.

\n

I used an Equinix Metal m3.small.x86 server, which has 8x Intel Xeon E-2378G cores @ 2.8 GHz. It also comes with a local SSD, local NVMe would have been faster here.

\n

The Firecracker VM that was launched had 12GB of RAM and 8x vCPUs allocated.

\n

Next up - Arm

\n

For the Arm build I created a new branch and had to change a few hard-coded references from \"_amd64.deb\" to \"_arm64.deb\" and then I was able to run the build. This is common enablement work. I've been doing Arm enablement for Cloud Native and OSS since 2015, so I'm very used to spotting this kind of thing.

\n

So the build took 6 hours, and didn't even complete when running with QEMU.

\n

How long did it take on bare-metal? 14 minutes 28 seconds.

\n

\"Improvements

\n

That's a 25x improvement.

\n

The Firecracker VM that we launched had 16GB of RAM and 8x vCPUs allocated.

\n

It was running on a Mac Mini M1 configured with 16GB RAM, running with Asahi Linux. I bought it for development and testing, as a one-off cost, and it's a very fast machine.

\n

But, this case-study is not specifically about using consumer hardware, or hardware plugged in under your desk.

\n

Equinix Metal and Hetzner both have the Ampere Altra bare-metal server available on either an hourly or monthly basis, and AWS customers can get access to the a1.metal instance on an hourly basis too.

\n
\n

To prove the point, that BYO means cloud servers, just as much as physically owned machines, I also ran the same build on an Ampere Altra from Equinix Metal with 20 GB of RAM, and 32 vCPUs, it completed in 9 minutes 39 seconds.

\n
\n

See our hosting recommendations: Actuated Docs: Provision a Server

\n

In October last year, I benchmarked a Raspberry Pi 4 as an actuated server and pitted it directly against QEMU and GitHub's Hosted runners.

\n

It was 24 minutes faster. That's how bad using QEMU can be instead of using bare-metal Arm.

\n

Then, just for run I scheduled the MicroVM on my @Raspberry_Pi instead of an @equinixmetal machine.

Poor little thing has 8GB RAM and 4 Cores with an SSD connected over USB-C.

Anyway, it still beat QEMU by 24 minutes! pic.twitter.com/ITyRpbnwEE

— Alex Ellis (@alexellisuk) October 20, 2022
\n

Wrapping up

\n

So, wrapping up - if you only build x86_64, and have very few build minutes, and are willing to upgrade to a Team or Enterprise Plan on GitHub, \"faster runners\" may be an option you want to consider.

\n

If you don't want to worry about how many minutes you're going to use, or surprise bills because your team got more productive, or grew in size, or is finally running those 2 hour E2E tests every night, then actuated may be faster and better value overall for you.

\n

But if you need Arm runners, and want to use them with public repos, then there are not many options for you which are going to be secure and easy to manage.

\n

A recap on the results

\n

\"The

\n
\n

The improvement on the x86 build

\n
\n

You can see the builds here:

\n

x86_64 - 4.4x improvement

\n\n

Arm - 25x improvement

\n\n

Want to work with us?

\n

Want to get in touch with us and try out actuated for your team?

\n

We're looking for pilot customers who want to speed up their builds, or make self-hosted runners simpler to manager, and ultimately, about as secure as they're going to get with MicroVM isolation.

\n

Set up a 30 min call with me to ask any questions you may have and find out next steps.

\n\n

Learn more about how it compares to other solutions in the FAQ: Actuated FAQ

\n

See also:

\n","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/cncf-arm-march-update.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/cncf-arm-march-update.json deleted file mode 100644 index aef7046d..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/cncf-arm-march-update.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","contentHtml":"

It's now been 4 months since we kicked off the sponsored program with the Cloud Native Computing Foundation (CNCF) and Ampere to manage CI for the foundation's open source projects. But even before that, Calyptia, the maintainer of Fluent approached us to run Arm CI for the open source fluent repos, so we've been running CI for CNCF projects since June 2023.

\n

Over that time, we've got to work directly with some really bright, friendly, and helpful maintainers, who wanted to have a safe, fast and secure way to create release artifacts, test PRs, and to run end to end tests. Their alternative until this point was either to go against GitHub's own advice, and to run an unsafe, self-hosted runner on an open source repo, or to use QEMU that in the case of Fluent meant their 5 minute build took over 6 hours before failing.

\n

You can find out more about why we put this program together in the original announcement: Announcing managed Arm CI for CNCF projects

\n

Measuring the impact

\n

When we started out, Chris Aniszczyk, the CNCF's CTO wanted to create a small pilot to see if there'd be enough demand for our service. The CNCF partnered with Ampere to co-fund the program, Ampere sell a number of Arm based CPUs - which they brand as \"Cloud Native\" because they're so dense in cores and highly power efficient. Equinix Metal provide the credits and the hosting via the Cloud Native Credits program.

\n

In a few weeks, not only did we fill up all available slots, but we personally hand-held and onboarded each of the project maintainers one by one, over Zoom, via GitHub, and Slack.

\n

Why would maintainers of top-tier projects need our help? Our team and community has extensive experience porting code to Arm, and building for multiple CPUs. We were able to advise on best practices for splitting up builds, how to right-size VMs, were there to turn on esoteric Kernel modules and configurations, and to generally give them a running start.

\n

Today, our records show that the CNCF projects enrolled have run almost 400k minutes. That's almost the equivalent of a computer running tasks 24/7 for a total of 2 months solid, without a break.

\n

Here's a list of the organisations we've onboarded so far, ordered by the total amount of build minutes. We added the date of their first actuated build to help add some context. As I mentioned in the introduction, fluent have been a paying customer since June 2023.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationDate of first actuated build
1etcd-io (etcd, boltdb)2023-10-24
2fluent2023-06-07
3falcosecurity2023-12-06
4containerd2023-12-02
5cilium (tetragon, cilium, ebpf-go)2023-10-31
6cri-o2023-11-27
7open-telemetry2024-02-14
8opencontainers (runc)2023-12-15
9argoproj2024-01-30
\n

Ranked by build minutes consumed

\n

Some organisations have been actuated on multiple projects like etcd-io, with boltdb adding to their minutes, and cilium where tetragon and ebpf-go are also now running Arm builds.

\n

It's tempting to look at build minutes as the only metric, however, now that containerd, runc, cilium, etcd, and various other core projects are built by actuated, the security of the supply chain has become far more certain.

\n

From 10,000ft

\n

Here's what we aimed for and have managed to achieve in a very short period of time:

\n\n

Making efficient use of shared resources

\n

After fluent, etcd was the second project to migrate off self-managed runners. They had the impression that one of their jobs needed 32 vCPU and 32GB of RAM, and when we monitored the shared server pool, we noticed barely any load on the servers. That led me to build a quick Linux profiling tool called vmmeter. When they ran the profiler, it turned out the job used a maximum of 1.3 vCPU and 3GB of RAM, that's not just a rounding error - that's a night and day difference.

\n

You can learn how to try out vmmeter to right-size your jobs on actuated, or on GitHub's hosted runners.

\n

Right sizing VMs for GitHub Actions

\n

The projects have had a fairly stable, steady-state of CI jobs throughout the day and night as contributors from around the globe send PRs and end to end tests run.

\n

But with etcd-io in particular we started to notice on Monday or Tuesday that there was a surge of up to 200 jobs all at once. When we asked them about this, they told us Dependabot was the cause. It would send a number of PRs to bump dependencies and that would in turn trigger dozens of jobs.

\n

\"Thundering

\n
\n

Thundering herd problem from dependabot

\n
\n

It would clear itself down in time, but we spent a little time to automate adding in 1-2 extra servers for this period of the week, and we managed to get the queue cleared several times quicker. When the machines are no longer needed, they drain themselves and get deleted. This is important for efficient use of the CNCF's credits and Equinix Metal's fleet of Ampere Altra Q80 Arm servers.

\n

Giving insights to maintainers

\n

I got to meet up with Phil Estes from the containerd project at FOSDEM. We are old friends and used to be Docker Captains together.

\n

We looked at the daily usage stats, looked at the total amount of contributors that month and how many builds they'd had.

\n

\"Phil

\n

Then we opened up the organisation insights page and found that containerd had accounted for 14% of the total build minutes having only been onboarded in Dec 2023.

\n

\"Detailed

\n

We saw that there was a huge peak in jobs last month compared to this month, so he went off to the containerd Slack to ask about what had happened.

\n

Catching build time increases early

\n

Phil also showed me that he used to have a jimmy-rigged dashboard of his own to track build time increases, and at FOSDEM, my team did a mini hackathon to release our own way to show people their job time increases.

\n

We call it \"Job Outliers\" and it can be used to track increases going back as far as 120 days from today.

\n

\"Job

\n

Clicking \"inspect\" on any of the workflows will open up a separate plot link with deep links to the longest job seen on each day of that period of time.

\n

So what changed for our own actuated VM builds in that week, to add 5+ minutes of build time?

\n

\"Maximum

\n

We started building eBPF into the Kernel image, and the impact was 2x 2.5 minutes of build time.

\n

This feature was originally requested by Toolpath, a commercial user of actuated with very intensive Julia builds, and they have been using it to keep their build times in check. We're pleased to be able to offer every enhancement to the CNCF project maintainers too.

\n

Wrapping up

\n

What are the project maintainers saying?

\n

Antoine Toulme, maintainer of OpenTelemetry collectors:

\n
\n

The OpenTelemetry project has been looking for ways to test arm64 to support it as a top tier distribution. Actuated offers a path for us to test on new operating systems, especially arm64, without having to spend any time setting up or maintaining runners. We were lucky to be the recipient of a loan from Ampere that gave us access to a dedicated ARM server, and it took us months to navigate setting up dedicated runners and has significant maintenance overhead. With Actuated, we just set a tag in our actions and everything else is taken care of.

\n
\n

Luca Guerra, maintainer of Falco:

\n
\n

Falco users need to deploy to ARM64 as a platform, and we as maintainers, need to make sure that this architecture is treated as a first class citizen. Falco is a complex piece of software that employs kernel instrumentation and so it is not trivial to properly test. Thanks to Actuated, we were able to quickly add ARM64 to our GitHub Actions CI/CD pipeline making it much easier to maintain, freeing up engineering time from infrastructure work.

\n
\n

Sascha Grunert, maintainer of Cri-o:

\n
\n

The CRI-O project was able to seamlessly integrate Arm based CI with the support of Actuated. We basically had to convert our existing tests to a GitHub Actions matrix utilizing their powerful Arm runners. Integration and unit testing on Arm is another big step for CRI-O to provide a generally broader platform support. We also had to improve the test suite itself for better compatibility with other architectures than x86_64/arm64. This makes contributing to CRI-O on those platforms even simpler. I personally don’t see any better option than Actuated right now, because managing our own hardware is something we’d like to avoid to mainly focus on open source software development. The simplicity of the integration using Actuated helped us a lot, and our future goal is to extend the CRI-O test scenarios for that.

\n
\n

Summing up the program so far

\n

Through the sponsored program, actuated has now almost 400k build minutes for around 10 CNCF projects, and we've heard from a growing number of projects who would like access.

\n

We've secured the supply chain by removing unsafe runners that GitHub says should definitely not be used for open source repositories, and we've lessened the burden of server management on already busy maintainers.

\n

Whilst the original pilot program is now full, we have the capacity to onboard many other projects and would love to work with you. We are happy to offer a discounted subscription if your employer that sponsors your time on the said CNCF project will pay for it. Otherwise, contact us anyway, and we'll put you into email contact with Chris Aniszczyk so you can let him know how this would help you.

","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/custom-sizes-bpf-kvm.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/custom-sizes-bpf-kvm.json deleted file mode 100644 index 82c95924..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/custom-sizes-bpf-kvm.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","contentHtml":"

In this December update, we've got three new updates to the platform that we think you'll benefit from. From requesting custom vCPU and RAM per job, to eBPF features, to spreading your plan across multiple machines dynamically, it's all new and makes actuated better value.

\n

New eBPF support and 5.10.201 Kernel

\n

And as part of our work to provide hosted Arm CI for CNCF projects, including Tetragon and Cilium, we've now enabled eBPF and BTF features within the Kernel.

\n
\n

Berkley Packet Filter (BPF) is an advanced way to integrate with the Kernel, for observability, security and networking. You'll see it included in various CNCF projects like Cilium, Falco, Kepler, and others.

\n
\n

Whilst BPF is powerful, it's also a very fast moving space, and was particularly complicated to patch to Firecracker's minimal Kernel configuration. We want to say a thank you to Mahé Tardy\n who maintains Tetragon and to Duffie Coolie both from Isovalent for pointers and collaboration.

\n

We've made a big jump in the supported Kernel version from 5.10.77 up to 5.10.201, with newer revisions being made available on a continual basis.

\n

To update your servers, log in via SSH and edit /etc/default/actuated.

\n

For amd64:

\n
AGENT_IMAGE_REF=\"ghcr.io/openfaasltd/actuated-ubuntu22.04:x86_64-latest\"\nAGENT_KERNEL_REF=\"ghcr.io/openfaasltd/actuated-kernel:x86_64-latest\"\n
\n

For arm64:

\n
AGENT_IMAGE_REF=\"ghcr.io/openfaasltd/actuated-ubuntu22.04:aarch64-latest\"\nAGENT_KERNEL_REF=\"ghcr.io/openfaasltd/actuated-kernel:aarch64-latest\"\n
\n

Once you have the new images in place, reboot the server. Updates to the Kernel and root filesystem will be delivered Over The Air (OTA) automatically by our team.

\n

Request custom vCPU and RAM per job

\n

Our initial version of actuated aimed to set a specific vCPU and RAM value for each build, designed to slice up a machine equally for the best mix of performance and concurrency. We would recommend it to teams during their onboarding call, then mostly leave it as it was. For a machine with 128GB RAM and 32 threads, you may have set it up for 8 jobs with 4x vCPU and 16GB RAM each, or 4 jobs with 8x vCPU and 32GB RAM.

\n

However, whilst working with Justin Gray, CTO at Toolpath, we found that their build needed increasing amounts of RAM to avoid an Out Of Memory (OOM) crash, and so implemented custom labels.

\n

These labels do not have any predetermined values, so you can change them to any value you like, independently. You're not locked into a set combinations.

\n

Small tasks, automation, publishing Helm charts?

\n
runs-on: actuated-2cpu-8gb\n
\n

Building a large application, or training an AI model?

\n
runs-on: actuated-32cpu-128gb\n
\n

Spreading your plan across all available hosts

\n

Previously, if you had a plan with 10 concurrent builds and both an Arm server and an amd64 server, we'd split your plan statically 50/50. So you could run a maximum of 5 Arm and 5 amd64 builds at the same time.

\n

Now, we've made this dynamic, all of your 10 builds can start on the Arm or amd64 server. Or, 1 could start on the Arm server, then 9 on the amd64 server, and so on.

\n

The change makes the product better value for money, and we had always wanted it to work this way.

\n

Thanks to Patrick Stephens at Fluent/Calyptia for the suggestion and for helping us test it out.

\n

KVM acceleration aka running VMs in your CI pipeline

\n

When we started actuated over 12 months ago, there was no support for using KVM acceleration, or running a VM within a GitHub Actions job within GitHub's infrastructure. We made it available for our customers first, with a custom Kernel configuration for x86_64 servers. Arm support for launching VMs within VMs is not currently available in the current generation of Ampere servers, but may be available within the next generation of chips and Kernels.

\n

We have several tutorials including how to run Firecracker itself within a CI job, Packer, Nix and more.

\n

When you run Packer in a VM, instead of with one of the cloud drivers, you save on time and costs, by not having to fire up cloud resources on AWS, GCP, Azure, and so forth. Instead, you can run a local VM to build the image, then convert it to an AMI or another format.

\n

One of our customers has started exploring launching a VM during a CI job in order to test air-gapped support for enterprise customers. This is a great example of how you can use nested virtualisation to test your own product in a repeatable way.

\n

Nix benefits particularly from being able to create a clean, isolated environment within a CI pipeline, to get a repeatable build. Graham Christensen from Determinate Systems reached out to collaborate on testing their Nix installer in actuated.

\n

He didn't expect it to run, but when it worked first time, he remarked: \"Perfect! I'm impressed and happy that our action works out of the box.\"

\n
jobs:\n  specs:\n    name: ci\n    runs-on: [actuated-16cpu-32gb]\n    steps:\n      - uses: DeterminateSystems/nix-installer-action@main\n      - run: |\n            nix-build '<nixpkgs/nixos/tests/doas.nix>'\n
\n\n

Wrapping up

\n

We've now released eBPF/BTF support as part of onboarding CNCF projects, updated to the latest Kernel revision, made scheduling better value for money & easier to customise, and have added a range of tutorials for getting the most out of nested virtualisation.

\n

If you'd like to try out actuated, you can get started same day.

\n

Talk to us..

\n

You may also like:

\n","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/develop-a-great-go-cli.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/develop-a-great-go-cli.json deleted file mode 100644 index da053560..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/develop-a-great-go-cli.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","contentHtml":"

Is your project's CLI growing with you? I'll cover some of the lessons learned writing the OpenFaaS, actuated, actions-usage, arkade and k3sup CLIs, going as far back as 2016. I hope you'll find some ideas or inspiration for your own projects - either to start them off, or to improve them as you go along.

\n
\n

Just starting your journey, or want to go deeper?

\n

You can master the fundamentals of Go (also called Golang) with my eBook Everyday Golang, which includes chapters on Go routines, HTTP clients and servers, text templates, unit testing and crafting a CLI. If you're on a budget, I would recommend checkout out the official Go tour, too.

\n
\n

Introduction

\n

The earliest CLI I wrote was for OpenFaaS, called faas-cli. It's a client for a REST API exposed over HTTP, and I remember how it felt to add the first command list functions, then one more, and one more, until it was a fully working CLI with a dozen commands.

\n

But it started with one command - something that was useful to us at the time, that was to list the available functions.

\n

The initial version used Go's built-in flags parser, which is rudimentary, but perfectly functional.

\n
faas-cli -list\nfaas-cli -describe\nfaas-cli -deploy\n
\n

Over time, you may outgrow this simple approach, and drift towards wanting sub-commands, each with their own set of options.

\n

An early contributor John McCabe introduced me to Cobra and asked if he could convert everything over.

\n
faas-cli list\nfaas-cli describe\nfaas-cli deploy\n
\n

Now each sub-command can have its set of flags, and even sub-commands in the case of faas-cli secret list/create/delete

\n

actions-usage is a free analytics tool we wrote for GitHub Actions users to iterate GitHub's API and summarise your usage over a certain period of time. It's also written in Go, but because it's mostly single-purpose, it'll probably never need sub-commands.

\n
actions-usage -days 28 \\\n    -token-file ~/pat.txt \\\n    -org openfaasltd\n
\n

Shortly after launching the tool for teams an open-source organisations, we had a feature request to run it on individual user accounts.

\n

That meant switching up some API calls and adding new CLI flags:

\n
actions-usage -days 7 \\\n    -token-file ~/pat.txt \\\n    -user alexellis\n
\n

We then got a bit clever and started adding some extra reports and details, you can see what it looks in the article Understand your usage of GitHub Actions

\n

What's new for actuated-cli

\n

I'm very much a believer in a Minimal Viable Product (MVP). If you can create some value or utility to users, you should ship it as early as possible, especially if you have a good feedback loop with them.

\n

A quick note about the actuated-cli, it's main use-cases are to:

\n\n

No more owner flags

\n

The actuated-cli was designed to work on a certain organisation, but it meant extra typing, so wherever possible, we've removed the flag completely.

\n
actuated-cli runners --owner openfaasltd\n
\n

becomes:

\n
actuated-cli runners\n
\n

How did we do this? We determine the intersection of organisations for which your account is authorized, and which are enrolled for actuated. It's much less typing and it's more intuitive.

\n

The host flag became a positional argument

\n

This was another exercise in reducing typing. Let's say we wanted to upgrade the agent for a certain host, we'd have to type:

\n
actuated-cli upgrade --owner openfaasltd --host server-1\n
\n

By looking at the \"args\" slice, instead of for a specific command, we can assume that any text after the flags is always the server name:

\n
actuated-cli upgrade --owner openfaasltd server-1\n
\n

Token management

\n

The actuated CLI uses a GitHub personal access token to authenticate with the API. This is a common pattern, but it's not always clear how to manage the token.

\n

We took inspiration from the gh CLI, which is a wrapper around the GitHub API.

\n

The gh CLI has a gh auth command which can be used to obtain a token, and save it to a local file, then any future usage of the CLI will use that token.

\n

Before, you had to create a Personal Access Token in the GitHub UI, then copy and paste it into a file, and decide where to put it, and what to name it. What's more, if you missed a permission, then the token wouldn't work.

\n
actuated-cli --token ~/pat.txt\n
\n

Now, you simply run:

\n
actuated-cli auth\n
\n

And as you saw from the previous commands, there's no longer any need for the --token flag. Unless of course, you want to supply it, then you can.

\n

A good way to have a default for a flag, and then an override, is to use the Cobra package's Changed() function. Read the default, unless .Changed() on the --token or --token-value flags return true.

\n

The --json flag

\n

From early on, I knew that I would want to be able to pipe output into .jq, or perhaps even do some scripting. I've seen this in docker, kubectl and numerous other CLI tools written in Go.

\n
actuated-cli runners --json | jq '.[] | .name'\n\n\"m1m1\"\n\"m1m2\"\n\"of-epyc-lon1\"\n
\n

The JSON format also allows you to get access to certain fields which the API call returns, which may not be printed by the default command's text-based formatter:

\n
|         NAME         |  CUSTOMER   |   STATUS    | VMS  | PING  |   UP    | CPUS |   RAM   | FREE RAM | ARCH  |                 VERSION                  |\n|----------------------|-------------|-------------|------|-------|---------|------|---------|----------|-------|------------------------------------------|\n| of-epyc-lon1         | openfaasltd | running     | 0/5  | 7ms   | 6 days  |   48 | 65.42GB | 62.85GB  | amd64 | 5f702001a952e496a9873d2e37643bdf4a91c229 |\n
\n

Instead, we get:

\n
[  {\n    \"name\": \"of-epyc-lon1\",\n    \"customer\": \"openfaasltd\",\n    \"pingNano\": 30994998,\n    \"uptimeNano\": 579599000000000,\n    \"cpus\": 48,\n    \"memory\": 65423184000,\n    \"memoryAvailable\": 62852432000,\n    \"vms\": 0,\n    \"maxVms\": 5,\n    \"reachable\": true,\n    \"status\": \"running\",\n    \"agentSHA\": \"5f702001a952e496a9873d2e37643bdf4a91c229\",\n    \"arch\": \"amd64\"\n  }\n]\n
\n

SSH commands and doing the right thing

\n

Actuated has a built-in SSH gateway, this means that any job can be debugged - whether running on a hosted or self-hosted runner, just by editing the workflow YAML.

\n

Add the following to the - steps: section, and the id_token: write permission, and your workflow will pause, and then you can connect over SSH using the CLI or the UI.

\n
    - uses: self-actuated/connect-ssh@master\n
\n

There are two sub-commands:

\n\n

Here's an example of having only one connection:

\n
actuated-cli ssh list\n| NO  |   ACTOR   |   HOSTNAME    | RX | TX | CONNECTED |\n|-----|-----------|---------------|----|----|-----------|\n|   1 | alexellis | fv-az1125-168 |  0 |  0 | 32s       |\n
\n

Now how do you think the ssh connect command should work?

\n

Here's the most obvious way:

\n
actuated-cli ssh connect --hostname fv-az1125-168\n
\n

This is a little obtuse, since we only have one server to connect to, we can improve it for the user, with:

\n
actuated-cli ssh connect\n
\n

That's right, we do the right thing, the obvious thing.

\n

Then when there is more than one connection, instead of adding two flags --no or --hostname, we can simply take the positional argument:

\n
actuated-cli ssh connect 1\nactuated-cli ssh connect fv-az1125-168\n
\n

Are there any places where you could simplify your own CLI?

\n

Read the source code here: ssh_connect.go

\n

The --verbose flag

\n

We haven't made any use of the --verbose flag yet in the CLI, but it's a common pattern which has been used in faas-cli and various others. Once your output gets to a certain width, it can be hard to view in a terminal, like the output from the previous command.

\n

To implement --verbose, you should reduce the columns to the absolute minimum to be useful, so maybe we could give up the Version, customer, ping, and CPUs columns in the standard view, then add them back in with --verbose.

\n

Table printing

\n

As you can see from the output of the commands above, we make heavy usage of a table printer.

\n

You don't necessarily need a 3rd-party table printer, Go has a fairly good \"tab writer\" which can create nicely formatted code:

\n
faas-cli list -g https://openfaas.example.com\nFunction                        Invocations     Replicas\nbcrypt                          9               1    \nfiglet                          0               1    \ninception                       0               1    \nnodeinfo                        2               1    \nping-url                        0               1  \n
\n

You can find the standard tabwriter package here.

\n

Or try out the tablewriter package by Olekukonko. We've been able to make use of it in arkade too - a free marketplace for developer tools.

\n

See usage in arkade here: table.go

\n

See usage in actuated-cli's SSH command here: ssh_ls.go

\n

Progress bars

\n

One thing that has been great about having open-source CLIs, is that other people make suggestions and help you learn about new patterns.

\n

For arkade, Ramiro from Okteto sent a PR to add a progress bar to show how long remained to download a big binary like the Kubernetes CLI.

\n
arkade get kubectl\nDownloading: kubectl\nDownloading: https://storage.googleapis.com/kubernetes-release/release/v1.24.2/bin/linux/amd64/kubectl\n\n15.28 MiB / 43.59 MiB [------------------------>____________________________________] 35.05%\n
\n

It's simple, but gives enough feedback to stop you from thinking the program is stuck. In my Human Design Interaction course at university, I learned that anything over 7s triggers uncertainty in an end-user.

\n

See how it's implemented: download.go

\n

HTTP and REST are not the only option

\n

When I wrote K3sup, a tool to install K3s on remote servers, I turned to SSH to automate the process. So rather than making HTTP calls, a Go library for SSH is used to open a connection and run remote commands.

\n

It also simplifies an annoying post-installation task - managing the kubeconfig file. By default this is a protected file on the initial server you set up, k3sup will download the file and merge it with your local kubeconfig.

\n
k3sup install \\\n    --host HOST1 \\\n    --user ubuntu \\\n    --merge \\\n    --local-path ~/.kube/config\n
\n

I'd recommend trying out golang.org/x/crypto/ssh in your own CLIs and tools. It's great for automation, and really simple to use.

\n

Document everything as best as you can

\n

Here's an example of a command with good documentation:

\n
Schedule additional VMs to repair the build queue.\nUse sparingly, check the build queue to see if there is a need for \nmore VMs to be launched. Then, allow ample time for the new VMs to \npick up a job by checking the build queue again for an in_progress\nstatus.\n\nUsage:\n  actuated-cli repair [flags]\n\nExamples:\n  ## Launch VMs for queued jobs in a given organisation\n  actuated repair OWNER\n\n  ## Launch VMs for queued jobs in a given organisation for a customer\n  actuated repair --staff OWNER\n\n\nFlags:\n  -h, --help    help for repair\n  -s, --staff   List staff repair\n\nGlobal Flags:\n  -t, --token string         File to read for Personal Access Token (default \"$HOME/.actuated/PAT\")\n      --token-value string   Personal Access Token\n
\n

Not only does it show example usage, so users can understand what can be done, but it has a detailed explanation of when to use the command.

\n
\tcmd := &cobra.Command{\n\t\tUse:   \"repair\",\n\t\tShort: \"Schedule additional VMs to repair the build queue\",\n\t\tLong: `Schedule additional VMs to repair the build queue.\nUse sparingly, check the build queue to see if there is a need for \nmore VMs to be launched. Then, allow ample time for the new VMs to \npick up a job by checking the build queue again for an in_progress\nstatus.`,\n\t\tExample: `  ## Launch VMs for queued jobs in a given organisation\n  actuated repair OWNER\n\n  ## Launch VMs for queued jobs in a given organisation for a customer\n  actuated repair --staff OWNER\n`\n    }\n
\n

Browse the source code: repair.go

\n

Wrapping up

\n

I covered just a few of the recent changes - some were driven by end-user feedback, others were open source contributions, and in some cases, we just wanted to make the CLI easier to use. I've been writing CLIs for a long time, and I still have a lot to learn.

\n

What CLIs do you maintain? Could you apply any of the above to them?

\n

Do you want to learn how to master the fundamentals of Go? Check out my eBook: Everyday Go.

\n

If you're on a budget, I would recommend checkout out the official Go tour, too. It'll help you understand some of the basics of the language and is a good primer for the e-book.

\n

Read the source code of the CLIs we mentioned:

\n","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-nix-builds.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-nix-builds.json deleted file mode 100644 index 5b2ffc67..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-nix-builds.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","contentHtml":"

faasd is a lightweight and portable version of OpenFaaS that was created to run on a single host. In my spare time I maintain faasd-nix, a project that packages faasd and exposes a NixOS module so it can be run with NixOS.

\n

The module itself depends on faasd, containerd and the CNI plugins and all of these binaries are built in CI with Nix and then cached using Cachix to save time on subsequent builds.

\n

I often deploy faasd with NixOS on a Raspberry Pi and to the cloud, so I build binaries for both x86_64 and aarch64. The build usually runs on the default GitHub hosted action runners. Now because GitHub currently doesn't have Arm support, I use QEMU instead which can emulate them. The drawback of this approach is that builds can sometimes be several times slower.

\n
\n

For some of our customers, their builds couldn't even complete in 6 hours using QEMU, and only took between 5-20 minutes using native Arm hardware. Alex Ellis, Founder of Actuated.

\n
\n

While upgrading to the latest nixpkgs release recently I decided to try and build the project on runners managed with Actuated to see the improvements that can be made by switching to both bigger x86_64 iron and native Arm hardware.

\n

Nix and GitHub actions

\n

One of the features Nix offers are reproducible builds. Once a package is declared it can be built on any system. There is no need to prepare your machine with all the build dependencies. The only requirement is that Nix is installed.

\n
\n

If you are new to Nix, then I'd recommend you read the Zero to Nix guide. It's what got me excited about the project.

\n
\n

Because Nix is declarative and offers reproducible builds, it is easy to setup a concise build pipeline for GitHub actions. A lot of steps usually required to setup the build environment can be left out. For instance, faasd requires Go, but there's no need to install it onto the build machine, and you'd normally have to install btrfs-progs to build containerd, but that's not something you have to think about, because Nix will take care of it for you.

\n

Another advantage of the reproducible builds is that if it works on your local machine it most likely also works in CI. No need to debug and find any discrepancies between your local and CI environment.

\n
\n

Of course, if you ever do get frustrated and want to debug a build, you can use the built-in SSH feature in Actuated. Alex Ellis, Founder of Actuated.

\n
\n

This is what the workflow looks like for building faasd and its related packages:

\n
jobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - uses: cachix/install-nix-action@v21\n      - name: Build faasd 🔧\n        run: |\n          nix build -L .#faasd\n      - name: Build containerd 🔧\n        run: |\n          nix build -L .#faasd-containerd\n      - name: Build cni-plugin 🔧\n        run: |\n          nix build -L .#faasd-cni-plugins\n
\n

All this pipeline does is install Nix, using the cachix/install-nix-action and run the nix build command for the packages that need to be built.

\n

Notes on the nix build for aarch64

\n

To build the packages for multiple architectures there are a couple of options:

\n\n

The preferred option would be to compile everything natively on an aarch64 machine as that would result in the best performance. However, at the time of writing GitHub does not provide Arm runners. That is why QEMU is used by many people to compile binaries in CI.

\n

Enabling the binfmt wrapper on NixOS can be done easily through the NixOS configuration. On non-NixOS machines, like on the GitHub runner VM, the QEMU static binaries need to be installed and the Nix daemon configuration updated.

\n

Instructions to configure Nix for compilation with QEMU can be found on the NixOS wiki

\n

The workflow for building aarch64 packages with QEMU on GitHub Actions looks like this:

\n
jobs:\n  build:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - uses: docker/setup-qemu-action@v2\n      - uses: cachix/install-nix-action@v21\n        with:\n          extra_nix_config: |\n            extra-platforms = aarch64-linux\n      - name: Build faasd 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd\n      - name: Build containerd 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd-containerd\n      - name: Build cni-plugin 🔧\n        run: |\n          nix build -L .#packages.aarch64-linux.faasd-cni-plugins\n\n
\n

Install the QEMU static binaries using docker/setup-qemu-action. Let the nix daemon know that it can build for aarch64 by adding extra-platforms = aarch64-linux via the extra_nix_config input on the install nix action. Update the nix build commands to specify platform e.g. nix build .#packages.aarch64-linux.faasd.

\n

Speeding up the build with a Raspberry Pi

\n

Nix has great support for caching and build speeds can be improved greatly by never building things twice. This project normally uses Cachix for caching and charing binaries across systems. For this comparison caching was disabled. All packages and their dependencies are built from scratch again each time.

\n

Building the project takes around 4 minutes and 20 seconds on the standard GitHub hosted runner. After switching to a more powerful Actuated runner with 4CPUs and 8GB of RAM the build time dropped to 2 minutes and 15 seconds.

\n

\"Comparison

\n
\n

Comparison of more powerful Actuated runner with GitHub hosted runner.

\n
\n

While build times are still acceptable for x86_64 this is not the case for the aarch64 build. It takes around 55 minutes to complete the Arm build with QEMU on a GitHub runner.

\n

Running the same build with QEMU on the Actuated runner already brings down the build time to 19 minutes and 40 seconds. Running the build natively on a Raspberry Pi 4 (8GB) completed in 11 minutes and 47 seconds. Building on a more powerful Arm machine would potentially reduce this time to a couple of minutes.

\n

\"Results

\n
\n

Results of the matrix build comparing the GitHub hosted runner and the 2 Actuated runners.

\n
\n

Running the build natively on the Pi did even beat the fast bare-metal machine that is using QEMU.

\n

My colleague Alex ran the same build on his Raspberry Pi using Actuated and an NVMe mounted over USB-C, he got the build time down even further. Why? Because it increased the I/O performance. In fact, if you build this on server-grade Arm like the Ampere Altra, it would be about 4x faster than the Pi 4.

\n

Building for Arm:

\n\n

Building for x86_64

\n\n

\"Alex's

\n

These results show that whatever the Arm hardware you pick, it'll likely be faster than QEMU, even when QEMU is run on the fastest bare-metal available, the slowest Arm hardware will beat it by minutes.

\n

Wrapping up

\n

Building your projects with Nix allows your GitHub actions pipelines to be concise and easy to maintain.

\n

Even when you are not using Nix to build your project it can still help you to create concise and easy to maintain GitHub Action workflows. With Nix shell environments you can use Nix to declare which dependencies you want to make available inside an isolated shell environment for your project: Streamline your GitHub Actions dependencies using Nix

\n

Building Nix packages or entire NixOS systems on GitHub Actions can be slow especially if you need to build for Arm. Bringing your own metal to GitHub actions can speed up your builds. If you need Arm runners, Actuated is one of the only options for securely isolated CI that is safe for Open Source and public repositories. Alex explains why in: Is the GitHub Actions self-hosted runner safe for Open Source?

\n

Another powerful feature of the Nix ecosystem is the ability to run integration tests using virtual machines (NixOS test). This feature requires hardware acceleration to be available in the CI runner. Actuated makes it possible to run these tests in GitHub Actions CI pipelines: how to run KVM guests in your GitHub Actions.

\n

See also:

\n","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-self-hosted-cache.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-self-hosted-cache.json deleted file mode 100644 index cd8dc34d..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/faster-self-hosted-cache.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","contentHtml":"

In some of our builds for actuated we cache things like the Linux Kernel, so we don't needlessly rebuild it when we update packages in our base images. It can shave minutes off every build meaning our servers can be used more efficiently. Most customers we've seen so far only make light to modest use of GitHub's hosted cache, so haven't noticed much of a latency problem.

\n

But you don't have to spend too long on the issuer tracker for GitHub Actions to find people complaining about the cache being slow or locking up completely for self-hosted runners.

\n

Go, Rust, Python and other languages don't tend to make heavy use of caches, and Docker has some of its own mechanisms like building cached steps into published images aka inline caching. But for the Node.js ecosystem, the node_modules folder and yarn cache can become huge and take a long time to download. That's one place where you may start to see tension between the speed of self-hosted runners and the latency of the cache. If your repository is a monorepo or has lots of large artifacts, you may get a speed boost by caching that too.

\n

So why is GitHub's cache so fast for hosted runners, and (sometimes) so slow self-hosted runners?

\n

Simply put - GitHub runs VMs and the accompanying cache on the same network, so they can talk over a high speed backbone connection. But when you run a self-hosted runner, then any download or upload operations are taking place over the public Internet.

\n

Something else that can slow builds down is having to download large base images from the Docker Hub. We've already covered how to solve that for actuated in the docs.

\n

Speeding up in the real world

\n

We recently worked with Roderik, the CTO of SettleMint to migrate their CI from a self-hosted Kubernetes solution Actions Runtime Controller (ARC) to actuated. He told me that they originally moved from GitHub's hosted runners to ARC to save money, increase speed and to lower the latency of their builds. Unfortunately, running container builds within Kubernetes provided very poor isolation, and side effects were being left over between builds, even with a pool of ephemeral containers. They also wanted to reduce the amount of effort required to maintain a Kubernetes cluster and control-plane for CI.

\n

Roderik explained that he'd been able to get times down by using pnpm instead of yarn, and said every Node project should try it out to see the speed increases. He believes the main improvement is due to efficient downloading and caching. pnpm is a drop-in replacement for npm and yarn, and is compatible with both.

\n
\n

In some cases, we found that downloading dependencies from the Internet was faster than using GitHub's remote cache. The speed for a hosted runner was often over 100MBs/sec, but for a self-hosted runner it was closer to 20MBs/sec.

\n
\n

That's when we started to look into how we could run a cache directly on the same network as our self-hosted runners, or even on the machine that was scheduling the Firecracker VMs.

\n
\n

\"With the local cache that Alex helped us set up, the cache is almost instantaneous. It doesn't even have time to show a progress bar.\"

\n
\n

Long story short, SettleMint have successfully migrated their CI for x86 and Arm to actuated for the whole developer team:

\n

Super happy with my new self hosted GHA runners powered by @selfactuated, native speeds on both AMD and ARM bare metal monster machines. Our CI now goes brrrr… pic.twitter.com/quZ4qfcLmu

— roderik.eth (@r0derik) May 23, 2023
\n

This post is about speed improvements for caching, but if you're finding that QEMU is too slow to build your Arm containers on hosted runners, you may benefit from switching to actuated with bare-metal Arm servers.

\n

See also:

\n\n

Set up a self-hosted cache for GitHub Actions

\n

In order to set up a self-hosted cache for GitHub Actions, we switched out the official actions/cache@v3 action for tespkg/actions-cache@v1 created by Target Energy Solutions, a UK-based company, which can target S3 instead of the proprietary GitHub cache.

\n

We then had to chose between Seaweedfs and Minio for the self-hosted S3 server. Of course, there's also nothing stopping you from actually using AWS S3, or Google Cloud Storage, or another hosted service.

\n

Then, the question was - should we run the S3 service directly on the server that was running Firecracker VMs, for ultimate near-loopback speed, or on a machine provisioned in the same region, just like GitHub does with Azure?

\n

Either would be a fine option. If you decide to host a public S3 cache, make sure that authentication and TLS are both enabled. You may also want to set up an IP whitelist just to deter any bots that may scan for public endpoints.

\n

Set up Seaweedfs

\n

The Seaweedfs README describes the project as:

\n
\n

\"a fast distributed storage system for blobs, objects, files, and data lake, for billions of files! Blob store has O(1) disk seek, cloud tiering. Filer supports Cloud Drive, cross-DC active-active replication, Kubernetes, POSIX FUSE mount, S3 API, S3 Gateway, Hadoop, WebDAV, encryption, Erasure Coding.\"

\n
\n

We liked it so much that we'd already added it to the arkade marketplace, arkade is a faster, developer-focused alternative to brew.

\n
arkade get seaweedfs\nsudo mv ~/.arkade/bin/seaweedfs /usr/local/bin\n
\n

Define a secret key and access key to be used from the CI jobs in the /etc/seaweedfs/s3.conf file:

\n
{\n  \"identities\": [\n    {\n      \"name\": \"actuated\",\n      \"credentials\": [\n        {\n          \"accessKey\": \"s3cr3t\",\n          \"secretKey\": \"s3cr3t\"\n        }\n      ],\n      \"actions\": [\n        \"Admin\",\n        \"Read\",\n        \"List\",\n        \"Tagging\",\n        \"Write\"\n      ]\n    }\n  ]\n}\n
\n

Create seaweedfs.service:

\n
[Unit]\nDescription=SeaweedFS\nAfter=network.target\n\n[Service]\nUser=root\nExecStart=/usr/local/bin/seaweedfs server -ip=192.168.128.1 -volume.max=0 -volume.fileSizeLimitMB=2048 -dir=/home/runner-cache -s3 -s3.config=/etc/seaweedfs/s3.conf\nRestart=on-failure\n\n[Install]\nWantedBy=multi-user.target\n
\n

We have set -volume.max=0 -volume.fileSizeLimitMB=2048 to minimize the amount of space used and to allow large zip files of up to 2GB, but you can change this to suit your needs. See seaweedfs server --help for more options.

\n

Install it and check that it started:

\n
sudo cp ./seaweedfs.service /etc/systemd/system/seaweedfs.service\nsudo systemctl enable seaweedfs\n\nsudo journalctl -u seaweedfs -f\n
\n

Try it out

\n

You'll need to decide what you want to cache and whether you want to use a hosted, or self-hosted S3 service - either directly on the actuated server or on a separate machine in the same region.

\n

Roderik explained that the pnpm cache was important for node_modules, but that actually caching the git checkout saved a lot of time too. So he added both into his builds.

\n

Here's an example:

\n
    - name: \"Set current date as env variable\"\n      shell: bash\n      run: |\n        echo \"CHECKOUT_DATE=$(date +'%V-%Y')\" >> $GITHUB_ENV\n      id: date\n    - uses: tespkg/actions-cache@v1\n      with:\n        endpoint: \"192.168.128.1\"\n        port: 8333\n        insecure: true\n        accessKey: \"s3cr3t\"\n        secretKey: \"s3cr3t\"\n        bucket: actuated-runners\n        region: local\n        use-fallback: true\n        path: ./.git\n        key: ${{ runner.os }}-checkout-${{ env.CHECKOUT_DATE }}\n        restore-keys: |\n          ${{ runner.os }}-checkout-\n
\n\n

See also: Official GitHub Actions Cache action

\n

You may also want to create a self-signed certificate for the S3 service and then set insecure: false to ensure that the connection is encrypted. If you're running these builds within private repositories, tampering is unlikely.

\n

Roderik explained that the cache key uses a week-year format, rather than a SHA. Why? Because a SHA would change on every build, meaning that a save and load would be performed on every build, using up more space and slowing things down. In this example, There's only ever 52 cache entries per year.

\n
\n

You define a key which is unique if the cache needs to be updated. Then you define a restore key that matches part or all of the key.\nPart means it takes the last one that matches, then updates at the end of the run, in the post part, it then uses the key to upload the zip file if the key is different from the one stored.

\n
\n

In one instance, a cached checkout went from 2m40s to 11s. That kind of time saving adds up quickly if you have a lot of builds.

\n

Roderik's pipeline has multiple steps, and may need to run multiple times, so we're looking at 55s instead of 13 minutes for 5 jobs or runs.

\n

\"Example

\n
\n

One of the team's pipelines

\n
\n

Here's how to enable a cache for pnpm:

\n
    - name: Install PNPM\n      uses: pnpm/action-setup@v2\n      with:\n        run_install: |\n          - args: [--global, node-gyp]\n\n    - name: Get pnpm store directory\n      id: pnpm-cache\n      shell: bash\n      run: |\n        echo \"STORE_PATH=$(pnpm store path)\" >> $GITHUB_OUTPUT\n\n    - uses: tespkg/actions-cache@v1\n      with:\n        endpoint: \"192.168.128.1\"\n        port: 8333\n        insecure: true\n        accessKey: \"s3cr3t\"\n        secretKey: \"s3cr3t\"\n        bucket: actuated-runners\n        region: local\n        use-fallback: true\n        path:\n          ${{ steps.pnpm-cache.outputs.STORE_PATH }}\n          ~/.cache\n          .cache\n        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}\n        restore-keys: |\n          ${{ runner.os }}-pnpm-store-\n\n    - name: Install dependencies\n      shell: bash\n      run: |\n        pnpm install --frozen-lockfile --prefer-offline\n      env:\n        HUSKY: '0'\n        NODE_ENV: development\n
\n

Picking a good key and restore key can help optimize when the cache is read from and written to:

\n
\n

\"You need to determine a good key and restore key. For pnpm, we use the hash of the lock file in the key, but leave it out of the restore key. So if I update the lock file, it starts from the last cache, updates it, and stores the new cache with the new hash\"

\n
\n

If you'd like a good starting-point for GitHub Actions Caching, Han Verstraete from our team wrote up a good primer for the actuated docs:

\n

Example: GitHub Actions cache

\n

Conclusion

\n

We were able to dramatically speed up caching for GitHub Actions by using a self-hosted S3 service. We used Seaweedfs directly on the server running Firecracker with a fallback to GitHub's cache if the S3 service was unavailable.

\n

\"Brr\"

\n
\n

An Ampere Altra Arm server running parallel VMs using Firecracker. The CPU is going brr. Find a server with our guide

\n
\n

We also tend to recommend that all customers enable a mirror of the Docker Hub to counter restrictive rate-limits. The other reason is to avoid any penalties that you'd see from downloading large base images - or from downloading small to medium sized images when running in high concurrency.

\n

You can find out how to configure a container mirror for the Docker Hub using actuated here: Set up a registry mirror. When testing builds for the Discourse team, there was a 2.5GB container image used for UI testing with various browsers preinstalled within it. We found that we could shave off a few minutes off the build time by using the local mirror. Imagine 10x of those builds running at once, needlessly downloading 250GB of data.

\n

What if you're not an actuated customer? Can you still benefit from a faster cache? You could try out a hosted service like AWS S3 or Google Cloud Storage, provisioned in a region closer to your runners. The speed probably won't quite be as good, but it should still be a lot faster than reaching over the Internet to GitHub's cache.

\n

If you'd like to try out actuated for your team, reach out to us to find out more.

\n

Book 20 mins with me if you think your team could benefit from the below for GitHub Actions:

🚀 Insights into CI usage across your organisation
🚀 Faster x86 builds
🚀 Native Arm builds that can actually finish
🚀 Fixed-costs & less managementhttps://t.co/iTiZsH9pgv

— Alex Ellis (@alexellisuk) May 10, 2023
","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/firecracker-container-lab.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/firecracker-container-lab.json deleted file mode 100644 index 9c5f7602..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/firecracker-container-lab.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","contentHtml":"

When I started learning Firecracker, I ran into frustration after frustration with broken tutorials that were popular in their day, but just hadn't been kept up to date. Almost nothing worked, or was far too complex for the level of interest I had at the time. Most recently, one of the Firecracker maintainers in an effort to make the quickstart better, made it even harder to use. (You can still get a copy of the original Firecracker quickstart in our tutorial on nested virtualisation)

\n

So I wrote a lab that takes a container image and converts it to a microVM. You'll get your hands dirty, you'll run a microVM, you'll be able to use curl and ssh, even expose a HTTP server to the Internet via inlets, if (like me), you find that kind of thing fun.

\n

Why would you want to explore Firecracker? A friend of mine, Ivan Velichko is a prolific writer on containers, and Docker. He is one of the biggest independent evangelists for containers and Kubernetes that I know.

\n

So when he wanted to build an online labs and training environment, why did he pick Firecracker instead? Simply put, he told us that containers don't cut it. He needed something that would mirror the type of machine that you'd encounter in production, when you provision an EC2 instance or a GCP VM. Running Docker, Kubernetes, and performing are hard to do securely within a container, and he knew that was important for his students.

\n

For us - we had very similar reasons for picking Firecracker for a secure CI solution. Too often the security issues around running privileged containers, and the slow speed of Docker In Docker's (DIND) Virtual Filesystem Driver (VFS), are ignored. Heads are put into the sand. We couldn't do that and developed actuated.dev as a result. Since we launched the pilot, we've now run over 110k VMs for customer CI jobs on GitHub Actions, and have a tech preview for GitLab CI where a job can be running within 1 second of pushing a \"commit\".

\n

So let's get that microVM running for you?

\n

How it works 🔬

\n

How to build a microVM from a container

\n

\"/images/2023-09-firecracker-lab/conceptual.png\"

\n
\n

Conceptual archicture of the lab

\n
\n

Here's what we'll be doing:

\n\n

Let's look at why we need a init, instead of just running the entrypoint of a container.

\n

Whilst in theory, you can start a microVM where the first process (PID 1) is your workload, in the same way as Docker, it will leave you with a system which is not properly initialised with things like a /proc/ filesystem, tempfs, hostname, and other things that you'd expect to find in a Linux system.

\n

For that reason, you'll need to either install systemd into the container image you want to use, or build your own basic init system, which sets up the machine, then starts your workload.

\n

We're doing the latter here.

\n

In the below program, you'll see key devices and files mounted, to make a functional system. The hostname is then set by using a syscall, and finally /bin/sh is started. You could also start a specific binary, or build an agent into the init for Remote Procedure Calls (RPC) to start and stop your workload, and to query metrics.

\n

The team at Fly.io built their own init and agent combined, and opened-sourced a very early version: github.com/superfly/init-snapshot.

\n

You'll find my init in: ./init/main.go:

\n
// Copyright Alex Ellis 2023\n\npackage main\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\n\t\"syscall\"\n)\n\nconst paths = \"PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin\"\n\n// main starts an init process that can prepare an environment and start a shell\n// after the Kernel has started.\nfunc main() {\n\tfmt.Printf(\"Lab init booting\\nCopyright Alex Ellis 2022, OpenFaaS Ltd\\n\")\n\n\tmount(\"none\", \"/proc\", \"proc\", 0)\n\tmount(\"none\", \"/dev/pts\", \"devpts\", 0)\n\tmount(\"none\", \"/dev/mqueue\", \"mqueue\", 0)\n\tmount(\"none\", \"/dev/shm\", \"tmpfs\", 0)\n\tmount(\"none\", \"/sys\", \"sysfs\", 0)\n\tmount(\"none\", \"/sys/fs/cgroup\", \"cgroup\", 0)\n\n\tsetHostname(\"lab-vm\")\n\n\tfmt.Printf(\"Lab starting /bin/sh\\n\")\n\n\tcmd := exec.Command(\"/bin/sh\")\n\n\tcmd.Env = append(cmd.Env, paths)\n\tcmd.Stdin = os.Stdin\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\n\terr := cmd.Start()\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not start /bin/sh, error: %s\", err))\n\t}\n\n\terr = cmd.Wait()\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not wait for /bin/sh, error: %s\", err))\n\t}\n}\n\nfunc setHostname(hostname string) {\n\terr := syscall.Sethostname([]byte(hostname))\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"cannot set hostname to %s, error: %s\", hostname, err))\n\t}\n}\n\nfunc mount(source, target, filesystemtype string, flags uintptr) {\n\n\tif _, err := os.Stat(target); os.IsNotExist(err) {\n\t\terr := os.MkdirAll(target, 0755)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Sprintf(\"error creating target folder: %s %s\", target, err))\n\t\t}\n\t}\n\n\terr := syscall.Mount(source, target, filesystemtype, flags, \"\")\n\tif err != nil {\n\t\tlog.Printf(\"%s\", fmt.Errorf(\"error mounting %s to %s, error: %s\", source, target, err))\n\t}\n}\n
\n

What you'll need

\n

Firecracker is a Virtual Machine Monitor (VMM) that leans on Linux's KVM functionality to run VMs. Its beauty is in its simplicity, however even though it doesn't need a lot, you will need KVM to be available. If you have a bare-metal machine, like your own PC, or an old server or laptop, you're all set. There's also plenty of options for bare-metal in the cloud - billed either on a per minute/hour basis or per month.

\n

And finally, for quick testing, DigitalOcean, GCP, and Azure all support what is known as \"Nested Virtualization\". That's where you obtain a VM, which itself can start further VMs, it's not as fast as bare-metal, but it's cheap and works.

\n

Finally, whilst Firecracker and actuated (our CI product) both support Arm, and Raspberry Pi, this tutorial is only available for `x86_64`` to keep the instructions simple.

\n

Provision the machine

\n

I'd recommend you use Ubuntu 22.04, so that you can copy and paste instructions from this tutorial.

\n

Install Docker CE

\n
curl -fsSL https://get.docker.com  | sudo sh\n
\n

Docker will be used to fetch an initial Operating System, to build the init system, and to customise the root filesystem.

\n

Install arkade, which gives you an easy way to install Firecracker:

\n
curl -sLS https://get.arkade.dev | sudo sh\n
\n

Install Firecracker:

\n
sudo arkade system install firecracker\n
\n

Clone the lab

\n

Clone the lab onto the machine:

\n
git clone https://github.com/alexellis/firecracker-init-lab --depth=1\n\ncd firecracker-init-lab\n
\n

Update the networking script

\n

Find out what the primary interface is on the machine using ip addr or ifconfig.

\n

Edit ./setup-networking.sh:

\n
IFNAME=enp8s0\n
\n

The script will configure a TAP device which bridges microVMs to your host, then sets up IP forwarding and masquerading so that the microVMs can access the Internet.

\n

Run ./setup-networking.sh to setup the TAP device.

\n

Download the Kernel

\n

Add make via build-essential:

\n
sudo apt update && sudo apt install -y build-essential\n
\n

Run make kernel to download the quickstart Kernel made available by the Firecracker team. Of course, you can build your own, but bear in mind that Firecracker does not have PCI support, so many of the ones you'll find on the Internet will not be appropriate.

\n

This Makefile target will not actually build a new Kernel, but wil download one that the Firecracker team have pre-built and uploaded to S3.

\n

Make the container image

\n

Here's the Dockerfile we'll use to build the init system in a multi-stage build, then derive from Alpine Linux for the runtime, this could of course be anything like Ubuntu 22.04, Python, or Node.

\n

./Dockerfile:

\n
FROM golang:1.20-alpine as build\n\nWORKDIR /go/src/github.com/alexellis/firecracker-init-lab/init\n\nCOPY init .\n\nRUN go build --tags netgo --ldflags '-s -w -extldflags \"-lm -lstdc++ -static\"' -o init main.go\n\nFROM alpine:3.18 as runtime\n\nRUN apk add --no-cache curl ca-certificates htop\n\nCOPY --from=build /go/src/github.com/alexellis/firecracker-init-lab/init/init /init\n
\n

I've added in a few extra packages to play with.

\n

Run make root, and you'll see an image in your library:

\n
docker images | grep alexellis2/custom-init\n\nREPOSITORY                                                          TAG                                                                      IMAGE ID       CREATED         SIZE\nalexellis2/custom-init                                              latest                                                                   f89aa7f3dd27   20 hours ago    13.7MB\n
\n

Build the disk image

\n

Firecracker needs a disk image, or an existing block device as its boot drive. You can make this dynamically as required, run make extract to extract the container image into the local filesystem as rootfs.tar.

\n

This step uses docker create followed by docker export to create a temporary container, and then to save its filesystem contents into a tar file.

\n

Run make extract

\n

If you want to see what a filesystem looks like, you could extract rootfs.tar into /tmp and have a poke around. This is not a required step.

\n

Then run make image.

\n

Here, a loopback file allocated with 5GB, then formatted as ext4, under the name rootfs.img. The script mounts the drive and then extracts the contents of the rootfs.tar file into it before unmounting the file.

\n

Start a Firecracker process

\n

Now, this may feel a little odd or different to Docker users. For each Firecracker VM you want to launch, you'll need to start a process, configure it via curl over a UNIX socket, then issue a boot command.

\n

To run multiple Firecracker microVMs at once, configure a different socket path for each.

\n
make start\n
\n

Boot the microVM

\n

In another window, issue the boot command:

\n
make boot\n
\n

Explore the system

\n

You're now booted into a serial console, this isn't a fully functional TTY, so some things won't work like Control + C. The serial console is really just designed for showing boot-up information, not interactive use. For proper remote administration, you should install an OpenSSH server and then connect to the VM using its IP address.

\n

That said, you can now explore a little.

\n

Add a DNS server to /etc/resolv.conf:

\n
echo \"nameserver 8.8.8.8\" > /etc/resolv.conf\n
\n

Then try to reach the Internet:

\n
ping -c 1 8.8.8.8\n\nping -c 4 google.com\n\ncurl --connect-timeout 1 -4 -i http://captive.apple.com/\n\ncurl --connect-timeout 1 -4 -i https://inlets.dev\n
\n

Check out the system specifications:

\n
free -m\ncat /proc/cpuinfo\nip addr\nip route\n
\n

When you're done, kill the firecracker process with sudo killall firecracker, or type in halt to the serial console.

\n

Wrapping up

\n

I was frustrated by the lack of a simple guide for tinkering with Firecracker, and so that's why I wrote this lab and am keeping it up to date.

\n

For production use, you could use a HTTP client to make the API requests to the UNIX socket, or an SDK, which abstracts away some of the complexity. There's an official SDK for Go and several unofficial ones for Rust. If you look at the sample code for either, you'll see that they are doing the same things we did in the lab, so you should find it relatively easy to convert the lab to use an SDK instead.

\n

Did you enjoy the lab? Have you got a use-case for Firecracker? Let me know on Twitter @alexellisuk

\n

If you'd like to see how we've applied Firecracker to bring fast and secure CI to teams, check out our product actuated.dev

\n

Here's a quick demo of our control-plane, scheduler and bare-metal agent in action:

\n","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/github-actions-usage-cli.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/github-actions-usage-cli.json deleted file mode 100644 index fcc81145..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/github-actions-usage-cli.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","contentHtml":"

Whenever GitHub Actions users get in touch with us to ask about actuated, we ask them a number of questions. What do you build? What pain points have you been running into? What are you currently spending? And then - how many minutes are you using?

\n

That final question is a hard one for many to answer because the GitHub UI and API will only show billable minutes. Why is that a problem? Some teams only use open-source repositories with free runners. Others may have a large free allowance of credit for one reason or another and so also don't really know what they're using. Then you have people who already use some form of self-hosted runners - they are also excluded from what GitHub shows you.

\n

So we built an Open Source CLI tool called actions-usage to generate a report of your total minutes by querying GitHub's REST API.

\n

And over time, we had requests to break-down per day - so for our customers in the Middle East like Kubiya, it's common to see a very busy day on Sunday, and not a lot of action on Friday. Given that some teams use mono-repos, we also added the ability to break-down per repository - so you can see which ones are the most active. And finally, we added the ability to see hot-spots of usage like the longest running repo or the most active day.

\n

You can run the tool in three ways:

\n\n

I'll show you each briefly, but the one I like the most is the third option because it's kind of recursive.

\n

Before we get started, download arkade, and use it to install the tool:

\n
# Move the binary yourself into $PATH\ncurl -sLS https://get.arkade.dev | sh\n\n# Have sudo move it for you\ncurl -sLS https://get.arkade.dev | sudo sh\narkade install actions-usage\n
\n

Or if you prefer - you can add my brew tap, or head over to the arkade releases page.

\n

Later on, I'll also show you how to use the alexellis/arkade-get action to install the tool for CI.

\n

Finding out about your organisation

\n

If you want to find out about your organisation, you can run the tool like this:

\n
actions-usage \\\n    -org $GITHUB_REPOSITORY_OWNER \\\n    -days 28 \\\n    -by-repo \\\n    -punch-card \\\n    -token-file ~/PAT.txt\n
\n

You'll need a Personal Access Token, there are instructions on how to create this in the actions-usage README file

\n

There are many log lines printed to stderr during the scan of repositories and the workflows. You can omit all of this by adding 2> /dev/null to the command.

\n

First off we show the totals:

\n
Fetching last 28 days of data (created>=2023-05-19)\n\nGenerated by: https://github.com/self-actuated/actions-usage\nReport for actuated-samples - last: 28 days.\n\nTotal repos: 24\nTotal private repos: 0\nTotal public repos: 24\n\nTotal workflow runs: 107\nTotal workflow jobs: 488\n\nTotal users: 1\n
\n

Then break down on success/failure and cancelled jobs overall, plus the biggest and average build time:

\n
Success: 369/488\nFailure: 45/488\nCancelled: 73/488\n\nLongest build: 29m32s\nAverage build time: 1m26s\n
\n

Next we have the day by day breakdown. You can see that we try to focus on our families on Sunday at OpenFaaS Ltd, instead of working:

\n
Day            Builds\nMonday         61\nTuesday        50\nWednesday      103\nThursday       110\nFriday         153\nSaturday       10\nSunday         0\nTotal          488\n
\n

Our customers in the Middle East work to a different week, and so you'd see Saturday with no builds or nothing, and Sunday like a normal working day.

\n

Then we have the repo-by-repo breakdown with some much more granular data:

\n
Repo                                      Builds         Success        Failure        Cancelled      Skipped        Total          Average        Longest\nactuated-samples/k3sup-matrix-test        355            273            20             62             0              2h59m1s        30s            1m29s\nactuated-samples/discourse                49             38             7              4              0              6h37m21s       8m7s           20m1s\nactuated-samples/specs                    35             31             1              3              0              10m20s         18s            32s\nactuated-samples/cypress-test             17             4              13             0              0              6m23s          23s            49s\nactuated-samples/cilium-test              9              4              2              3              0              1h10m41s       7m51s          29m32s\nactuated-samples/kernel-builder-linux-6.0 9              9              0              0              0              11m28s         1m16s          1m27s\nactuated-samples/actions-usage-job        8              4              2              1              0              46s            6s             11s\nactuated-samples/faasd-nix                6              6              0              0              0              24m20s         4m3s           10m49s\n
\n

Finally, we have the original value that the tool set out to display:

\n
Total usage: 11h40m20s (700 mins)\n
\n

We display the value in a Go duration for readability and in minutes because that's the number that GitHub uses to talk about usage.

\n

One customer told us that they were running into rate limits when querying for 28 days of data, so they dropped down to 14 days and then multiplied the result by two to get a rough estimate.

\n
-days 14 \n
\n

The team at Todoist got in touch with us to see if actuated could reduce their bill on GitHub Actions. When he tried to run the tool the rate-limit was exhausted even when he changed the flag to -days 1. Why? They were using 550,000 minutes!

\n

So we can see one of the limitations already of this approach. Fortunately, actuated customers have their job stats recorded in a database and can generate reports from the dashboard very quickly.

\n

Finding out about your personal account

\n

Actuated isn't built for personal users, but for teams, so we didn't add this feature initially. Then we saw a few people reach out via Twitter and GitHub and decided to add it for them.

\n

For your personal account, you only have to change one of the input parameters:

\n
actions-usage \\\n    -user alexellis \\\n    -days 28 \\\n    -by-repo \\\n    -punch-card \\\n    -token-file ~/ae-pat.txt 2> /dev/null \n
\n

Now I actually have > 250 repositories and most of them don't even have Actions enabled, so this makes the tool less useful for me personally. So it was great when a community member suggested offering a way to filter repos when you have so many that the tool takes a long time to run or can't complete due to rate-limits.

\n

Being that today I used it to get the same insights from a Github Org where I currently work, which contains 1.4k of repositories.

And this was running for a considerable time. I am only related to only a few repositories within this organization.

— lbrealdeveloper (@lbrealdeveloper) June 15, 2023
\n

I've already created an issue and have found someone who'd like to contribute the change: Offer a way to filter repos for large organisations / users #8

\n

This is the beauty of open source and community. We all get to benefit from each other's ideas and contributions.

\n

Running actions-usage with a GitHub Action

\n

Now this is my favourite way to run the tool. I can run it on a schedule and get a report sent to me via email or Slack.

\n

\"Example

\n
\n

Example output from running the tool as a GitHub Action

\n
\n

Create actions-usage.yaml in your .github/workflows folder:

\n
name: actions-usage\n\non:\n  push:\n    branches:\n      - master\n      - main\n  workflow_dispatch:\n\npermissions:\n  actions: read\n\njobs:\n  actions-usage:\n    name: daily-stats\n    runs-on: actuated-any-1cpu-2gb\n    steps:\n    - uses: alexellis/arkade-get@master\n      with:\n        actions-usage: latest\n        print-summary: false\n    - name: Generate actions-usage report\n      run: |\n       echo \"### Actions Usage report by [actuated.dev](https://actuated.com)\" >> SUMMARY\n       echo \"\\`\\`\\`\" >> SUMMARY\n       actions-usage \\\n        -org $GITHUB_REPOSITORY_OWNER \\\n        -days 1 \\\n        -by-repo \\\n        -punch-card \\\n        -token ${{ secrets.GITHUB_TOKEN }}  2> /dev/null  >> SUMMARY\n       echo \"\\`\\`\\`\" >> SUMMARY\n       cat SUMMARY >> $GITHUB_STEP_SUMMARY\n
\n

Ths is designed to run within an organisation, but you can change the -org flag to -user and then use your own username.

\n

The days are for the past day of activity, but you can change this to any number like 7, 14 or 28 days.

\n

You can learn about the other flags by running actions-usage --help on your own machine.

\n

Wrapping up

\n

actions-usage is a practical tool that we use with customers to get an idea of their usage and how we can help with actuated. That said, it's also a completely free and open source tool for which the community is finding their own set of use-cases.

\n

And there are no worries about privacy, we've gone very low tech here. The output is only printed to the console, and we never receive any of your data unless you specifically copy and paste the output into an email.

\n

Feel free to create an issue if you have a feature request or a question.

\n

Check out self-actuated/actions-usage on GitHub

\n

I wrote an eBook writing CLIs like this in Go and keep it up to date on a regular basis - adding new examples and features of Go.

\n

Why not check out what people are saying about it on Gumroad?

\n

Everyday Go - The Fast Track

","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/gpus-for-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/gpus-for-github-actions.json deleted file mode 100644 index d549381c..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/gpus-for-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","contentHtml":"

With the surge of interest in AI and machine learning models, it's not hard to think of reasons why people want GPUs in their workstations and production environments. They make building & training, fine-tuning and serving (inference) from a machine learning model just that much quicker than running with a CPU alone.

\n

So if you build and test code for CPUs in CI pipelines like GitHub Actions, why wouldn't you do the same with code built for GPUs? Why exercise only a portion of your codebase?

\n

GPUs for GitHub Actions

\n

One of our earliest customers moved all their GitHub Actions to actuated for a team of around 30 people, but since Firecracker has no support for GPUs, they had to keep a few self-hosted runners around for testing their models. Their second hand Dell servers were racked in their own datacentre, with 8x 3090 GPUs in each machine.

\n

Their request for GPU support in actuated predated the hype around OpenAI, and was the catalyst for us doing this work.

\n

They told us how many issues they had keeping drivers in sync when trying to use self-hosted runners, and the security issues they ran into with mounting Docker sockets or running privileged containers in Kubernetes.

\n

With a microVM and actuated, you'll be able to test out different versions of drivers as you see fit, and know there will never be side-effects between builds. You can read more in our FAQ on how actuated differs from other solutions which rely on the poor isolation afforded by containers. Actuated is the closest you can get to a hosted runner, whilst having full access to your own hardware.

\n

I'll tell you a bit more about it, how to build your own workstation with commodity hardware, or where to rent a powerful bare-metal host with a capable GPU for less than 200 USD / mo that you can use with actuated.

\n

Available in early access

\n

So today, we're announcing early access to actuated for GPUs. Whether your machine has one GPU, two, or ten, you can allocate them directly to a microVM for a CI job, giving strong isolation, and the same ephemeral environment that you're used to with GitHub's hosted runners.

\n

\"Our

\n
\n

Our test rig has 2x Nvidia 3060 GPUs and is available for customer demos and early testing.

\n
\n

We've compiled a list of vendors that provide access to fast, bare-metal compute, but at the moment, there are only a few options for bare-metal with GPUs.

\n\n

We have a full bill of materials available for anyone who wants to build a workstation with 2x Nvidia 3060 graphics cards, giving 24GB of usage RAM at a relatively low maximum power consumption of 170W. It's ideal for CI and end to end testing.

\n

If you'd like to go even more premium, the Nvidia RTX 4000 card comes with 20GB of RAM, so two of those would give you 40GB of RAM available for Large Language Models (LLMs).

\n

For Hetzner, you can get started with an i5 bare-metal host with 14 cores, 64GB RAM and a dedicated Nvidia RTX 4000 for around 184 EUR / mo (less than 200 USD / mo). If that sounds like ridiculously good value, it's because it is.

\n

What does the build look like?

\n

Once you've installed the actuated agent, it's the same process as a regular bare-metal host.

\n

It'll show up on your actuated dashboard, and you can start sending jobs to it immediately.

\n

\"The

\n
\n

The server with 2x GPUs showing up in the dashboard

\n
\n

Here's how we install the Nvidia driver for a consumer-grade card. The process is very similar for the datacenter range of GPUs found in enterprise servers.

\n
name: nvidia-smi\n\njobs:\n    nvidia-smi:\n        name: nvidia-smi\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - name: Download Nvidia install package\n          run: |\n           curl -s -S -L -O https://us.download.nvidia.com/XFree86/Linux-x86_64/525.60.11/NVIDIA-Linux-x86_64-525.60.11.run \\\n              && chmod +x ./NVIDIA-Linux-x86_64-525.60.11.run\n        - name: Install Nvidia driver and Kernel module\n          run: |\n              sudo ./NVIDIA-Linux-x86_64-525.60.11.run \\\n                  --accept-license \\\n                  --ui=none \\\n                  --no-questions \\\n                  --no-x-check \\\n                  --no-check-for-alternate-installs \\\n                  --no-nouveau-check\n        - name: Run nvidia-smi\n          run: |\n            nvidia-smi\n
\n

This is a very similar approach to installing a driver on your own machine, just without any interactive prompts. It took around 38s which is not very long considering how much time AI and ML operations can run for when doing end to end testing. The process installs some binaries like nvidia-smi and compiles a Kernel module to load the graphics driver, these could easily be cached with GitHub Action's built-in caching mechanism.

\n

For convenience, we created a composite action that reduces the duplication if you have lots of workflows with the Nvidia driver installed.

\n
name: gpu-job\n\njobs:\n    gpu-job:\n        name: gpu-job\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Run nvidia-smi\n          run: |\n            nvidia-smi\n
\n

Of course, if you have an AMD graphics card, or even an ML accelerator like a PCIe Google Corale, that can also be passed through into a VM in a dedicated way.

\n

The mechanism being used is called VFIO, and allows a VM to take full, dedicated, isolated control over a PCI device.

\n

A quick example to get started

\n

To show the difference between using a GPU and CPU, I ran OpenAI's Whisper project, which transcribes audio or video to a text file.

\n

With the following demo video of Actuated's SSH gateway, running with the tiny model.

\n\n

That's over 2x quicker, for a 5:34 minute video. If you process a lot of clips, or much longer clips then the difference may be even more marked.

\n

The tiny model is really designed for demos, and in production you'd use the medium or large model which is much more resource intensive.

\n

Here's a screenshot showing what this looks like with the medium model, which is much larger and more accurate:

\n

Medium model running on a GPU via actuated

\n
\n

Medium model running on a GPU via actuated

\n
\n

With a CPU, even with 16 vCPU, all of them get pinned at 100%, and then it takes a significantly longer time to process.

\n

\"You

\n
\n

You can run the medium model on CPU, but would you want to?

\n
\n

With the medium model:

\n\n

The GPU increased the speed by 9x, imagine how much quicker it'd be if you used an Nvidia 3090, 4090, or even an RTX 4000.

\n

If you want to just explore the system, and run commands interactively, you can use actuated's SSH feature to get a shell. Once you know the commands you want to run, you can copy them into your workflow YAML file for GitHub Actions.

\n

We took the SSH debug session for a test-drive. We installed the NVIDIA Container Toolkit, then ran the ollama tool to test out some Large Language Models (LLMs).

\n

Ollama is an open source tool for downloading and testing prepackaged models like Mistral or Llama2.

\n

\"Our

\n
\n

Our experiment with ollama within a GitHub Actions runner

\n
\n

The technical details

\n

Since launch, actuated powered by Firecracker has securely isolated over 220k CI jobs for GitHub Actions users. Whilst it's a complex project to integrate, it has been very reliable in production.

\n

Now in order to bring GPUs to actuated, we needed to add support for a second Virtual Machine Manager (VMM), and we picked cloud-hypervisor.

\n

cloud-hypervisor was originally a fork from Firecracker and shares a significant amount of code. One place it diverged was adding support for PCI devices, such as GPUs. Through VFIO, cloud-hypervisor allows for a GPU to be passed through to a VM in a dedicated way, so it can be used in isolation.

\n

Here's the first demo that I ran when we had everything working, showing the output from nvidia-smi:

\n

\"The

\n
\n

The first run of nvidia-smi

\n
\n

Reach out for more

\n

In a relatively short period of time, we were able to update our codebase to support both Firecracker and cloud-hypervisor, and to enable consumer-grade GPUs to be passed through to VMs in isolation.

\n

You can rent a really powerful and capable machine from Hetzner for under 200 USD / mo, or build your own workstation with dual graphics cards like our demo rig, for less than 2000 USD and then you own that and can use it as much as you want, plugged in under your desk or left in a cabinet in your office.

\n

A quick recap on use-cases

\n

Let's say you want to run end to end tests for an application that uses a GPU? Perhaps it runs on Kubernetes? You can do that.

\n

Do you want to fine-tune, train, or run a batch of inferences on a model? You can do that. GitHub Actions has a 6 hour timeout, which is plenty for many tasks.

\n

Would it make sense to run Stable Diffusion in the background, with different versions, different inputs, across a matrix? GitHub Actions makes that easy, and actuated can manage the GPU allocations for you.

\n

Do you run inference from OpenFaaS functions? We have a tutorial on OpenAI Whisper within a function with GPU acceleration here and a separate one on how to serve Server Sent Events (SSE) from OpenAI or self-hosted models, which is popular for chat-style interfaces to AI models.

\n

If you're interested in GPU support for GitHub Actions, then reach out to talk to us with this form.

","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/how-to-run-multi-arch-builds-natively.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/how-to-run-multi-arch-builds-natively.json deleted file mode 100644 index ad743e6f..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/how-to-run-multi-arch-builds-natively.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","contentHtml":"

In two previous articles, we covered huge improvements in performance for the Parca project and VPP (Network Service Mesh) simply by switching to actuated with Arm64 runners instead of using QEMU and hosted runners.

\n

In the first case, using QEMU took over 33 minutes, and bare-metal Arm showed a 22x improvement at only 1 minute 26 seconds. For Network Service Mesh, VPP couldn't even complete a build in 6 hours using QEMU - and I got it down to 9 minutes flat using a bare-metal Ampere Altra server.

\n

What are we going to see and why is it better?

\n

In this article, I'll show you how to run multi-arch builds natively on bare-metal hardware using GitHub Actions and actuated.

\n

Actuated is a SaaS service that we built so that you can Bring Your Own compute to GitHub Actions, and have every build run in an immutable, single-use VM.

\n

\"Comparison

\n
\n

Comparison of splitting out to run in parallel on native hardware and QEMU.

\n
\n

Not every build will see such a dramatic increase as the ones I mentioned in the introduction. Here, with the inlets-operator, we gained 4 minutes on each commit. But I often speak to users who are running past 30 minutes to over an hour because of QEMU.

\n

Three things got us a speed bump here:

\n\n

Only last week an engineer at Calyptia (the team behind fluent-bit) reached out for help after telling me they had to disable and stop publishing open source images for Arm, it was simply timing out at the 6 hour mark.

\n

So how does this thing work, and is QEMU actually \"OK\"?

\n

QEMU can be slow, but it's actually \"OK\"

\n

So if the timings are so bad, why does anyone use QEMU?

\n

Well it's free - as in beer, there's no cost at all to use it. And many builds can complete in a reasonable amount of time using QEMU, even if it's not as fast as native.

\n

That's why we wrote up how we build 80+ multi-arch images for various products like OpenFaaS and Inlets:

\n

The efficient way to publish multi-arch containers from GitHub Actions

\n

Here's what the build looks like with QEMU:

\n
name: split-operator\n\non:\n  push:\n    branches: [ master, qemu ]\n\njobs:\n\n  publish_qemu:\n    concurrency: \n      group: ${{ github.ref }}-qemu\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v2\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/amd64,linux/arm64\n          file: ./Dockerfile\n          context: .\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          provenance: false\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-qemu\n
\n

This is the kind of build that is failing or causing serious delays for projects like Parca, VPP and Fluent Bit.

\n

Let's look at the alternative.

\n

Building on native hardware

\n

Whilst QEMU emulates the architecture you need within a build, it's not the same as running on the real hardware. This is why we see such a big difference in performance.

\n

The downside is that we have to write a bit more CI configuration and run two builds instead of one, but there is some good news - we can now run them in parallel.

\n

In parallel we:

\n
    \n
  1. We publish the x86_64 image - ghcr.io/owner/repo:sha-amd64
    2. \n
  2. We publish the ARM image - ghcr.io/owner/repo:sha-arm64
    3. \n
\n

Then:

\n
    \n
  1. We create a manifest with its own name - ghcr.io/owner/repo:sha
    2. \n
  2. We annotate the manifest with the images we built earlier
    3. \n
  3. We push the manifest
    4. \n
\n

In this way, anyone can pull the image with the name ghcr.io/owner/repo:sha and it will map to either of the two images for Arm64 or Amd64.

\n

\"Parallel

\n
\n

The two builds on the left ran on two separate bare-metal hosts, and the manifest was published using one of GitHub's hosted runners.

\n
\n

Here's a sample for the inlets-operator, a Go binary which connects to the Kubernetes API.

\n

First up, we have the x86 build:

\n
name: split-operator\n\non:\n  push:\n    branches: [ master ]\n\njobs:\n\n  publish_x86:\n    concurrency: \n      group: ${{ github.ref }}-x86\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Setup mirror\n        uses: self-actuated/hub-mirror@master\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/amd64\n          file: ./Dockerfile\n          context: .\n          provenance: false\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64\n
\n

Then we have the arm64 build which is almost identical, but we specify a different value for platforms and the runs-on field.

\n
\n  publish_aarch64:\n    concurrency: \n      group: ${{ github.ref }}-aarch64\n      cancel-in-progress: true\n    permissions:\n      packages: write\n\n    runs-on: actuated-aarch64\n    steps:\n      - uses: actions/checkout@master\n        with:\n          repository: inlets/inlets-operator\n          path: \"./\"\n\n      - name: Setup mirror\n        uses: self-actuated/hub-mirror@master\n\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          platforms: linux/arm64\n          file: ./Dockerfile\n          context: .\n          provenance: false\n          build-args: |\n            Version=dev\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n
\n

Finally, we need to create the manifest. GitHub Actions has a needs variable that we can set to control the execution order:

\n
  publish_manifest:\n    runs-on: ubuntu-latest\n    needs: [publish_x86, publish_aarch64]\n    steps:\n\n    - name: Get Repo Owner\n      id: get_repo_owner\n      run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n    - name: Login to container Registry\n      uses: docker/login-action@v2\n      with:\n        username: ${{ github.repository_owner }}\n        password: ${{ secrets.GITHUB_TOKEN }}\n        registry: ghcr.io\n\n    - name: Create manifest\n      run: |\n        docker manifest create ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} \\\n          --amend ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64 \\\n          --amend ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n        docker manifest annotate --arch amd64 --os linux ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-amd64\n        docker manifest annotate --arch arm64 --os linux ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }} ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}-aarch64\n        docker manifest inspect ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n\n        docker manifest push ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n
\n

One thing I really dislike about this final stage is how much repetition we get. Fortunately, it's relatively simple to hide this complexity behind a custom GitHub Action.

\n

Note that this is just an example at the moment, but I could make a custom composite action in Bash in about 30 minutes, including testing. So it's not a lot of work and it would make our whole workflow a lot less repetitive.

\n
   uses: self-actuated/compile-manifest@master\n    with:\n      image: ghcr.io/${{ env.REPO_OWNER }}/inlets-operator\n      sha: ${{ github.sha }}\n      platforms: amd64,arm64\n
\n

As a final note, we recently saw that with upgrading from docker/build-push-action@v3 to docker/build-push-action@v4, buildx no longer publishes an image, but a manifest for each architecture. This is because a new \"provenance\" feature is enabled which under the hood is publishing multiple artifacts instead of a single image. We've turned this off with provenance: false and are awaiting a response from Docker on how to enable provenance for multi-arch images built with a split build.

\n

Wrapping up

\n

Yesterday we took a new customer on for actuated who wanted to improve the speed of Arm builds, but on the call we both knew they would need to leave QEMU behind. I put this write-up together to show what would be involved, and I hope it's useful to you.

\n

Where can you run these builds?

\n

Couldn't you just add a low-cost Arm VM from AWS, Oracle Cloud, Azure or Google Cloud?

\n

The answer unfortunately is no.

\n

The self-hosted runner is not suitable for open source / public repositories, the GitHub documentation has a stark warning about this.

\n

The Kubernetes controller that's available has the same issues, because it re-uses the Pods by default, and runs in a dangerous Docker In Docker Mode as a privileged container or by mounting the Docker Socket. I'm not sure which is worse, but both mean that code in CI can take over the host, potentially even the whole cluster.

\n

Hosted runners solve this by creating a fresh VM per job, and destroying it immediately. That's the same approach that we took with actuated, but you get to bring your own metal along, so that you keep costs from growing out of control. Actuated also supports Arm, out of the box.

\n

Want to know more about the security of self-hosted runners? Read more in our FAQ.

\n

Want to talk to us about your CI/CD needs? We're happy to help.

\n","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/is-the-self-hosted-runner-safe-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/is-the-self-hosted-runner-safe-github-actions.json deleted file mode 100644 index ce94093b..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/is-the-self-hosted-runner-safe-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","contentHtml":"

First of all, why would someone working on an open source project need a self-hosted runner?

\n

Having contributed to dozens of open source projects, and gotten to know many different maintainers, the primary reason tends to be out of necessity. They face an 18 minute build time upon every commit or Pull Request revision, and want to make the best of what little time they can give over to Open Source.

\n

Having faster builds also lowers friction for contributors, and since many contributors are unpaid and rely on their own internal drive and enthusiasm, a fast build time can be the difference between them fixing a broken test or waiting another few days.

\n

To sum up, there are probably just a few main reasons:

\n
    \n
  1. Faster builds, higher concurrency, more disk space
    2. \n
  2. Needing to build and test Arm binaries or containers on real hardware
    3. \n
  3. Access to services on private networks
    4. \n
\n

The first point is probably one most people can relate to. Simply by provisioning an AMD bare-metal host, or a high spec VM with NVMe, you can probably shave minutes off a build.

\n

For the second case, some projects like Flagger from the CNCF felt their only option to support users deploying to AWS Graviton, was to seek sponsorship for a large Arm server and to install a self-hosted runner on it.

\n

The third option is more nuanced, and specialist. This may or may not be something you can relate to, but it's worth mentioning. VPNs have very limited speed and there may be significant bandwidth costs to transfer data out of a region into GitHub's hosted runner environment. Self-hosted runners eliminate the cost and give full local link bandwidth, even as high as 10GbE. You just won't get anywhere near that with IPSec or Wireguard over the public Internet.

\n

Just a couple of days ago Ed Warnicke, Distinguished Engineer at Cisco reached out to us to pilot actuated. Why?

\n

Ed, who had Network Service Mesh in mind said:

\n
\n

I'd kill for proper Arm support. I'd love to be able to build our many containers for Arm natively, and run our KIND based testing on Arm natively.\nWe want to build for Arm - Arm builds is what brought us to actuated

\n
\n

But are self-hosted runners safe?

\n

The GitHub team has a stark warning for those of us who are tempted to deploy a self-hosted runner and to connect it to a public repository.

\n
\n

Untrusted workflows running on your self-hosted runner pose significant security risks for your machine and network environment, especially if your machine persists its environment between jobs. Some of the risks include:

\n\n
\n

See also: Self-hosted runner security

\n

Now you may be thinking \"I won't approve pull requests from bad actors\", but quite often the workflow goes this way: the contributor gets approval, then you don't need to approve subsequent pull requests after that.

\n

An additional risk is if that user's account is compromised, then the attacker can submit a pull request with malicious code or malware. There is no way in GitHub to enforce Multi-Factor Authentication (MFA) for pull requests, even if you have it enabled on your Open Source Organisation.

\n

Here are a few points to consider:

\n\n

The chances are that if you're running the Flagger or Network Service Mesh project, you are shipping code that enterprise companies will deploy in production with sensitive customer data.

\n

If you are not worried, try explaining the above to them, to see how they may see the risk differently.

\n

Doesn't Kubernetes fix all of this?

\n

Kubernetes is a well known platform built for orchestrating containers. It's especially suited to running microservices, webpages and APIs, but has support for batch-style workloads like CI runners too.

\n

You could make a container image and install the self-hosted runner binary within in, then deploy that as a Pod to a cluster. You could even scale it up with a few replicas.

\n

If you are only building Java code, Python or Node.js, you may find this resolves many of the issues that we covered above, but it's hard to scale, and you still get side-effects as the environment is not immutable.

\n

That's where the community project \"actions-runtime-controller\" or ARC comes in. It's a controller that launches a pool of Pods with the self-hosted runner.

\n
\n

How much work does ARC need?

\n

Some of the teams I have interviewed over the past 3 months told me that ARC took them a lot of time to set up and maintain, whilst others have told us it was a lot easier for them. It may depend on your use-case, and whether you're more of a personal user, or part of a team with 10-30 people committing code several times per day.\nThe first customer for actuated, which I'll mention later in the article was a team of ~ 20 people who were using ARC and had grew tired of the maintenance overhead and certain reliability issues.

\n
\n

Unfortunately, by default ARC uses the same Pod many times as a persistent runner, so side effects still build up, malware can still be introduced and you have to maintain a Docker image with all the software needed for your builds.

\n

You may be happy with those trade-offs, especially if you're only building private repositories.

\n

But those trade-offs gets a lot worse if you use Docker or Kubernetes.

\n

Out of the box, you simply cannot start a Docker container, build a container image or start a Kubernetes cluster.

\n

And to do so, you'll need to resort to what can only be described as dangerous hacks:

\n
    \n
  1. You expose the Docker socket from the host, and mount it into each Pod - any CI job can take over the host, game over.
    2. \n
  2. You run in Docker in Docker (DIND) mode. DIND requires a privileged Pod, which means that any CI job can take over the host, game over.
    3. \n
\n

There is some early work on running Docker In Docker in user-space mode, but this is slow, tricky to set up and complicated. By default, user-space mode uses a non-root account. So you can't install software packages or run commands like apt-get.

\n

See also: Using Docker-in-Docker for your CI or testing environment? Think twice.

\n

Have you heard of Kaniko?

\n

Kaniko is a tool for building container images from a Dockerfile, without the need for a Docker daemon. It's a great option, but it's not a replacement for running containers, it can only build them.

\n

And when it builds them, in nearly every situation it will need root access in order to mount each layer to build up the image.

\n

See also: The easiest way to prove that root inside the container is also root on the host

\n

And what about Kubernetes?

\n

To run a KinD, Minikube or K3s cluster within your CI job, you're going to have to sort to one of the dangerous hacks we mentioned earlier which mean a bad actor could potentially take over the host.

\n

Some of you may be running these Kubernetes Pods in your production cluster, whilst others have taken some due diligence and deployed a separate cluster just for these CI workloads. I think that's a slightly better option, but it's still not ideal and requires even more access control and maintenance.

\n

Ultimately, there is a fine line between overconfidence and negligence. When building code on a public repository, we have to assume that the worst case scenario will happen one day. When using DinD or privileged containers, we're simply making that day come sooner.

\n

Containers are great for running internal microservices and Kubernetes excels here, but there is a reason that AWS insists on hard multi-tenancy with Virtual Machines for their customers.

\n
\n

See also: Firecracker whitepaper

\n
\n

What's the alternative?

\n

When GitHub cautioned us against using self-hosted runners, on public repos, they also said:

\n
\n

This is not an issue with GitHub-hosted runners because each GitHub-hosted runner is always a clean isolated virtual machine, and it is destroyed at the end of the job execution.

\n
\n

So using GitHub's hosted runners are probably the most secure option for Open Source projects and for public repositories - if you are happy with the build speed, and don't need Arm runners.

\n

But that's why I'm writing this post, sometimes we need faster builds, or access to specialist hardware like Arm servers.

\n

The Kubernetes solution is fast, but it uses a Pod which runs many jobs, and in order to make it useful enough to run docker run, docker build or to start a Kubernetes cluster, we have to make our machines vulnerable.

\n

With actuated, we set out to re-build the same user experience as GitHub's hosted runners, but without the downsides of self-hosted runners or using Kubernetes Pods for runners.

\n

Actuated runs each build in a microVM on servers that you alone provision and control.

\n

Its centralised control-plane schedules microVMs to each server using an immutable Operating System that is re-built with automation and kept up to date with the latest security patches.

\n

Once the microVM has launched, it connects to GitHub, receives a job, runs to completion and is completely erased thereafter.

\n

You get all of the upsides of self-hosted runners, with a user experience that is as close to GitHub's hosted runners as possible.

\n

Pictured - an Arm Server with 270 GB of RAM and 80 cores - that's a lot of builds.

\n

\"\"

\n

You get to run the following, without worrying about security or side-effects:

\n\n

Need to test against a dozen different Kubernetes versions?

\n

Not a problem:

\n

\"Testing

\n

What about running the same on Arm servers?

\n

Just change runs-on: actuated to runs-on: actuated-aarch64 and you're good to go. We test and maintain support for Docker and Kubernetes for both Intel and Arm CPU architectures.

\n

Do you need insights for your Open Source Program Office (OSPO) or for the Technical Steering Committee (TSC)?

\n

\"\"

\n

We know that no open source project has a single repository that represents all of its activity. Actuated provides insights across an organisation, including total build time and the time queued - which is a reflection of whether you could do with more or fewer build machines.

\n

And we are only just getting started with compiling insights, there's a lot more to come.

\n

Get involved today

\n

We've already launched 10,000 VMs for customers jobs, and are now ready to open up the platform to the wider community. So if you'd like to try out what we're offering, we'd love to hear from you. As you offer feedback, you'll get hands on support from our engineering team and get to shape the product through collaboration.

\n

So what does it cost? There is a subscription fee which includes - the control plane for your organisation, the agent software, maintenance of the OS images and our support via Slack. But all the plans are flat-rate, so it may even work out cheaper than paying GitHub for the bigger instances that they offer.

\n

Professional Open Source developers like the ones you see at Red Hat, VMware, Google and IBM, that know how to work in community and understand cloud native are highly sought after and paid exceptionally well. So the open source project you work on has professional full-time engineers allocated to it by one or more companies, as is often the case, then using actuated could pay for itself in a short period of time.

\n

If you represent an open source project that has no funding and is purely maintained by volunteers, what we have to offer may not be suited to your current position. And in that case, we'd recommend you stick with the slower GitHub Runners. Who knows? Perhaps one day GitHub may offer sponsored faster runners at no cost for certain projects?

\n

And finally, what if your repositories are private? Well, we've made you aware of the trade-offs with a static self-hosted runner, or running builds within Kubernetes. It's up to you to decide what's best for your team, and your customers. Actuated works just as well with private repositories as it does with public ones.

\n

See microVMs launching in ~ 1s during a matrix build for testing a Custom Resource Definition (CRD) on different Kubernetes versions:

\n\n

Want to know how actuated works? Read the FAQ for more technical details.

\n\n

Follow us on Twitter - selfactuated

","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/kvm-in-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/kvm-in-github-actions.json deleted file mode 100644 index 5c82f171..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/kvm-in-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","contentHtml":"

GitHub's hosted runners do not support nested virtualization. This means some frequently used tools that require KVM like packer, the Android emulator, etc can not be used in GitHub Actions CI pipelines.

\n

We noticed there are quite a few issues for people requesting KVM support for GitHub Actions:

\n\n

As mentioned in some of these issues, an alternative would be to run your own self-hosted runner on a bare metal host. This comes with the downside that builds can conflict and cause side effects to system-level packages. On top if this self-hosted runners are considered insecure for public repositories.

\n

Solutions like the \"actions-runtime-controller\" or ARC that use Kubernetes to orchestrate and run self-hosted runners in Pods are also out of scope if you need to run VMs.

\n

With Actuated we make it possible to launch a Virtual Machine (VM) within a GitHub Action. Jobs are launched in isolated VMs just like GitHub hosted runners but with support for nested virtualization.

\n

Case Study: Githedgehog

\n

One of our customers Sergei Lukianov, founding engineer at Githedgehog told us he needed somewhere to build Docker images and to test them with Kubernetes, he uses KinD for that.

\n

Prior to adopting Actuated, his team used hosted runners which are considerably slower, and paid on a per minute basis. Actuated made his builds both faster, and more secure than using any of the alternatives for self-hosted runners.

\n

It turned out that he also needed to launch VMs in those jobs, and that's something else that hosted runners cannot cater for right now. Actuated’s KVM guest support means he can run all of his workloads on fast hardware.

\n

Some other common use cases that require KVM support on the CI runner:

\n\n

Running VMs in GitHub Actions

\n

In this section we will walk you through a couple of hands-on examples.

\n

Firecracker microVM

\n

In this example we are going to follow the Firecracker quickstart guide to boot up a Firecracker VM but instead of running it on our local machine we will run it from within a GitHub Actions workflow.

\n

The workflow instals Firecracker, configures and boots a guest VM and then waits 20 seconds before shutting down the VM and exiting the workflow. The image below shows the run logs of the workflow. We see the login prompt of the running microVM.

\n

\"Running

\n
\n

Running a firecracker microVM in a GitHub Actions job

\n
\n

Here is the workflow file used by this job:

\n
name: run-vm\n\non: push\njobs:\n  vm-run:\n    runs-on: actuated-4cpu-8gb\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: Install arkade\n        uses: alexellis/setup-arkade@v2\n      - name: Install firecracker\n        run: |\n          sudo arkade system install firecracker\n      - name: Run microVM\n        run: sudo -E ./run-vm.sh\n
\n

The setup-arkade is to install arkade on the runner. Next firecracker is installed from the arkade system apps.

\n

As a last step we run a firecracker microVM. The run-vm.sh script is based on the firecracker quickstart and collects all the steps into a single script that can be run in the CI pipeline.

\n

It script will:

\n\n

The run-vm.sh script:

\n
#!/bin/bash\n\n# Get a kernel and rootfs\narch=`uname -m`\ndest_kernel=\"hello-vmlinux.bin\"\ndest_rootfs=\"hello-rootfs.ext4\"\nimage_bucket_url=\"https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$arch\"\n\nif [ ${arch} = \"x86_64\" ]; then\n    kernel=\"${image_bucket_url}/kernels/vmlinux.bin\"\n    rootfs=\"${image_bucket_url}/rootfs/bionic.rootfs.ext4\"\nelif [ ${arch} = \"aarch64\" ]; then\n    kernel=\"${image_bucket_url}/kernels/vmlinux.bin\"\n    rootfs=\"${image_bucket_url}/rootfs/bionic.rootfs.ext4\"\nelse\n    echo \"Cannot run firecracker on $arch architecture!\"\n    exit 1\nfi\n\necho \"Downloading $kernel...\"\ncurl -fsSL -o $dest_kernel $kernel\n\necho \"Downloading $rootfs...\"\ncurl -fsSL -o $dest_rootfs $rootfs\n\necho \"Saved kernel file to $dest_kernel and root block device to $dest_rootfs.\"\n\n# Start firecracker\necho \"Starting firecracker\"\nfirecracker --api-sock /tmp/firecracker.socket &\nfirecracker_pid=$!\n\n# Set the guest kernel and rootfs\nrch=`uname -m`\nkernel_path=$(pwd)\"/hello-vmlinux.bin\"\n\nif [ ${arch} = \"x86_64\" ]; then\n    curl --unix-socket /tmp/firecracker.socket -i \\\n      -X PUT 'http://localhost/boot-source'   \\\n      -H 'Accept: application/json'           \\\n      -H 'Content-Type: application/json'     \\\n      -d \"{\n            \\\"kernel_image_path\\\": \\\"${kernel_path}\\\",\n            \\\"boot_args\\\": \\\"console=ttyS0 reboot=k panic=1 pci=off\\\"\n       }\"\nelif [ ${arch} = \"aarch64\" ]; then\n    curl --unix-socket /tmp/firecracker.socket -i \\\n      -X PUT 'http://localhost/boot-source'   \\\n      -H 'Accept: application/json'           \\\n      -H 'Content-Type: application/json'     \\\n      -d \"{\n            \\\"kernel_image_path\\\": \\\"${kernel_path}\\\",\n            \\\"boot_args\\\": \\\"keep_bootcon console=ttyS0 reboot=k panic=1 pci=off\\\"\n       }\"\nelse\n    echo \"Cannot run firecracker on $arch architecture!\"\n    exit 1\nfi\n\nrootfs_path=$(pwd)\"/hello-rootfs.ext4\"\ncurl --unix-socket /tmp/firecracker.socket -i \\\n  -X PUT 'http://localhost/drives/rootfs' \\\n  -H 'Accept: application/json'           \\\n  -H 'Content-Type: application/json'     \\\n  -d \"{\n        \\\"drive_id\\\": \\\"rootfs\\\",\n        \\\"path_on_host\\\": \\\"${rootfs_path}\\\",\n        \\\"is_root_device\\\": true,\n        \\\"is_read_only\\\": false\n   }\"\n\n# Start the guest machine\ncurl --unix-socket /tmp/firecracker.socket -i \\\n  -X PUT 'http://localhost/actions'       \\\n  -H  'Accept: application/json'          \\\n  -H  'Content-Type: application/json'    \\\n  -d '{\n      \"action_type\": \"InstanceStart\"\n   }'\n\n# Kill the firecracker process to exit the workflow\nsleep 20\nkill -9 $firecracker_pid\n\n
\n

The full example can be found on GitHub

\n

If you'd like to know more about how Firecracker works and how it compares to traditional VMs and Docker you can watch Alex's webinar on the topic.

\n\n
\n

Join Alex and Richard Case for a cracking time. The pair share what's got them so excited about Firecracker, the kinds of use-cases they see for microVMs, fundamentals of Linux Operating Systems and plenty of demos.

\n
\n

NixOS integration tests

\n

With nix there is the ability to provide a set of declarative configuration to define integration tests that spin up virtual machines using QEMU as the backend. While running these tests in CI without hardware acceleration is supported this is considerably slower.

\n

For a more detailed overview of the test setup and configuration see the original tutorial on nix.dev:

\n\n

The workflow file for running NixOS tests on GitHub Actions:

\n
name: nixos-tests\n\non: push\njobs:\n  nixos-test:\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - uses: actions/setup-python@v3\n        with:\n          python-version: '3.x'\n      - uses: cachix/install-nix-action@v16\n        with:\n          extra_nix_config: \"system-features = nixos-test benchmark big-parallel kvm\"\n      - name: NixOS test\n        run: nix build -L .#checks.x86_64-linux.postgres\n
\n

We just install Nix using the install-nix-action and run the tests in the next step.

\n

The full example is available on GitHub

\n

Other examples of using a VM

\n

In the previous section we showed you some brief examples for the kind of workflows you can run. Here are some other resources and tutorials that should be easy to adapt and run in CI.

\n\n

Conclusion

\n

Hosted runners do not support nested virtualization. That makes them unsuitable for running CI jobs that require KVM support.

\n

For Actuated runners we provide a custom Kernel that enables KVM support. This will allow you to run Virtual Machines within your CI jobs.

\n

At time of writing there is no support for aarch64 runners. Only Intel and AMD CPUs support nested virtualisation.

\n

While it is possible to deploy your own self-hosted runners to run jobs that need KVM support, this is not recommended:

\n\n

Want to see a demo or talk to our team? Contact us here

\n

Just want to try it out instead? Register your GitHub Organisation and set-up a subscription

","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/local-caching-for-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/local-caching-for-github-actions.json deleted file mode 100644 index 583ba808..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/local-caching-for-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","contentHtml":"

We heard from the Discourse project last year because they were looking to speed up their builds. After trying out a couple of solutions that automated self-hosted runners, they found out that whilst faster CPUs were nice, reliability was a problem and the cache hosted on GitHub's network became the new bottleneck. We ran some tests to compare the hosted cache with hosted runners, to self-hosted with a local cache running with S3. This post will cover what we found.

\n\"Discourse\n
\n

Discourse is the online home for your community. We offer a 100% open source community platform to those who want complete control over how and where their site is run.

\n
\n

Set up a local cache

\n

Hosted runners are placed close to the cache which means the latency is very low. Self-hosted runners can also make good use of this cache but the added latency can negate the advantage of switching to these faster runners. Running a local S3 cache with Minio or Seaweedfs on the self hosted runner or in the same region/network can solve this problem.

\n

For this test we ran the cache on the runner host. Instructions to set up a local S3 cache with Seaweedfs can be found in our docs.

\n

The Discourse repo is already using the actions/cachein their tests workflow which makes it easy to switch out the official actions/cache with tespkg/actions-cache.

\n

The S3 cache is not directly compatible with the official actions/cache and some changes to the workflows are required to start using the cache.

\n

The tespkg/actions-cache supports the same properties as the actions cache and only requires some additional parameters to configure the S3 connection.

\n
 - name: Bundler cache\n-  uses: actions/cache@v3\n+  uses: tespkg/actions-cache@v1\n   with:\n+    endpoint: \"192.168.128.1\"\n+    port: 8333\n+    insecure: true\n+    accessKey: ${{ secrets.ACTIONS_CACHE_ACCESS_KEY }}\n+    secretKey: ${{ secrets.ACTIONS_CACHE_SECRET_KEY }}\n+    bucket: actuated-runners\n+    region: local\n+    use-fallback: false\n\n     path: vendor/bundle\n     key: ${{ runner.os }}-${{ matrix.ruby }}-gem-${{ hashFiles('**/Gemfile.lock') }}-cachev2\n
\n

The endpoint could also be a HTTPS URL to a S3 server hosted within the same network as the self-hosted runners.

\n

If you are relying on the built-in cache support that is included in some actions like setup-node and setup-go you will need to add an additional caching step to your workflow as they are not directly compatible with the self-hosted S3 cache.

\n

The impact of switching to a local cache

\n

The Tests workflow from the Discourse repository was used to test the impact of switching to a local cache. We ran the workflow on a self-hosted Actuated runner, both with the S3 local cache and with the GitHub cache.

\n

Next we looked at the time required to restore the caches in our two environments and compared it with the times we saw on GitHub hosted runners:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Bundler cache
(±273MB)		Yarn cache
(±433MB)		Plugins gems cache
(±51MB)		App state cache
(±1MB)
Actuated with local cache5s11s1s0s
Actuated with hosted cache13s19s3s2s
Runner & cache hosted on GitHub6s11s3s2s
\n

While the GitHub runner and the self-hosted runner with a local cache perform very similarly, cache restores on the self-hosted runner that uses the GitHub cache take a bit longer.

\n

If we take a look at the yarn cache, which is the biggest cache, we can see that switching to the local S3 cache saved 8s for the cache size in this test vs using GitHub's cache from a self-hosted runner. This is a 42% improvement.

\n

Depending on your workflow and the cache size this can add up quickly. If a pipeline has multiple steps or when you are running matrix builds a cache step may need to run multiple times. In the case of the Discourse repo this cache step runs nine times which adds up to 1m12s that can be saved per workflow run.

\n

When Discourse approached us, we found that they had around a dozen jobs running for each pull request, all with varying sizes of caches. At busy times of the day, their global team could have 10 or more of those pull requests running, so these savings could add up to a significant amount.

\n

What if you also cached git checkout

\n

If your repository is a monorepo or has lots of large artifacts, you may get a speed boost caching the git checkout step too. Depending on where your runners are hosted, pulling from GitHub can take some time vs. restoring the same files from a local cache.

\n

We demonstrated what impact that had for Settlemint's CTO in this case study. They saw a cached checkout using a GitHub's hosted cache from from 2m40s to 11s.

\n

How we improved testpkg's custom action

\n

During our testing we noticed that every cache restore took a minimum of 10 seconds regardless of the cache size. It turned out to be an issue with timeouts in the tespkg/actions-cache action when listing objects in S3. We reported it and sent them a pull request with a fix.

\n

With the fix in place restoring small caches from the local cache dropped from 10s to sub 1s.

\n

The impact of switching to faster runners

\n

The Discourse repo uses the larger GitHub hosted runners to run tests. The jobs we are going to compare are part of the Tests workflow. They are using runners with 8 CPUs and 32GB of ram so we replaced the runs-on label with an actuated label actuated-8cpu-24gb to run the jobs on similar sized microVMs.

\n

All jobs ran on the same Hetzner AX102 bare metal host.

\n

This table compares the time it took to run each job on the hosted runner and on our Actuated runner.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
JobGitHub hosted runnerActuated runnerSpeedup
core annotations3m23s1m22s59%
core backend7m11s6m0s16%
plugins backend7m42s5m54s23%
plugins frontend5m28s4m3s26%
themes frontend4m20s2m46s36%
chat system9m37s6m33s32%
core system7m12s5m24s25%
plugin system5m32s3m56s29%
themes system4m32s2m4141%
\n

The first thing we notice is that all jobs completed faster on the Actuated runner. On average we see an improvement of around 1m40s seconds for each individual job.

\n

Conclusion

\n

While switching to faster self-hosted runners is the most obvious way to speed up your builds, the cache hosted on GitHub's network can become a new bottleneck if you use caching in your actions. After switching to a local S3 cache we saw a very significant improvement in the cache latency. Depending on how heavily the cache is used in your workflow and the size of your cache artifacts, switching to a local S3 cache might even have a bigger impact on build times.

\n

Both Seaweedfs and Minio were tested in our setup and they performed in a very similar way. Both have different open source licenses, so we'd recommend reading those before picking one or the other. Of course you could also use AWS S3, Google Cloud Storage, or another S3 compatible hosted service.

\n

In addition to the reduced latency, switching to a self hosted cache has a couple of other benefits.

\n\n

GitHub's caching action does not yet support using a custom S3 server, so we had to make some minor adjustments to the Discourse's workflow files. For this reason, if you use something like setup-go or setup-node, you won't be able to just set cache: true. Instead you'll need an independent caching step with the testpkg/actions-cache action.

\n

If you'd like to reach out to us and see if we can advise you on how to optmise your builds, you can set up a call with us here..

\n

If you want to learn more about caching for GitHub Actions checkout some of our other blog posts:

\n

You may also like:

\n","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/managing-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/managing-github-actions.json deleted file mode 100644 index 61e3b1a9..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/managing-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","contentHtml":"

Over the past few months, we've launched over 20,000 VMs for customers and have handled over 60,000 webhook messages from the GitHub API. We've learned a lot from every customer PoC and from our own usage in OpenFaaS.

\n

First of all - what is it we're offering? And how is it different to managed runners and the self-hosted runner that GitHub offers?

\n

Actuated replicates the hosted experience you get from paying for hosted runners, and brings it to hardware under your own control. That could be a bare-metal Arm server, or a regular Intel/AMD cloud VM that has nested virtualisation enabled.

\n

Just like managed runners - every time actuated starts up a runner, it's within a single-tenant virtual machine (VM), with an immutable filesystem.

\n

\"The

\n
\n

Asahi Linux running on my lab of two M1 Mac Minis - used for building the Arm64 base images and Kernels.

\n
\n

Can't you just use a self-hosted runner on a VM? Yes, of course you can. But it's actually more nuanced than that. The self-hosted runner isn't safe for OSS or public repos. And whether you run it directly on the host, or in Kubernetes - it's subject to side-effects, poor isolation, malware and in some cases uses very high privileges that could result in taking over a host completely.

\n

You can learn more in the actuated announcement and FAQ.

\n

how does it work?

\n

We run a SaaS - a managed control-plane which is installed onto your organisation as a GitHub App. At that point, we'll receive webhooks about jobs in a queued state.

\n

\"Conceptual

\n

As you can see in the diagram above, when a webhook is received, and we determine it's for your organisation, we'll schedule a Firecracker MicroVM on one of your servers.

\n

We have no access to your code or build secrets. We just obtain a registration token and send the runner a bit of metadata. Then we get out the way and let the self-hosted runner do its thing - in an isolated Kernel, with an immutable filesystem and its own Docker daemon.

\n

Onboarding doesn't take very long - you can use your own servers or get them from a cloud provider. We've got a detailed guide, but can also recommend an option on a discovery call.

\n

Want to learn more about how Firecracker compares to VMs and containers? Watch my webinar on YouTube

\n

Lesson 1 - GitHub's images are big and beautiful

\n

The first thing we noticed when building our actuated VM images was that the GitHub ones are huge.

\n

And if you've ever tried to find out how they're built, or hoped to find a nice little Dockerfile, you can may be disappointed. The images for Linux, Windows and MacOS are built through a set of bespoke scripts, and are hard to adapt for your own use.

\n

Don't get me wrong. The scripts are very clever and they work well. GitHub have been tuning these runner images for years, and they cover a variety of different use-cases.

\n

The first challenge for actuated before launching a pilot was getting enough of the most common packages installed through a Dockerfile. Most of our own internal software is built with Docker, so we can get by with quite a spartan environment.

\n

We also had to adapt the sample Kernel configuration provided by the Firecracker team so that it could launch Docker and so it had everything it required to launch Kubernetes.

\n

\"M1s

\n
\n

Two M1 Mac Minis running Asahi Linux and four separate versions of K3s

\n
\n

So by following the 80/20 principle, and focusing on the most common use-cases, we were able to launch quite quickly and cover 80% of the use-cases.

\n

I don't know if you realised, things like Node.js are pre-installed in the environment, but many Node developers also add the \"setup-node\" action which guess what? Downloads and installs Node.js again. The same is true for many other languages and tools. We do ship Node.js and Python in the image, but the chances are that we could probably remove them at some point.

\n

With one of our earliest pilots, a customer wanted to use a terraform action. It failed and I felt a bit embarrassed by the reason. We were missing unzip in our images.

\n

The cure? Go and add unzip to the Dockerfile, and hit publish on our builder repository. In 3 minutes the problem was solved.

\n

But GitHub Actions is also incredibly versatile and it means even if something is missing, we don't necessary have to publish a new image for you to continue your work. Just add a step to your workflow to install the missing package.

\n
- name: Add unzip\n  run: sudo apt-get install -qy unzip\n
\n

With every customer pilot we've done, there's tended to be one or two packages like this that they expected to see. For another customer it was \"libpq\". As a rule, if something is available in the hosted runner, we'll strongly consider adding it to ours.

\n

Lesson 2 - It’s not GitHub, it can’t be GitHub. It was GitHub

\n

Since actuated is a control-plane, a SaaS, a full-service - supported product, we are always asking first - is it us? Is it our code? Is it our infrastructure? Is it our network? Is it our hardware?

\n

If you open up the GitHub status page, you'll notice an outage almost every week - at times on consecutive days, or every few days on GitHub Actions or a service that affects them indirectly - like the package registry, Pages or Pull Requests.

\n

\"Outage

\n
\n

The second outage this week that unfortunately affected actuated customers.

\n
\n

I'm not bashing on GitHub here, we're paying a high R&D cost to build on their platform. We want them to do well.

\n

But this is getting embarrassing. On a recent call, a customer told us: \"it's not your solution, it looks great for us, it's the reliability of GitHub, we're worried about adopting it\"

\n

What can you say to that? I can't tell them that their concerns are misplaced, because they're not.

\n

I reached out to Martin Woodward - Director of DevRel at GitHub. He told me that \"leadership are taking this very seriously. We're doing better than we were 12 months ago.\"

\n

GitHub is too big to fail. Let's hope they smooth out these bumps.

\n

Lesson 3 - Webhooks can be unreliable

\n

There's no good API to collect this historical data at the moment but we do have an open-source tool (self-actuated/actions-usage) we give to customers to get a summary of their builds before they start out with us.

\n

So we mirror a summary of job events from GitHub into our database, so that we can show customers trends in behaviour, and identify hot-spots on specific repos - long build times, or spikes in failure rates.

\n

\"Insights

\n
\n

Insights chart from the actuated dashboard

\n
\n

We noticed that from time to time, jobs would show in our database as \"queued\" or \"in_progress\" and we couldn't work out why. A VM had been scheduled, the job had run, and completed.

\n

In some circumstances, GitHub forgot to send us an In Progress event, or they never sent us a queued event.

\n

Or they sent us queued, in progress, then completed, but in the reverse order.

\n

It took us longer than I'm comfortable with to track down this issue, but we've now adapted our API to handle these edge-cases.

\n

Some deeper digging showed that people have also had issues with Stripe webhooks coming out of order. We saw this issue only very recently, after handling 60k webhooks - so perhaps it was a change in the system being used at GitHub?

\n

Lesson 4 - Autoscaling is hard and the API is sometimes wrong

\n

We launch a VM on your servers for every time we receive a queued event. But we have no good way of saying that a particular VM can only run for a certain job.

\n

If there were five jobs queued up, then GitHub would send us five queued events, and we'd launch five VMs. But if the first job was cancelled, we'd still have all of those VMs running.

\n

Why? Why can't we delete the 5th?

\n

Because there is no determinism. It'd be a great improvement for user experience if we could tell GitHub's API - \"great, we see you queued build X, it must run on a runner with label Y\". But we can't do that today.

\n

So we developed a \"reaper\" - a background task that tracks launched VMs and can delete them after a period of inactivity. We did have an initial issue where GitHub was taking over a minute to send a job to a ready runner, which we fixed by increasing the idle timeout value. Right now it's working really well.

\n

There is still one remaining quirk where GitHub's API reports that an active runner where a job is running as idle. This happens surprisingly often - but it's not a big deal, the VM deletion call gets rejected by the GitHub API.

\n

Lesson 5 - We can launch in under one second, but what about GitHub?

\n

The way we have things tuned today, the delay from you hitting commit in GitHub, to the job executing is similar to that of hosted runners. But sometimes, GitHub lags a little - especially during an outage or when they're under heavy load.

\n

\"VM

\n
\n

Grafana Cloud showing a gauge of microVMs per managed host

\n
\n

There could be a delay between when you commit, and when GitHub delivers the \"queued\" webhook.

\n

Scoring and placing a VM on your servers is very quick, then the boot time of the microVM is generally less than 1 second including starting up a dedicated Docker daemon inside the VM.

\n

Then the runner has to run a configuration script to register itself on the API

\n

Finally, the runner connects to a queue, and GitHub has to send it a payload to start the job.

\n

On those last two steps - we see a high success rate, but occasionally, GitHub's API will fail on either of those two operations. We receive an alert via Grafana Cloud and Discord - then investigate. In the worst case, we re-queue via our API the job and the new VM will pick up the pending job.

\n

Want to watch a demo?

\n\n

Lesson 6 - Sometimes we need to debug a runner

\n

When I announced actuated, I heard a lot of people asking for CircleCI's debug experience, so I built something similar and it's proved to be really useful for us in building actuated.

\n

Only yesterday, Ivan Subotic from Dasch Swiss messaged me and said:

\n
\n

\"How cool!!! you don’t know how many hours I have lost on GitHub Actions without this.\"

\n
\n

Recently there were two cases where we needed to debug a runner with an SSH shell.

\n

The first was for a machine on Hetzner, where the Docker Daemon was unable to pull images due to a DNS failure. I added steps to print out /etc/resolv.conf and that would be my first port of call. Debugging is great, but it's slow, if an extra step in the workflow can help us diagnose the problem, it's worth it.

\n

In the end, it took me about a day and a half to work out that Hetzner was blocking outgoing traffic on port 53 to Google and Cloudflare. What was worse - was that it was an intermittent problem.

\n

When we did other customer PoCs on Hetzner, we did not run into this issue. I even launched a \"cloud\" VM in the same region and performed a couple of nslookups - they worked as expected for me.

\n

So I developed a custom GitHub Action to unblock the customer:

\n
steps:\n    - uses: self-actuated/hetzner-dns-action@v1\n
\n

Was this environmental issue with Hetzner our responsibility? Arguably not, but our customers pay us to provide a \"like managed\" solution, and we are currently able to help them be successful.

\n

In the second case, Ivan needed to launch headless Chrome, and was using one of the many setup-X actions from the marketplace.

\n

I opened a debug session on one of our own runners, then worked backwards:

\n
curl -sLS -O https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n\ndpkg -i ./google-chrome-stable_current_amd64.deb\n
\n

This reported that some packages were missing, I got which packages by running apt-get --fix-missing --no-recommends and provided an example of how to add them.

\n
jobs:\n    chrome:\n        name: chrome\n        runs-on: actuated\n        steps:\n        - name: Add extra packages for Chrome\n          run: |\n            sudo apt install -qyyy --no-install-recommends adwaita-icon-theme fontconfig fontconfig-config fonts-liberation gtk-update-icon-cache hicolor-icon-theme humanity-icon-theme libatk-bridge2.0-0 libatk1.0-0 libatk1.0-data libatspi2.0-0 ...\n        - uses: browser-actions/setup-chrome@v1\n        - run: chrome --version\n
\n

We could also add these to the base image by editing the Dockerfile that we maintain.

\n

Lesson 7 - Docker Hub Rate Limits are a pain

\n

Docker Hub rate limits are more of a pain on self-hosted runners than they are on GitHub's own runners.

\n

I ran into this problem whilst trying to rebuild around 20 OpenFaaS Pro repositories to upgrade a base image. So after a very short period of time, all code ground to a halt and every build failed.

\n

GitHub has a deal to pay Docker Inc so that you don't run into rate limits. At time of writing, you'll find a valid Docker Hub credential in the $HOME/.docker/config.json file on any hosted runner.

\n

Actuated customers would need to login at the top of every one of their builds that used Docker, and create an organisation-level secret with a pull token from the Docker Hub.

\n

We found a way to automate this, and speed up subsequent jobs by caching images directly on the customer's server.

\n

All they need to add to their builds is:

\n
- uses: self-actuated/hub-mirror@master\n
\n

Wrapping up

\n

I hope that you've enjoyed hearing a bit about our journey so far. With every new pilot customer we learn something new, and improve the offering.

\n

Whilst there was a significant amount of very technical work at the beginning of actuated, most of our time now is spent on customer support, education, and improving the onboarding experience.

\n

If you'd like to know how actuated compares to hosted runners or managing the self-hosted runner on your own, we'd encourage checking out the blog and FAQ.

\n

Are your builds slowing the team down? Do you need better organisation-level insights and reporting? Or do you need Arm support? Are you frustrated with managing self-hosted runners?

\n

Reach out to us take part in the pilot

","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/millions-of-cncf-minutes.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/millions-of-cncf-minutes.json deleted file mode 100644 index 162be2c7..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/millions-of-cncf-minutes.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","contentHtml":"

In this post I'll cover:

\n\n

What's Actuated?

\n
\n

Actuated is the only solution that gives a managed experience for self-hosted runners, on your own hardware through immutable microVMs.

\n
\n

The immutability is key to both security and reliability. Each build is run in its own ephemeral and immutable microVM, meaning side effects cannot be left behind by prior builds. In addition, our team manages reliability for your servers and the integration with GitHub, for minimal disruption during GitHub outages.

\n

Why did the CNCF want Actuated for its projects?

\n

We've been working with Ampere and Equinix Metal to provide CI via GitHub Actions for Cloud Native Computing (CNCF) projects. Ampere manufacture Arm-based CPUs with a focus on efficiency and high core density. Equinix Metal provide access to the Ampere Altra in their datacenters around the world.

\n

Last December, we met with Chris Aniszczyck - CTO Linux Foundation/CNCF, Ed Vielmetti - Open Source Manager Equinix, Dave Neary - Director of Developer Relations at Ampere and myself to discuss the program and what impact it was having so far.

\n

Watch the recap on YouTube: The Ampere Developer Impact: CNCF Pilot Discussion

\n

Past articles on the blog include:

\n\n

Before and after the program

\n

Before we started the program, CNCF projects could be divided into three buckets:

\n
    \n
  1. They were running Arm-based CI jobs on static servers that they managed
    2. \n
\n

In this case, etcd for instance had a team of half a dozen maintainers who were responsible for setting up, maintaining, and upgrading statically provisioned CI servers for the project. This was a significant overhead for the project maintainers, and the servers were often underutilized. The risk of side-effects being left behind between builds also posed a serious supply chain risk since etcd is consumed in virtually every Kubernetes deployment.

\n
    \n
  1. They were running Arm-based CI using emulation (QEMU)
    2. \n
\n

QEMU can be combined with Docker's buildx for a quick and convenient way to build container images for x86 and Arm architectures. In the best case, it's a small change and may add a few minutes of extra overhead. In the worst case, we saw that jobs that ran in ~ 5 minutes, took over 6 hours to complete using QEMU and hosted runners. A prime example was fluentd, read their case-study here: Scaling ARM builds with Actuated

\n
    \n
  1. They had no Arm CI at all
    2. \n
\n

In the third case, we saw projects like OpenTelemetry which had no support for Arm at all, but demand from their community to bring it up to on par with x86 builds. The need to self-manage insecure CI servers meant that Arm was a blocker for them.

\n

After the program

\n

After the program was live, teams who had been maintaining their own servers got to remove lengthy documentation on server configuration and maintenance, and relied on our team to manage a pool of servers used for scheduling microVMs.

\n

As demand grew, we saw OpenTelemetry and etcd starve the shared pool of resources through very high usage patterns. This is a classic and known problem called \"Tragedy of the Commons\" - when a shared resource is overused by a subset of users, it can lead to a degradation of service for all users. To combat the problem, we added code to provision self-destructing servers for a period of 24-48 hours as need arose, and prevented the higher usage projects from running on at least on of the permanent servers through scheduling rules. One other issue we saw with OpenTelemetry in particular was that the various Go proxies that offer up Go modules appeared to be rejecting requests when too many jobs were running concurrently. As a workaround, we added a private Go proxy for them into the private network space where the CNCF servers run, this also massively reduced the bandwidth costs for the shared infrastructure.

\n

Teams like fluent moved from flakey builds that couldn't finish in 6 hours, to builds that finished in 5-10 minutes. This meant they could expand on their suite of tests.

\n

Where teams such as Cilium, Falco, or OpenTelemetry had no Arm CI support, we saw them quickly ramp up to running thousands of builds per month.

\n

Here's a quote from Federico Di Pierro, Senior Open Source Engineer @ Sysdig and maintainer of Falco:

\n
\n

Falco really needed arm64 GitHub runners to elevate its support for the architecture and enlarge its userbase.\nActuated was the perfect solution for us because it was easy to leverage and relieved any burden for the maintainers. This way, we as maintainers, can focus on what really matters for the project, instead of fighting with maintaining and deploying self-hosted infrastructure.\nNow we are building, testing and releasing artifacts for arm64 leveraging Actuated for many of our projects, and it works flawlessly.\nSupport from Alex's team is always on point, and new kernel features are coming through super quickly!

\n
\n

Akihiro Suda, Software Engineer at NTT Corp, and maintainer of several open source projects including: runc, containerd and lima had this to say:

\n
\n

Huge thanks to Actuated for enabling us to run ARM64 tests without any mess.\nIt is very important for the runc project to run the tests on ARM64, as runc depends on several architecture-dependent components such as seccomp and criu.\nIt is also so nice that the Arm instance specification can be adjusted in a single line in the GitHub Actions workflow file.

\n
\n

Wei Fu, a maintainer for containerd said:

\n
\n

The containerd project was able to test each pull request for the Linux arm64 platform with the support of Actuated.\nIt's a significant step for containerd to mark the Linux arm64 platform as a top-tier supported platform, similar to amd64, since containerd has been widely used in the Arm world.

\n

Thanks to Actuated, we, the containerd community, were able to test container features (like mount-idmapping) on the new kernel without significant maintenance overhead for the test infrastructure.\nWith Actuated, we can focus on open-source deployment to cover more use case scenarios.

\n
\n

Maintainers have direct access to discuss issues and improvements with us via a private Slack community. One of the things we've done in addition to adding burst capacity to the pool, was to provide a tool to help teams right-size VMs for jobs and to add support for eBPF technologies like BTF in the Kernel.

\n

Numbers at a glance

\n

In our last update, 3 months ago, we'd run just under 400k build minutes for the CNCF. That number has now increased to 1.52M minutes, which is a ~ 300x increase in demand in a short period of time.

\n

Here's a breakdown of the top 9 projects by total minutes run, bearing in mind that this only includes jobs that ran to completion, there are thousands of minutes which ran, but were cancelled mid-way or by automation.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationTotal minsTotal JobsFirst job
1open-telemetry593726400932024-02-15
2etcd-io372080213472023-10-24
3cri-o163927111312023-11-27
4falcosecurity138469132742023-12-06
5fluent89856106582023-06-07
6containerd87007111922023-12-02
7cilium7340662522023-10-31
8opencontainers37164582023-12-15
9argoproj187122024-01-30
(all)(Total)1520464116217
\n

Most organisations build for several projects or repositories. In the case of etcd, the numbers also include the boltdb project, and for cilium, tetragon, and the Go bindings for ebpf are also included. Open Telemetry is mainly focused around the collectors and SDKs.

\n

runc which is within the opencontainers organisation is technically an Open Container Initiative (OCI) project under the LinuxFoundation, rather than a CNCF project, but we gave them access since it is a key dependency for containerd and cri-o.

\n

What's next?

\n

With the exception of Argo, all of the projects are now relatively heavy users of the platform, with demand growing month on month, as you can see from the uptick from 389k minutes in March to a record high of 1.52 million minutes by the end of May of the same year. In the case of Argo, if you're a contributor or have done previous open source enablement, perhaps you could help them expand their Arm support via a series of Pull Requests to enable unit/e2e tests to run on Arm64?

\n

We're continuing to improve the platform to support users during peak demand, outages on GitHub, and to provide a reliable way for CNCF projects to run their CI on real Arm hardware, at full speed.

\n

For instance, last month we just released a new 6.1 Kernel for the Ampere Altra, which means projects like Cilium and Falco can make use of new eBPF features introduced in recent Kernel versions, and will bring support for newer Kernels as the Firecracker team make them available. The runc and container teams also benefit from the newer Kernel and have been able to enable further tests for (Checkpoint/Restore In Userspace) CRIU and User namespaces for containerd.

\n

You can watch the interview I mentioned earlier with Chris, Ed, Dave and myself on YouTube:

\n\n

We could help you too

\n

Actuated can manage x86 and Arm64 servers for GitHub Actions and self-managed GitLab CI. If you'd like to speak to us about how we can speed up your jobs, reduce your maintenance efforts and lower your CI costs, reach out via this page.

","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/multi-arch-docker-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/multi-arch-docker-github-actions.json deleted file mode 100644 index 4a2bc7ea..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/multi-arch-docker-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","contentHtml":"

In 2017, I wrote an article on multi-stage builds with Docker, and it's now part of the Docker Documentation. In my opinion, multi-arch builds were the proceeding step in the evolution of container images.

\n

What's multi-arch and why should you care?

\n

If you want users to be able to use your containers on different types of computer, then you'll often need to build different versions of your binaries and containers.

\n

The faas-cli tool is how users interact with OpenFaaS.

\n

It's distributed in binary format for users, with builds for Windows, MacOS and Linux.

\n\n

But why are there six different binaries for three Operating Systems? With the advent of Raspberry Pi, M1 Macs (Apple Silicon) and AWS Graviton servers, we have had to start building binaries for more than just Intel systems.

\n

If you're curious how to build multi-arch binaries with Go, you can check out the release process for the open source arkade tool here, which is a simpler example than faas-cli: arkade Makefile and GitHub Actions publish job

\n

So if we have to support at least six different binaries for Open Source CLIs, what about container images?

\n

Do we need multi-arch containers too?

\n

Until recently, it was common to hear people say: \"I can't find any containers that work for Arm\". This was because the majority of container images were built only for Intel. Docker Inc has done a sterling job of making their \"official\" images work on different platforms, that's why you can now run docker run -t -i ubuntu /bin/bash on a Raspberry Pi, M1 Mac and your regular PC.

\n

Many open source projects have also caught on to the need for multi-arch images, but there are still a few like Bitnami, haven't yet seen value. I think that is OK, this kind work does take time and effort. Ultimately, it's up to the project maintainers to listen to their users and decide if they have enough interest to add support for Arm.

\n

A multi-arch image is a container that will work on two or more different combinations of operating system and CPU architecture.

\n

Typically, this would be:

\n\n

So multi-arch is really about catering for the needs of Arm users. Arm hardware platforms like the Ampere Altra come with 80 efficient CPU cores, have a very low TDP compared to traditional Intel hardware, and are available from various cloud providers.

\n

How do we build multi-arch containers work?

\n

There are a few tools and tricks that we can combine together to take a single Dockerfile and output an image that anyone can pull, which will be right for their machine.

\n

Let's take the: ghcr.io/inlets-operator:latest image from inlets.

\n

When a user types in docker pull, or deploys a Pod to Kubernetes, their local containerd daemon will fetch the manifest file and inspect it to see what SHA reference to use for to download the required layers for the image.

\n

\"How

\n
\n

How manifests work

\n
\n

Let's look at a manifest file with the crane tool. I'm going to use arkade to install crane:

\n
arkade get crane\n\ncrane manifest ghcr.io/inlets/inlets-operator:latest\n
\n

You'll see a manifests array, with a platform section for each image:

\n
{\n  \"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\",\n  \"manifests\": [\n    {\n      \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n      \"digest\": \"sha256:bae8025e080d05f1db0e337daae54016ada179152e44613bf3f8c4243ad939df\",\n      \"platform\": {\n        \"architecture\": \"amd64\",\n        \"os\": \"linux\"\n      }\n    },\n    {\n      \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n      \"digest\": \"sha256:3ddc045e2655f06653fc36ac88d1d85e0f077c111a3d1abf01d05e6bbc79c89f\",\n      \"platform\": {\n        \"architecture\": \"arm64\",\n        \"os\": \"linux\"\n      }\n    }\n  ]\n}\n
\n

How do we convert a Dockerfile to multi-arch?

\n

Instead of using the classic version of Docker, we can enable the buildx and Buildkit plugins which provide a way to build multi-arch images.

\n

We'll continue with the Dockerfile from the open source inlets-operator project.

\n

Within the Dockerfile, we need to make a couple of changes.

\n
- FROM golang:1.18 as builder\n+ FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.18 as builder\n\n+ ARG TARGETPLATFORM\n+ ARG BUILDPLATFORM\n+ ARG TARGETOS\n+ ARG TARGETARCH\n
\n

The BUILDPLATFORM variable is the native architecture and platform of the machine performing the build, this is usually amd64.

\n

The TARGETPLATFORM is important for the final step of the build, and will normally be injected based upon one each of the platforms you have specified for the build command.

\n

For Go specifically, we also updated the go build command to tell Go to use cross-compilation based upon the TARGETOS and TARGETARCH environment variables, which are populated by Docker.

\n
- go build -o inlets-operator\n+ GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o inlets-operator\n
\n

Here's the full example:

\n
FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.18 as builder\n\nARG TARGETPLATFORM\nARG BUILDPLATFORM\nARG TARGETOS\nARG TARGETARCH\n\nARG Version\nARG GitCommit\n\nENV CGO_ENABLED=0\nENV GO111MODULE=on\n\nWORKDIR /go/src/github.com/inlets/inlets-operator\n\n# Cache the download before continuing\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY .  .\n\nRUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \\\n  go test -v ./...\n\nRUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \\\n  go build -ldflags \"-s -w -X github.com/inlets/inlets-operator/pkg/version.Release=${Version} -X github.com/inlets/inlets-operator/pkg/version.SHA=${GitCommit}\" \\\n  -a -installsuffix cgo -o /usr/bin/inlets-operator .\n\nFROM --platform=${BUILDPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot\n\nLABEL org.opencontainers.image.source=https://github.com/inlets/inlets-operator\n\nWORKDIR /\nCOPY --from=builder /usr/bin/inlets-operator /\nUSER nonroot:nonroot\n\nCMD [\"/inlets-operator\"]\n
\n

How to do you configure GitHub Actions to publish multi-arch images?

\n

Now that the Dockerfile has been configured, it's time to start working on the GitHub Action.

\n

This example is taken from the Open Source inlets-operator. It builds a container image containing a Go binary and uses a Dockerfile in the root of the repository.

\n

View publish.yaml, adapted for actuated:

\n
name: publish\n\non:\n  push:\n    tags:\n      - '*'\n\njobs:\n  publish:\n+    permissions:\n+      packages: write\n\n-   runs-on: ubuntu-latest\n+   runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n\n+     - name: Setup mirror\n+       uses: self-actuated/hub-mirror@master\n      - name: Get TAG\n        id: get_tag\n        run: echo TAG=${GITHUB_REF#refs/tags/} >> $GITHUB_ENV\n      - name: Get Repo Owner\n        id: get_repo_owner\n        run: echo \"REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')\" > $GITHUB_ENV\n\n+     - name: Set up QEMU\n+       uses: docker/setup-qemu-action@v2\n+     - name: Set up Docker Buildx\n+       uses: docker/setup-buildx-action@v2\n      - name: Login to container Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Release build\n        id: release_build\n        uses: docker/build-push-action@v4\n        with:\n          outputs: \"type=registry,push=true\"\n          provenance: false\n+         platforms: linux/amd64,linux/arm/v6,linux/arm64\n          build-args: |\n            Version=${{  env.TAG }}\n            GitCommit=${{ github.sha }}\n          tags: |\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ github.sha }}\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:${{ env.TAG }}\n            ghcr.io/${{ env.REPO_OWNER }}/inlets-operator:latest\n
\n

All of the images and corresponding manifest are published to GitHub's Container Registry (GHCR). The action itself is able to authenticate to GHCR using a built-in, short-lived token. This is dependent on the \"permissions\" section and \"packages: write\" being set.

\n

You'll see that we added a Setup mirror step, this explained in the Registry Mirror example and is not required for Hosted Runners.

\n

The docker/setup-qemu-action@v2 step is responsible for setting up QEMU, which is used to emulate the different CPU architectures.

\n

The docker/build-push-action@v4 step is responsible for passing in a number of platform combinations such as: linux/amd64 for cloud, linux/arm64 for Arm servers and linux/arm/v6 for Raspberry Pi.

\n

What if you're not using GitHub Actions?

\n

The various GitHub Actions published by the Docker team are a great way to get started, but if you look under the hood, they're just syntactic sugar for the Docker CLI.

\n
export DOCKER_CLI_EXPERIMENTAL=enabled\n\n# Have Docker download the latest buildx plugin\ndocker buildx install\n\n# Create a buildkit daemon with the name \"multiarch\"\ndocker buildx create \\\n    --use \\\n    --name=multiarch \\\n    --node=multiarch\n\n# Install QEMU\ndocker run --rm --privileged \\\n    multiarch/qemu-user-static --reset -p yes\n\n# Run a build for the different platforms\ndocker buildx build \\\n    --platform=linux/arm64,linux/amd64 \\\n    --output=type=registry,push=true --tag image:tag .\n
\n

For OpenFaaS users, we do all of the above any time you type in faas-cli publish and the faas-cli build command just runs a regular Docker build, without any of the multi-arch steps.

\n

If you're interested, you can checkout the code here: publish.go.

\n

Putting it all together

\n\n

In our experience with OpenFaaS, inlets and actuated, once you have converted one or two projects to build multi-arch images, it becomes a lot easier to do it again, and make all software available for Arm servers.

\n

You can learn more about Multi-platform images in the Docker Documentation.

\n

Want more multi-arch examples?

\n

OpenFaaS uses multi-arch Dockerfiles for all of its templates, and the examples are freely available on GitHub including Python, Node, Java and Go.

\n

See also: OpenFaaS templates

\n

A word of caution

\n

QEMU can be incredibly slow at times when using a hosted runner, where a build takes takes 1-2 minutes can extend to over half an hour. If you do run into that, one option is to check out actuated or another solution, which can build directly on an Arm server with a securely isolated Virtual Machine.

\n

In How to make GitHub Actions 22x faster with bare-metal Arm, we showed how we decreased the build time of an open-source Go project from 30.5 mins to 1.5 mins. If this is the direction you go in, you can use a matrix-build instead of a QEMU-based multi-arch build.

\n

See also: Recommended bare-metal Arm servers

","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/native-arm64-for-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/native-arm64-for-github-actions.json deleted file mode 100644 index d74cc275..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/native-arm64-for-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","contentHtml":"

GitHub Actions is a modern, fast and efficient way to build and test software, with free runners available. We use the free runners for various open source projects and are generally very pleased with them, after all, who can argue with good enough and free? But one of the main caveats is that GitHub's hosted runners don't yet support the Arm architecture.

\n

So many people turn to software-based emulation using QEMU. QEMU is tricky to set up, and requires specific code and tricks if you want to use software in a standard way, without modifying it. But QEMU is great when it runs with hardware acceleration. Unfortunately, the hosted runners on GitHub do not have KVM available, so builds tend to be incredibly slow, and I mean so slow that it's going to distract you and your team from your work.

\n

This was even more evident when Frederic Branczyk tweeted about his experience with QEMU on GitHub Actions for his open source observability project named Parca.

\n

Does anyone have a @github actions self-hosted runner manifest for me to throw at a @kubernetesio cluster? I'm tired of waiting for emulated arm64 CI runs taking ages.

— Frederic 🧊 Branczyk @brancz@hachyderm.io (@fredbrancz) October 19, 2022
\n

I checked out his build and expected \"ages\" to mean 3 minutes, in fact, it meant 33.5 minutes. I know because I forked his project and ran a test build.

\n

After migrating it to actuated and one of our build agents, the time dropped to 1 minute and 26 seconds, a 22x improvement for zero effort.

\n

This morning @fredbrancz said that his ARM64 build was taking 33 minutes using QEMU in a GitHub Action and a hosted runner.

I ran it on @selfactuated using an ARM64 machine and a microVM.

That took the time down to 1m 26s!! About a 22x speed increase. https://t.co/zwF3j08vEV pic.twitter.com/ps21An7B9B

— Alex Ellis (@alexellisuk) October 20, 2022
\n

You can see the results here:

\n

\"Results

\n

As a general rule, the download speed is going to be roughly the same with a hosted runner, it may even be slightly faster due to the connection speed of Azure's network.

\n

But the compilation times speak for themselves - in the Parca build, go test was being run with QEMU. Moving it to run on the ARM64 host directly, resulted in the marked increase in speed. In fact, the team had introduced lots of complicated code to try and set up a Docker container to use QEMU, all that could be stripped out, replacing it with a very standard looking test step:

\n
  - name: Run the go tests\n    run: go test ./...\n
\n

Can't I just install the self-hosted runner on an Arm VM?

\n

There are relatively cheap Arm VMs available from Oracle OCI, Google and Azure based upon the Ampere Altra CPU. AWS have their own Arm VMs available in the Graviton line.

\n

So why shouldn't you just go ahead and install the runner and add them to your repos?

\n

The moment you do that you run into three issues:

\n\n

Chasing your tail with package updates, faulty builds due to caching and conflicts is not fun, you may feel like you're saving money, but you are paying with your time and if you have a team, you're paying with their time too.

\n

Most importantly, GitHub say that it cannot be used safely with a public repository. There's no security isolation, and state can be left over from one build to the next, including harmful code left intentionally by bad actors, or accidentally from malware.

\n

So how do we get to a safer, more efficient Arm runner?

\n

The answer is to get us as close as possible to a hosted runner, but with the benefits of a self-hosted runner.

\n

That's where actuated comes in.

\n

We run a SaaS that manages bare-metal for you, and talks to GitHub upon your behalf to schedule jobs efficiently.

\n\n

microVMs on Arm require a bare-metal server, and we have tested all the options available to us. Note that the Arm VMs discussed above do not currently support KVM or nested virtualisation.

\n\n

If you're already an AWS customer, the a1.metal is a good place to start. If you need expert support, networking and a high speed uplink, you can't beat Equinix Metal (we have access to hardware there and can help you get started) - you can even pay per minute and provision machines via API. The Mac Mini <1 has a really fast NVMe and we're running one of these with Asahi Linux for our own Kernel builds for actuated. The RX Line from Hetzner has serious power and is really quite affordable, but just be aware that you're limited to a 1Gbps connection, a setup fee and monthly commitment, unless you pay significantly more.

\n

I even tried Frederic's Parca job on my 8GB Raspberry Pi with a USB NVMe. Why even bother, do I hear you say? Well for a one-time payment of 80 USD, it was 26m30s quicker than a hosted runner with QEMU!

\n

Learn how to connect an NVMe over USB-C to your Raspberry Pi 4

\n

What does an Arm job look like?

\n

Since I first started trying to build code for Arm in 2015, I noticed a group of people who had a passion for this efficient CPU and platform. They would show up on GitHub issue trackers, ready to send patches, get access to hardware and test out new features on Arm chips. It was a tough time, and we should all be grateful for their efforts which go largely unrecognised.

\n
\n

If you're looking to make your software compatible with Arm, feel free to reach out to me via Twitter.

\n
\n

In 2020 when Apple released their M1 chip, Arm went mainstream, and projects that had been putting off Arm support like KinD and Minikube, finally had that extra push to get it done.

\n

I've had several calls with teams who use Docker on their M1/M2 Macs exclusively, meaning they build only Arm binaries and use only Arm images from the Docker Hub. Some of them even ship to project using Arm images, but I think we're still a little behind the curve there.

\n

That means Kubernetes - KinD/Minikube/K3s and Docker - including Buildkit, compose etc, all work out of the box.

\n

I'm going to use the arkade CLI to download KinD and kubectl, however you can absolutely find the download links and do all this manually. I don't recommend it!

\n
name: e2e-kind-test\n\non: push\njobs:\n  start-kind:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: get arkade\n        uses: alexellis/setup-arkade@v1\n      - name: get kubectl and kubectl\n        uses: alexellis/arkade-get@master\n        with:\n          kubectl: latest\n          kind: latest\n      - name: Create a KinD cluster\n        run: |\n          mkdir -p $HOME/.kube/\n          kind create cluster --wait 300s\n      - name: Wait until CoreDNS is ready\n        run: |\n          kubectl rollout status deploy/coredns -n kube-system --timeout=300s\n      - name: Explore nodes\n        run: kubectl get nodes -o wide\n      - name: Explore pods\n        run: kubectl get pod -A -o wide\n
\n

That's our x86_64 build, or Intel/AMD build that will run on a hosted runner, but will be kind of slow.

\n

Let's convert it to run on an actuated ARM64 runner:

\n
jobs:\n  start-kind:\n-    runs-on: ubuntu-latest\n+    runs-on: actuated-aarch64\n
\n

That's it, we've changed the runner type and we're ready to go.

\n

\"In

\n
\n

An in progress build on the dashboard

\n
\n

Behind the scenes, actuated, the SaaS schedules the build on a bare-metal ARM64 server, the boot up takes less than 1 second, and then the standard GitHub Actions Runner talks securely to GitHub to run the build. The build is isolated from other builds, and the runner is destroyed after the build is complete.

\n

\"Setting

\n
\n

Setting up an Arm KinD cluster took about 49s

\n
\n

Setting up an Arm KinD cluster took about 49s, then it's over to you to test your Arm images, or binaries.

\n

If I were setting up CI and needed to test software on both Arm and x86_64, then I'd probably create two separate builds, one for each architecture, with a runs-on label of actuated and actuated-aarch64 respectively.

\n

Do you need to test multiple versions of Kubernetes? Let's face it, it changes so often, that who doesn't need to do that. You can use the matrix feature to test multiple versions of Kubernetes on Arm and x86_64.

\n

I show 5x clusters being launched in parallel in the video below:

\n

Demo - Actuated - secure, isolated CI for containers and Kubernetes

\n

What about Docker?

\n

Docker comes pre-installed in the actuated OS images, so you can simply use docker build, without any need to install extra tools like Buildx, or to have to worry about multi-arch Dockerfiles. Although these are always good to have, and are available out of the box in OpenFaaS, if you're curious what a multi-arch Dockerfile looks like.

\n

Wrapping up

\n

Building on bare-metal Arm hosts is more secure because side effects cannot be left over between builds, even if malware is installed by a bad actor. It's more efficient because you can run multiple builds at once, and you can use the latest software with our automated Operating System image. Enabling actuated on a build is as simple as changing the runner type.

\n

And as you've seen from the example with the OSS Parca project, moving to a native Arm server can improve speed by 22x, shaving off a massive 34 minutes per build.

\n

Who wouldn't want that?

\n

Parca isn't a one-off, I was also told by Connor Hicks from Suborbital that they have an Arm build that takes a good 45 minutes due to using QEMU.

\n

Just a couple of days ago Ed Warnicke, Distinguished Engineer at Cisco reached out to us to pilot actuated. Why?

\n

Ed, who had Network Service Mesh in mind said:

\n
\n

I'd kill for proper Arm support. I'd love to be able to build our many containers for Arm natively, and run our KIND based testing on Arm natively.\nWe want to build for Arm - Arm builds is what brought us to actuated

\n
\n

So if that sounds like where you are, reach out to us and we'll get you set up.

\n\n

Additional links:

\n","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/oidc-proxy-for-openfaas.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/oidc-proxy-for-openfaas.json deleted file mode 100644 index 489ca30a..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/oidc-proxy-for-openfaas.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","contentHtml":"

In 2021, GitHub released OpenID Connect (OIDC) support for CI jobs running under GitHub Actions. This was a huge step forward for security meaning that any GitHub Action could mint an OIDC token and use it to securely federate into another system without having to store long-lived credentials in the repository.

\n

I wrote a prototype for OpenFaaS shortly after the announcement and a deep dive explaining how it works. I used inlets to set up a HTTPS tunnel, and send the token to my machine for inspection. Various individuals and technical teams have used my content as a reference guide when working with GitHub Actions and OIDC.

\n

See the article: Deploy without credentials with GitHub Actions and OIDC

\n

Since then, custom actions for GCP, AWS and Azure have been created which allow an OIDC token from a GitHub Action to be exchanged for a short-lived access token for their API - meaning you can manage cloud resources securely. For example, see: Configuring OpenID Connect in Amazon Web Services - we have actuated customers who use this approach to deploy to ECR from their self-hosted runners without having to store long-lived credentials in their repositories.

\n

Why OIDC is important for OpenFaaS customers

\n

Before we talk about the new OIDC proxy for OpenFaaS, I should say that OpenFaaS Enterprise also has an IAM feature which includes OIDC support for the CLI, dashboard and API. It supports any trusted OIDC provider, not just GitHub Actions. Rather than acting as a proxy, it actually implements a full fine-grained authorization and permissions policy language that resembles the one you'll be used to from AWS.

\n

However, not everyone needs this level of granularity.

\n

Shaked, the CTO of Kubiya.ai is an OpenFaaS & inlets customer. His team at Kubiya is building a conversational AI for DevOps - if you're ever tried ChatGPT, imagine that it was hooked up to your infrastructure and had superpowers. On a recent call, he told me that their team now has 30 different repositories which deploy OpenFaaS functions to their various AWS EKS clusters. That means that a secret has to be maintained at the organisation level and then consumed via faas-cli login in each job.

\n

It gets a little worse for them - because different branches deploy to different OpenFaaS gateways and to different EKS clusters.

\n

In addition to managing various credentials for each cluster they add - they were uncomfortable with exposing all of their functions on the Internet.

\n

So today the team working on actuated is releasing a new OIDC proxy which can be deployed to any OpenFaaS cluster to avoid the need to manage and share long-lived credentials with GitHub.

\n

\"Conceptual

\n
\n

Conceptual design of the OIDC proxy for OpenFaaS

\n
\n

About the OIDC proxy for OpenFaaS

\n\n

Best of all, unlike OpenFaaS Enterprise, it's free for all actuated customers - whether they're using OpenFaaS CE, Standard or Enterprise.

\n

Here's what Shaked had to say about the new proxy:

\n
\n

That's great - thank you! Looking forward to it as it will simplify our usage of the openfaas templates and will speed up our development process\nShaked, CTO, Kubiya.ai

\n
\n

How to deploy the proxy for OpenFaaS

\n

Here's what you need to do:

\n\n
\n

My cluster is not publicly exposed on the Internet, so I'm using an inlets tunnel to expose the OIDC Proxy from my local KinD cluster. I'll be using the domain minty.exit.o6s.io but you'd create something more like oidc-proxy.example.com for your own cluster.

\n
\n

First Set up your values.yaml for Helm:

\n
# The public URL to access the proxy\npublicURL: https://oidc-proxy.example.com\n\n# Comma separated list of repository owners for which short-lived OIDC tokens are authorized.\n# For example: alexellis,self-actuated\nrepositoryOwners: 'alexellis,self-actuated'\ningress:\n    host: oidc-proxy.example.com\n    issuer: letsencrypt-prod\n
\n

The chart will create an Ingress record for you using an existing issuer. If you want to use something else like Inlets or Istio to expose the OIDC proxy, then simply set enabled: false under the ingress: section.

\n

Create a secret for the actuated subscription key:

\n
kubectl create secret generic actuated-license \\\n  -n openfaas \\\n  --from-file=actuated-license=$HOME/.actuated/LICENSE\n
\n

Then run:

\n
helm repo add actuated https://self-actuated.github.io/charts/\nhelm repo update\n\nhelm upgrade --install actuated/openfaas-oidc-proxy \\\n    -f ./values.yaml\n
\n

For the full setup - see the README for the Helm chart

\n

You can now go to one of your repositories and update the workflow to authenticate to the REST API via an OIDC token.

\n

In order to get an OIDC token within a build, add the id_token: write permission to the permissions list.

\n
name: keyless_deploy\n\non:\n  workflow_dispatch:\n  push:\n    branches:\n    - '*'\njobs:\n  keyless_deploy:\n    permissions:\n      contents: 'read'\n      id-token: 'write'\n
\n

Then set runs-on to actuated to use your faster actuated servers:

\n
-   runs-on: ubuntu-latest\n+   runs-on: actuated\n
\n

Then in the workflow, install the OpenFaaS CLI:

\n
steps:\n    - uses: actions/checkout@master\n    with:\n        fetch-depth: 1\n    - name: Install faas-cli\n    run: curl -sLS https://cli.openfaas.com | sudo sh\n
\n

Then get a token:

\n
- name: Get token and use the CLI\n    run: |\n        OPENFAAS_URL=https://minty.exit.o6s.io\n        OIDC_TOKEN=$(curl -sLS \"${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=$OPENFAAS_URL\" -H \"User-Agent: actions/oidc-client\" -H \"Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN\")\n        JWT=$(echo $OIDC_TOKEN | jq -j '.value')\n
\n

Finally, use the token whenever you need it by passing in the --token flag to any of the faas-cli commands:

\n
faas-cli list -n openfaas-fn --token \"$JWT\"\nfaas-cli ns --token \"$JWT\"\nfaas-cli store deploy printer --name p1 --token \"$JWT\"\n\nfaas-cli describe p1 --token \"$JWT\"\n
\n

Since we have a lot of experience with GitHub Actions, we decided to make the above simpler by creating a custom Composite Action. If you check out the code for self-actuated/openfaas-oidc you'll see that it obtains a token, then writes it into an openfaaas config file, so that the --token flag isn't required.

\n

Here's how it changes:

\n
- uses: self-actuated/openfaas-oidc@v1\n  with: \n    gateway: https://minty.exit.o6s.io\n- name: Check OpenFaaS version\n  run: |\n    OPENFAAS_CONFIG=$HOME/.openfaas/\n    faas-cli version\n
\n

Here's the complete example:

\n
name: federate\n\non:\n  workflow_dispatch:\n  push:\n    branches:\n    - '*'\njobs:\n  auth:\n    # Add \"id-token\" with the intended permissions.\n    permissions:\n      contents: 'read'\n      id-token: 'write'\n\n    runs-on: actuated\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n      - name: Install faas-cli\n        run: curl -sLS https://cli.openfaas.com | sudo sh\n      - uses: self-actuated/openfaas-oidc@v1\n        with:\n          gateway: https://minty.exit.o6s.io\n\n      - name: Get token and use the CLI\n        run: |\n          export OPENFAAS_URL=https://minty.exit.o6s.io\n          faas-cli store deploy env --name http-header-printer\n          faas-cli list\n
\n

How can we be sure that our functions cannot be invoked over the proxy?

\n

Just add an extra line to test it out:

\n
      - name: Get token and use the CLI\n        run: |\n          export OPENFAAS_URL=https://minty.exit.o6s.io\n          faas-cli store deploy env --name http-header-printer\n          sleep 5\n\n          echo | faas-cli invoke http-header-printer\n
\n

\"A

\n
\n

A failed invocation over the proxy

\n
\n

Best of all, now that you're using OIDC, you can now go and delete any of those long lived basic auth credentials from your secrets!

\n

Wrapping up

\n

The new OIDC proxy for OpenFaaS is available for all actuated customers and works with OpenFaaS CE, Standard and Enterprise. You can use it on as many clusters as you like, whilst you have an active subscription for actuated at no extra cost.

\n

In a short period of time, you can set up the Helm chart for the OIDC proxy and no longer have to worry about storing various secrets in GitHub Actions for all your clusters, simply obtain a token and use it to deploy to any cluster - securely. There's no risk that your functions will be exposed on the Internet, because the OIDC proxy only works for the /system endpoints of the OpenFaaS REST API.

\n

An alternative for those who need it

\n

OpenFaaS Enterprise has its own OIDC integration with much more fine-grained permissions implemented. It means that team members using the CLI, Dashboard or API do not need to memorise or share basic authentication credentials with each other, or worry about getting the right password for the right cluster.

\n

An OpenFaaS Enterprise policy can restrict all the way down to read/write permissions on a number of namespaces, and also integrates with OIDC.

\n

See an example:

\n","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/ollama-in-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/ollama-in-github-actions.json deleted file mode 100644 index 8ba091d2..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/ollama-in-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","contentHtml":"

That means you can run real end to end tests in CI with the same models you may use in dev and production. And if you use OpenAI or AWS SageMaker extensively, you could perhaps swap out what can be a very expensive API endpoint for your CI or testing environments to save money.

\n

If you'd like to learn more about how and why you'd want access to GPUs in CI, read my past update: Accelerate GitHub Actions with dedicated GPUs.

\n

We'll first cover what ollama is, why it's so popular, how to get it, what kinds of fun things you can do with it, then how to access it from actuated using a real GPU.

\n

\"ollama

\n
\n

ollama can now run in CI with isolated GPU acceleration using actuated

\n
\n

What's ollama?

\n

ollama is an open source project that aims to do for AI models, what Docker did for Linux containers. Whilst Docker created a user experience to share and run containers using container images in the Open Container Initiative (OCI) format, ollama bundles well-known AI models and makes it easy to run them without having to think about Python versions or Nvidia CUDA libraries.

\n

The project packages and runs various models, but seems to take its name from Meta's popular llama2 model, which whilst not released under an open source license, allows for a generous amount of free usage for most types of users.

\n

The ollama project can be run directly on a Linux, MacOS or Windows host, or within a container. There's a server component, and a CLI that acts as a client to pre-trained models. The main use-case today is that of inference - exercising the model with input data. A more recent feature means that you can create embeddings, if you pull a model that supports them.

\n

On Linux, ollama can be installed using a utility script:

\n
curl -fsSL https://ollama.com/install.sh | sh\n
\n

This provides the ollama CLI command.

\n

A quick tour of ollama

\n

After the initial installation, you can start a server:

\n
ollama serve\n
\n

By default, its REST API will listen on port 11434 on 127.0.0.1.

\n

You can find the reference for ollama's REST API here: API endpoints - which includes things like: creating a chat completion, pulling a model, or generating embeddings.

\n

You can then browse available models on the official website, which resembles the Docker Hub. This set currently includes: gemma (built upon Google's DeepMind), mistral (an LLM), codellama (for generating Code), phi (from Microsoft research), vicuna (for chat, based upon llama2), llava (a vision encoder), and many more.

\n

Most models will download with a default parameter size that's small enough to run on most CPUs or GPUs, but if you need to access it, there are larger models for higher accuracy.

\n

For instance, the llama2 model by Meta will default to the 7b model which needs around 8GB of RAM.

\n
# Pull the default model size:\n\nollama pull llama2\n\n# Override the parameter size\n\nollama pull llama2:13b\n
\n

Once you have a model, you can then either \"run\" it, where you'll be able to ask it questions and interact with it like you would with ChatGPT, or you can send it API requests from your own applications using REST and HTTP.

\n

For an interactive prompt, give no parameters:

\n
ollama run llama2\n
\n

To get an immediate response for use in i.e. scripts:

\n
ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n
\n

And you can use the REST API via curl, or your own codebase:

\n
curl -s http://localhost:11434/api/generate -d '{\n    \"model\": \"llama2\",\n    \"stream\": false,\n    \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n}' | jq\n
\n

We are just scratching the surface with what ollama can do, with a focus on testing and pulling pre-built models, but you can also create and share models using a Modelfile, which is another homage to the Docker experience by the ollama developers.

\n

Access ollama from Python code

\n

Here's how to access the API via Python, the stream parameter will emit JSON progressively when set to True, block until done if set to False. With Node.js, Python, Java, C#, etc the code will be very similar, but using your own preferred HTTP client. For Golang (Go) users, ollama founder Jeffrey Morgan maintains a higher-level Go SDK.

\n
import requests\nimport json\n\nurl = \"http://localhost:11434/api/generate\"\npayload = {\n    \"model\": \"llama2\",\n    \"stream\": False,\n    \"prompt\": \"What are the risks of running privileged Docker containers for CI workloads?\"\n}\nheaders = {\n    \"Content-Type\": \"application/json\"\n}\n\nresponse = requests.post(url, data=json.dumps(payload), headers=headers)\n\n# Parse the JSON response\nresponse_json = response.json()\n\n# Pretty print the JSON response\nprint(json.dumps(response_json, indent=4))\n
\n

When you're constructing a request by API, make sure you include any tags in the model name, if you've used one. I.e. \"model\": \"llama2:13b\".

\n

I hear from so many organisations who have gone to lengths to get SOC2 compliance, doing CVE scanning, or who are running Open Policy Agent or Kyverno to enforce strict Pod admission policies in Kubernetes, but then are happy to run their CI in Pods in privileged mode. So I asked the model why that may not be a smart idea. You can run the sample for yourself or see the response here. We also go into detail in the actuated FAQ, the security situation around self-hosted runners and containers is the main reason we built the solution.

\n

Putting it together for a GitHub Action

\n

The following GitHub Action will run on for customers who are enrolled for GPU support for actuated. If you'd like to gain access, contact us via the form on the Pricing page.

\n

The self-actuated/nvidia-run installs either the consumer or datacenter driver for Nvidia, depending on what you have in your system. This only takes about 30 seconds and could be cached if you like. The ollama models could also be cached using a local S3 bucket.

\n

Then, we simply run the equivalent bash commands from the previous section to:

\n\n
name: ollama-e2e\n\non:\n    workflow_dispatch:\n\njobs:\n    ollama-e2e:\n        name: ollama-e2e\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Install Ollama\n          run: |\n            curl -fsSL https://ollama.com/install.sh | sudo -E sh\n        - name: Start serving\n          run: |\n              # Run the background, there is no way to daemonise at the moment\n              ollama serve &\n\n              # A short pause is required before the HTTP port is opened\n              sleep 5\n\n              # This endpoint blocks until ready\n              time curl -i http://localhost:11434\n\n        - name: Pull llama2\n          run: |\n              ollama pull llama2\n\n        - name: Invoke via the CLI\n          run: |\n              ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n\n        - name: Invoke via API\n          run: |\n            curl -s http://localhost:11434/api/generate -d '{\n              \"model\": \"llama2\",\n              \"stream\": false,\n              \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n            }' | jq\n
\n

There is no built-in way to daemonise the ollama server, so for now we run it in the background using bash. The readiness endpoint can then be accessed which blocks until the server has completed its initialisation.

\n

Interactive access with SSH

\n

By modifying your CI job, you can drop into a remote SSH session and run interactive commands at any point in the workflow.

\n

That's how I came up with the commands for the Nvidia driver installation, and for the various ollama commands I shared.

\n

Find out more about SSH for GitHub Actions in the actuated docs.

\n

\"Pulling

\n
\n

Pulling one of the larger llama2 models interactively in an SSH session, directly to the runner VM

\n
\n

Wrapping up

\n

Within a very short period of time ollama helped us pull a popular AI model that can be used for chat and completions. We were then able to take what we learned and run it on a GPU at an accelerated speed and accuracy by using actuated's new GPU support for GitHub Actions and GitLab CI. Most hosted CI systems provide a relatively small amount of disk space for jobs, with actuated you can customise this and that may be important if you're going to be downloading large AI models. You can also easily customise the amount of RAM and vCPU using the runs-on label to any combination you need.

\n

ollama isn't the only way to find, download and run AI models, just like Docker wasn't the only way to download and install Nginx or Postgresql, but it provides a useful and convenient interface for those of us who are still learning about AI, and are not as concerned with the internal workings of the models.

\n

Over on the OpenFaaS blog, in the tutorial Stream OpenAI responses from functions using Server Sent Events, we covered how to stream a response from a model to a function, and then back to a user. There, we used the llama-api open source project, which is a single-purpose HTTP API for simulating llama2.

\n

One of the benefits of ollama is the detailed range of examples in the docs, and the ability to run other models that may include computer vision such as with the LLaVA: Large Language and Vision Assistant model or generating code with Code Llama.

\n

Right now, many of us are running and tuning models in development, some of us are using OpenAI's API or self-hosted models in production, but there's very little talk about doing thorough end to end testing or exercising models in CI. That's where actuated can help.

\n

Feel free to reach out for early access, or to see if we can help your team with your CI needs.

","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/sbom-in-github-actions.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/sbom-in-github-actions.json deleted file mode 100644 index c238a6df..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/sbom-in-github-actions.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","contentHtml":"

What is a Software Bill of Materials (SBOM)?

\n

In April 2022 Justin Cormack, CTO of Docker announced that Docker was adding support to generate a Software Bill of Materials (SBOM) for container images.

\n

An SBOM is an inventory of the components that make up a software application. It is a list of the components that make up a software application including the version of each component. The version is important because it can be cross-reference with a vulnerability database to determine if the component has any known vulnerabilities.

\n

Many organisations are also required to company with certain Open Source Software (OSS) licenses. So if SBOMs are included in the software they purchase or consume from vendors, then it can be used to determine if the software is compliant with their specific license requirements, lowering legal and compliance risk.

\n

Docker's enhancements to Docker Desktop and their open source Buildkit tool were the result of a collaboration with Anchore, a company that provides a commercial SBOM solution.

\n

Check out an SBOM for yourself

\n

Anchore provides commercial solutions for creating, managing and inspecting SBOMs, however they also have two very useful open source tools that we can try out for free.

\n\n

OpenFaaS Community Edition (CE) is a popular open source serverless platform for Kubernetes. It's maintained by open source developers, and is free to use.

\n

Let's pick a container image from the Community Edition of OpenFaaS like the container image for the OpenFaaS gateway.

\n

We can browse the GitHub UI to find the latest revision, or we can use Google's crane tool:

\n
crane ls ghcr.io/openfaas/gateway | tail -n 5\n0.26.0\n8e1c34e222d6c194302c649270737c516fe33edf\n0.26.1\nc26ec5221e453071216f5e15c3409168446fd563\n0.26.2\n
\n

Now we can introduce one of those tags to syft:

\n
syft ghcr.io/openfaas/gateway:0.26.2\n ✔ Pulled image            \n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [39 packages]\nNAME                                              VERSION                               TYPE      \nalpine-baselayout                                 3.4.0-r0                              apk        \nalpine-baselayout-data                            3.4.0-r0                              apk        \nalpine-keys                                       2.4-r1                                apk        \napk-tools                                         2.12.10-r1                            apk        \nbusybox                                           1.35.0                                binary     \nbusybox                                           1.35.0-r29                            apk        \nbusybox-binsh                                     1.35.0-r29                            apk        \nca-certificates                                   20220614-r4                           apk        \nca-certificates-bundle                            20220614-r4                           apk        \ngithub.com/beorn7/perks                           v1.0.1                                go-module  \ngithub.com/cespare/xxhash/v2                      v2.1.2                                go-module  \ngithub.com/docker/distribution                    v2.8.1+incompatible                   go-module  \ngithub.com/gogo/protobuf                          v1.3.2                                go-module  \ngithub.com/golang/protobuf                        v1.5.2                                go-module  \ngithub.com/gorilla/mux                            v1.8.0                                go-module  \ngithub.com/matttproud/golang_protobuf_extensions  v1.0.1                                go-module  \ngithub.com/nats-io/nats.go                        v1.22.1                               go-module  \ngithub.com/nats-io/nkeys                          v0.3.0                                go-module  \ngithub.com/nats-io/nuid                           v1.0.1                                go-module  \ngithub.com/nats-io/stan.go                        v0.10.4                               go-module  \ngithub.com/openfaas/faas-provider                 v0.19.1                               go-module  \ngithub.com/openfaas/faas/gateway                  (devel)                               go-module  \ngithub.com/openfaas/nats-queue-worker             v0.0.0-20230117214128-3615ccb286cc    go-module  \ngithub.com/prometheus/client_golang               v1.13.0                               go-module  \ngithub.com/prometheus/client_model                v0.2.0                                go-module  \ngithub.com/prometheus/common                      v0.37.0                               go-module  \ngithub.com/prometheus/procfs                      v0.8.0                                go-module  \ngolang.org/x/crypto                               v0.5.0                                go-module  \ngolang.org/x/sync                                 v0.1.0                                go-module  \ngolang.org/x/sys                                  v0.4.1-0.20230105183443-b8be2fde2a9e  go-module  \ngoogle.golang.org/protobuf                        v1.28.1                               go-module  \nlibc-utils                                        0.7.2-r3                              apk        \nlibcrypto3                                        3.0.7-r2                              apk        \nlibssl3                                           3.0.7-r2                              apk        \nmusl                                              1.2.3-r4                              apk        \nmusl-utils                                        1.2.3-r4                              apk        \nscanelf                                           1.3.5-r1                              apk        \nssl_client                                        1.35.0-r29                            apk        \nzlib                                              1.2.13-r0                             apk  \n
\n

These are all the components that syft found in the container image. We can see that it found 39 packages, including the OpenFaaS gateway itself.

\n

Some of the packages are Go modules, others are packages that have been installed with apk (Alpine Linux's package manager).

\n

Checking for vulnerabilities

\n

Now that we have an SBOM, we can use grype to check for vulnerabilities.

\n
grype ghcr.io/openfaas/gateway:0.26.2\n ✔ Vulnerability DB        [no update available]\n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [39 packages]\n ✔ Scanned image           [2 vulnerabilities]\nNAME                        INSTALLED  FIXED-IN  TYPE       VULNERABILITY   SEVERITY \ngoogle.golang.org/protobuf  v1.28.1              go-module  CVE-2015-5237   High      \ngoogle.golang.org/protobuf  v1.28.1              go-module  CVE-2021-22570  Medium  \n
\n

In this instance, we can see there are only two vulnerabilities, both of which are in the google.golang.org/protobuf Go module, and neither of them have been fixed yet.

\n\n

With this scenario, I wanted to show that different people care about the supply chain, and have different responsibilities for it.

\n

Generate an SBOM from within GitHub Actions

\n

The examples above were all run locally, but we can also generate an SBOM from within a GitHub Actions workflow. In this way, the SBOM is shipped with the container image and is made available without having to scan the image each time.

\n

Imagine you have the following Dockerfile:

\n
FROM alpine:3.17.0\n\nRUN apk add --no-cache curl ca-certificates\n\nCMD [\"curl\", \"https://www.google.com\"]\n
\n

I know that there's a vulnerability in alpine 3.17.0 in the OpenSSL library. How do I know that? I recently updated every OpenFaaS Pro component to use 3.17.1 to fix a specific vulnerability.

\n

Now a typical workflow for this Dockerfile would look like the below:

\n
name: build\n\non:\n  push:\n    branches: [ master, main ]\n  pull_request:\n    branches: [ master, main ]\n\npermissions:\n  actions: read\n  checks: write\n  contents: read\n  packages: write\n\njobs:\n  publish:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n        with:\n          fetch-depth: 1\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v2\n\n      - name: Login to Docker Registry\n        uses: docker/login-action@v2\n        with:\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n          registry: ghcr.io\n\n      - name: Publish image\n        uses: docker/build-push-action@v4\n        with:\n          build-args: |\n            GitCommit=${{ github.sha }}\n          outputs: \"type=registry,push=true\"\n          provenance: false\n          tags: |\n            ghcr.io/alexellis/gha-sbom:${{ github.sha }}\n
\n

Upon each commit, an image is published to GitHub's Container Registry with the image name of: ghcr.io/alexellis/gha-sbom:SHA.

\n

To generate an SBOM, we just need to update the docker/build-push-action to use the sbom flag:

\n
      - name: Local build\n        id: local_build\n        uses: docker/build-push-action@v4\n        with:\n          sbom: true\n          provenance: false\n
\n

By checking the logs from the action, we can see that the image has been published with an SBOM:

\n
#16 [linux/amd64] generating sbom using docker.io/docker/buildkit-syft-scanner:stable-1\n#0 0.120 time=\"2023-01-25T15:35:19Z\" level=info msg=\"starting syft scanner for buildkit v1.0.0\"\n#16 DONE 1.0s\n
\n

The SBOM can be viewed as before:

\n
syft ghcr.io/alexellis/gha-sbom:46bc16cb4033364233fad3caf8f3a255b5b4d10d@sha256:7229e15004d8899f5446a40ebdd072db6ff9c651311d86e0c8fd8f999a32a61a\n\ngrype ghcr.io/alexellis/gha-sbom:46bc16cb4033364233fad3caf8f3a255b5b4d10d@sha256:7229e15004d8899f5446a40ebdd072db6ff9c651311d86e0c8fd8f999a32a61a\n ✔ Vulnerability DB        [updated]\n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [21 packages]\n ✔ Scanned image           [2 vulnerabilities]\nNAME        INSTALLED  FIXED-IN  TYPE  VULNERABILITY  SEVERITY \nlibcrypto3  3.0.7-r0   3.0.7-r2  apk   CVE-2022-3996  High      \nlibssl3     3.0.7-r0   3.0.7-r2  apk   CVE-2022-3996  High  \n
\n

The image: alpine:3.17.0 contains two High vulnerabilities, and from reading the notes, we can see that both have been fixed.

\n

We can resolve the issue by changing the Dockerfile to use alpine:3.17.1 instead, and re-running the build.

\n
grype ghcr.io/alexellis/gha-sbom:63c6952d1ded1f53b1afa3f8addbba9efa37b52b\n ✔ Vulnerability DB        [no update available]\n ✔ Pulled image            \n ✔ Loaded image            \n ✔ Parsed image            \n ✔ Cataloged packages      [21 packages]\n ✔ Scanned image           [0 vulnerabilities]\nNo vulnerabilities found\n
\n

Wrapping up

\n

There is a lot written on the topic of supply chain security, so I wanted to give you a quick overview, and how to get started wth it.

\n

We looked at Anchore's two open source tools: Syft and Grype, and how they can be used to generate an SBOM and scan for vulnerabilities.

\n

We then produced an SBOM for a pre-existing Dockerfile and GitHub Action, introducing a vulnerability by using an older base image, and then fixing it by upgrading it. We did this by adding additional flags to the docker/build-push-action. We added the sbom flag, and set the provenance flag to false. Provenance is a separate but related topic, which is explained well in an article by Justin Chadwell of Docker (linked below).

\n

I maintain an Open Source alternative to brew for developer-focused CLIs called arkade. This already includes Google's crane project, and there's a Pull Request coming shortly to add Syft and Grype to the project.

\n

It can be a convenient way to install these tools on MacOS, Windows or Linux:

\n
# Available now\narkade get crane syft\n\n# Coming shortly\narkade get grype\n
\n

In the beginning of the article we mentioned license compliance. SBOMs generated by syft do not seem to include license information, but in my experience, corporations which take this risk seriously tend to run their own scanning infrastructure with commercial tools like Blackduck or Twistlock.

\n

Tools like Twistlock, and certain registries like JFrog Artifactory and the CNCF's Harbor, can be configured to scan images. GitHub has a free, built-in service called Dependabot that won't just scan, but will also send Pull Requests to fix issues.

\n

But with the SBOM approach, the responsibility is rebalanced, with the supplier taking on an active role in security. The consumer can then use the supplier's SBOMs, or run their own scanning infrastructure - or perhaps both.

\n

See also:

\n","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/secure-microvm-ci-gitlab.json b/_next/data/qP6XrePfh_ktdNhbSnok_/blog/secure-microvm-ci-gitlab.json deleted file mode 100644 index 97b079aa..00000000 --- a/_next/data/qP6XrePfh_ktdNhbSnok_/blog/secure-microvm-ci-gitlab.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"post":{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","contentHtml":"

We started building actuated for GitHub Actions because we at OpenFaaS Ltd had a need for: unmetered CI minutes, faster & more powerful x86 machines, native Arm builds and low maintenance CI builds.

\n

And most importantly, we needed it to be low-maintenance, and securely isolated.

\n

None of the solutions at the time could satisfy all of those requirements, and even today with GitHub adopting the community-based Kubernetes controller to run CI in Pods, there is still a lot lacking.

\n

As we've gained more experience with customers who largely had the same needs as we did for GitHub Actions, we started to hear more and more from GitLab CI users. From large enterprise companies who are concerned about the security risks of running CI with privileged Docker containers, Docker socket binding (from the host!) or the flakey nature and slow speed of VFS with Docker In Docker (DIND).

\n
\n

The GitLab docs have a stark warning about using both of these approaches. It was no surprise that when a consultant at Thoughtworks reached out to me, he listed off the pain points and concerns that we'd set out to solve for GitHub Actions.

\n

At KubeCon, I also spoke to several developers who worked at Deutsche Telekom who had numerous frustrations with the user-experience and management overhead of the Kubernetes executor.

\n
\n

So with growing interest from customers, we built a solution for GitLab CI - just like we did for GitHub Actions. We're excited to share it with you today in tech preview.

\n

\"actuated

\n

For every build that requires a runner, we will schedule and boot a complete system with Firecracker using Linux KVM for secure isolation. After the job is completed, the VM will be destroyed and removed from the GitLab instance.

\n

actuated for GitLab is for self-hosted GitLab instances, whether hosted on-premises or on the public cloud.

\n

If you'd like to use it or find out more, please apply here: Sign-up for the Actuated pilot

\n

Secure CI with Firecracker microVMs

\n

Firecracker is the open-source technology that provides isolation between tenants on certain AWS products like Lambda and Fargate. There's a growing number of cloud native solutions evolving around Firecracker, and we believe that it's the only way to run CI/CD securely.

\n

Firecracker is a virtual machine monitor (VMM) that uses the Linux Kernel-based Virtual Machine (KVM) to create and manage microVMs. It's lightweight, fast, and most importantly, provides proper isolation, which anything based upon Docker cannot.

\n

There are no horrible Kernel tricks or workarounds to be able to use user namespaces, no need to change your tooling from what developers love - Docker, to Kaninko or Buildah or similar.

\n

You'll get sudo, plus a fresh Docker engine in every VM, booted up with systemd, so things like Kubernetes work out of the box, if you need them for end to end testing (as so many of us do these days).

\n

You can learn the differences between VMs, containers and microVMs like Firecracker in my video from Cloud Native Rejekts at KubeCon Amsterdam:

\n\n

Many people have also told me that they learned how to use Firecracker from my webinar last year with Richard Case: A cracking time: Exploring Firecracker & MicroVMs.

\n

Let's see it then

\n

Here's a video demo of the tech preview we have available for customers today.

\n\n

You'll see that when I create a commit in our self-hosted copy of GitLab Enterprise, within 1 second a microVM is booting up and running the CI job.

\n

Shortly after that the VM is destroyed which means there are absolutely no side-effects or any chance of leakage between jobs.

\n

Here's a later demo of three jobs within a single pipeline, all set to run in parallel.

\n

Here's 3x @GitLab CI jobs running in parallel within the same Pipeline demoed by @alexellisuk

All in their own ephemeral VM powered by Firecracker 🔥#cicd #secure #isolation #microvm #baremetal pic.twitter.com/fe5HaxMsGB

— actuated (@selfactuated) June 13, 2023
\n

Everything's completed before I have a chance to even open the logs in the UI of GitLab.

\n

Wrapping up

\n

actuated for GitLab is for self-hosted GitLab instances, whether hosted on-premises or on the public cloud.

\n

Here's what we bring to the table:

\n\n

Runners are registered and running a job in a dedicated VM within less than one second. Our scheduler can pack in jobs across a fleet of servers, they just need to have KVM available.

\n

If you think your automation for runners could be improved, or work with customers who need faster builds, better isolation or Arm support, get in touch with us.

\n\n

You can follow @selfactuated on Twitter, or find me there too to keep an eye on what we're building.

","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/static/chunks/pages/_app-a058602dc4f1b3be.js b/_next/static/chunks/pages/_app-70ff176000ac1f10.js similarity index 99% rename from _next/static/chunks/pages/_app-a058602dc4f1b3be.js rename to _next/static/chunks/pages/_app-70ff176000ac1f10.js index fad0cbf9..2a8793c0 100644 --- a/_next/static/chunks/pages/_app-a058602dc4f1b3be.js +++ b/_next/static/chunks/pages/_app-70ff176000ac1f10.js @@ -1,4 +1,4 @@ -(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[888],{1118:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/_app",function(){return n(1613)}])},227:function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getDomainLocale=function(e,t,n,r){return!1},("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},1551:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(4941).Z;n(5753).default,Object.defineProperty(t,"__esModule",{value:!0}),t.default=void 0;var o=n(2648).Z,a=n(7273).Z,l=o(n(7294)),i=n(1003),u=n(7795),s=n(4465),c=n(2692),d=n(8245),f=n(9246),p=n(227),m=n(3468),v=new Set;function h(e,t,n,r){if(i.isLocalURL(t)){if(!r.bypassPrefetchedCheck){var o=t+"%"+n+"%"+(void 0!==r.locale?r.locale:"locale"in e?e.locale:void 0);if(v.has(o))return;v.add(o)}Promise.resolve(e.prefetch(t,n,r)).catch(function(e){})}}function g(e){return"string"==typeof e?e:u.formatUrl(e)}var b=l.default.forwardRef(function(e,t){var n,o,u=e.href,v=e.as,b=e.children,y=e.prefetch,x=e.passHref,w=e.replace,E=e.shallow,j=e.scroll,P=e.locale,S=e.onClick,N=e.onMouseEnter,O=e.onTouchStart,C=e.legacyBehavior,M=void 0!==C&&C,T=a(e,["href","as","children","prefetch","passHref","replace","shallow","scroll","locale","onClick","onMouseEnter","onTouchStart","legacyBehavior"]);n=b,M&&("string"==typeof n||"number"==typeof n)&&(n=l.default.createElement("a",null,n));var F=!1!==y,A=l.default.useContext(c.RouterContext),R=l.default.useContext(d.AppRouterContext),L=null!=A?A:R,k=!A,I=l.default.useMemo(function(){if(!A){var e=g(u);return{href:e,as:v?g(v):e}}var t=r(i.resolveHref(A,u,!0),2),n=t[0],o=t[1];return{href:n,as:v?i.resolveHref(A,v):o||n}},[A,u,v]),H=I.href,_=I.as,B=l.default.useRef(H),D=l.default.useRef(_);M&&(o=l.default.Children.only(n));var z=M?o&&"object"==typeof o&&o.ref:t,Z=r(f.useIntersection({rootMargin:"200px"}),3),V=Z[0],U=Z[1],$=Z[2],q=l.default.useCallback(function(e){(D.current!==_||B.current!==H)&&($(),D.current=_,B.current=H),V(e),z&&("function"==typeof z?z(e):"object"==typeof z&&(z.current=e))},[_,z,H,$,V]);l.default.useEffect(function(){L&&U&&F&&h(L,H,_,{locale:P})},[_,H,U,P,F,null==A?void 0:A.locale,L]);var K={ref:q,onClick:function(e){M||"function"!=typeof S||S(e),M&&o.props&&"function"==typeof o.props.onClick&&o.props.onClick(e),L&&!e.defaultPrevented&&function(e,t,n,r,o,a,u,s,c,d){if("A"!==e.currentTarget.nodeName.toUpperCase()||(!(p=(f=e).currentTarget.target)||"_self"===p)&&!f.metaKey&&!f.ctrlKey&&!f.shiftKey&&!f.altKey&&(!f.nativeEvent||2!==f.nativeEvent.which)&&i.isLocalURL(n)){e.preventDefault();var f,p,m=function(){"beforePopState"in t?t[o?"replace":"push"](n,r,{shallow:a,locale:s,scroll:u}):t[o?"replace":"push"](r||n,{forceOptimisticNavigation:!d})};c?l.default.startTransition(m):m()}}(e,L,H,_,w,E,j,P,k,F)},onMouseEnter:function(e){M||"function"!=typeof N||N(e),M&&o.props&&"function"==typeof o.props.onMouseEnter&&o.props.onMouseEnter(e),L&&(F||!k)&&h(L,H,_,{locale:P,priority:!0,bypassPrefetchedCheck:!0})},onTouchStart:function(e){M||"function"!=typeof O||O(e),M&&o.props&&"function"==typeof o.props.onTouchStart&&o.props.onTouchStart(e),L&&(F||!k)&&h(L,H,_,{locale:P,priority:!0,bypassPrefetchedCheck:!0})}};if(!M||x||"a"===o.type&&!("href"in o.props)){var W=void 0!==P?P:null==A?void 0:A.locale,G=(null==A?void 0:A.isLocaleDomain)&&p.getDomainLocale(_,W,null==A?void 0:A.locales,null==A?void 0:A.domainLocales);K.href=G||m.addBasePath(s.addLocale(_,W,null==A?void 0:A.defaultLocale))}return M?l.default.cloneElement(o,K):l.default.createElement("a",Object.assign({},T,K),n)});t.default=b,("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},9246:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(4941).Z;Object.defineProperty(t,"__esModule",{value:!0}),t.useIntersection=function(e){var t=e.rootRef,n=e.rootMargin,s=e.disabled||!l,c=r(o.useState(!1),2),d=c[0],f=c[1],p=r(o.useState(null),2),m=p[0],v=p[1];return o.useEffect(function(){if(l){if(!s&&!d&&m&&m.tagName){var e,r,o,c;return r=(e=function(e){var t,n={root:e.root||null,margin:e.rootMargin||""},r=u.find(function(e){return e.root===n.root&&e.margin===n.margin});if(r&&(t=i.get(r)))return t;var o=new Map;return t={id:n,observer:new IntersectionObserver(function(e){e.forEach(function(e){var t=o.get(e.target),n=e.isIntersecting||e.intersectionRatio>0;t&&n&&t(n)})},e),elements:o},u.push(n),i.set(n,t),t}({root:null==t?void 0:t.current,rootMargin:n})).id,o=e.observer,(c=e.elements).set(m,function(e){return e&&f(e)}),o.observe(m),function(){if(c.delete(m),o.unobserve(m),0===c.size){o.disconnect(),i.delete(r);var e=u.findIndex(function(e){return e.root===r.root&&e.margin===r.margin});e>-1&&u.splice(e,1)}}}}else if(!d){var p=a.requestIdleCallback(function(){return f(!0)});return function(){return a.cancelIdleCallback(p)}}},[m,s,n,t,d]),[v,d,o.useCallback(function(){f(!1)},[])]};var o=n(7294),a=n(4686),l="function"==typeof IntersectionObserver,i=new Map,u=[];("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},1613:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return e$},reportWebVitals:function(){return eU}});var r,o,a,l,i,u,s,c,d,f=n(1799),p=n(5893),m=n(9008),v=n.n(m),h=n(7294),g=n(1664),b=n.n(g),y=n(2984),x=n(2351),w=n(3784),E=n(9946),j=n(1363),P=n(4103),S=n(5466);let N=["[contentEditable=true]","[tabindex]","a[href]","area[href]","button:not([disabled])","iframe","input:not([disabled])","select:not([disabled])","textarea:not([disabled])"].map(e=>`${e}:not([tabindex='-1'])`).join(",");var O=((r=O||{})[r.First=1]="First",r[r.Previous=2]="Previous",r[r.Next=4]="Next",r[r.Last=8]="Last",r[r.WrapAround=16]="WrapAround",r[r.NoScroll=32]="NoScroll",r),C=((o=C||{})[o.Error=0]="Error",o[o.Overflow=1]="Overflow",o[o.Success=2]="Success",o[o.Underflow=3]="Underflow",o),M=((a=M||{})[a.Previous=-1]="Previous",a[a.Next=1]="Next",a);function T(e=document.body){return null==e?[]:Array.from(e.querySelectorAll(N))}var F=((l=F||{})[l.Strict=0]="Strict",l[l.Loose=1]="Loose",l);function A(e,t=0){var n;return e!==(null==(n=(0,S.r)(e))?void 0:n.body)&&(0,y.E)(t,{0:()=>e.matches(N),1(){let t=e;for(;null!==t;){if(t.matches(N))return!0;t=t.parentElement}return!1}})}function R(e,t,n=!0,r=null){var o,a,l;let i=Array.isArray(e)?e.length>0?e[0].ownerDocument:document:e.ownerDocument,u=Array.isArray(e)?n?function(e,t=e=>e){return e.slice().sort((e,n)=>{let r=t(e),o=t(n);if(null===r||null===o)return 0;let a=r.compareDocumentPosition(o);return a&Node.DOCUMENT_POSITION_FOLLOWING?-1:a&Node.DOCUMENT_POSITION_PRECEDING?1:0})}(e):e:T(e);r=null!=r?r:i.activeElement;let s=(()=>{if(5&t)return 1;if(10&t)return -1;throw Error("Missing Focus.First, Focus.Previous, Focus.Next or Focus.Last")})(),c=(()=>{if(1&t)return 0;if(2&t)return Math.max(0,u.indexOf(r))-1;if(4&t)return Math.max(0,u.indexOf(r))+1;if(8&t)return u.length-1;throw Error("Missing Focus.First, Focus.Previous, Focus.Next or Focus.Last")})(),d=32&t?{preventScroll:!0}:{},f=0,p=u.length,m;do{if(f>=p||f+p<=0)return 0;let v=c+f;if(16&t)v=(v+p)%p;else{if(v<0)return 3;if(v>=p)return 1}null==(m=u[v])||m.focus(d),f+=s}while(m!==i.activeElement);return 6&t&&null!=(l=null==(a=null==(o=m)?void 0:o.matches)?void 0:a.call(o,"textarea,input"))&&l&&m.select(),m.hasAttribute("tabindex")||m.setAttribute("tabindex","0"),2}var L=n(6567),k=n(4157),I=n(3855);function H(e,t,n){let r=(0,I.E)(t);(0,h.useEffect)(()=>{function t(e){r.current(e)}return document.addEventListener(e,t,n),()=>document.removeEventListener(e,t,n)},[e,n])}function _(...e){return(0,h.useMemo)(()=>(0,S.r)(...e),[...e])}var B=((i=B||{})[i.None=1]="None",i[i.Focusable=2]="Focusable",i[i.Hidden=4]="Hidden",i);let D=(0,x.yV)(function(e,t){let{features:n=1,...r}=e,o={ref:t,"aria-hidden":(2&n)==2||void 0,style:{position:"fixed",top:1,left:1,width:1,height:0,padding:0,margin:-1,overflow:"hidden",clip:"rect(0, 0, 0, 0)",whiteSpace:"nowrap",borderWidth:"0",...(4&n)==4&&(2&n)!=2&&{display:"none"}}};return(0,x.sY)({ourProps:o,theirProps:r,slot:{},defaultTag:"div",name:"Hidden"})});var z=n(3781),Z=((u=Z||{})[u.Forwards=0]="Forwards",u[u.Backwards=1]="Backwards",u);function V(){var e,t;let n,r=(0,h.useRef)(0);return e="keydown",t=e=>{"Tab"===e.key&&(r.current=e.shiftKey?1:0)},n=(0,I.E)(t),(0,h.useEffect)(()=>{function t(e){n.current(e)}return window.addEventListener(e,t,!0),()=>window.removeEventListener(e,t,!0)},[e,!0]),r}var U=((s=U||{})[s.Open=0]="Open",s[s.Closed=1]="Closed",s),$=((c=$||{})[c.TogglePopover=0]="TogglePopover",c[c.ClosePopover=1]="ClosePopover",c[c.SetButton=2]="SetButton",c[c.SetButtonId=3]="SetButtonId",c[c.SetPanel=4]="SetPanel",c[c.SetPanelId=5]="SetPanelId",c);let q={0:e=>({...e,popoverState:(0,y.E)(e.popoverState,{0:1,1:0})}),1:e=>1===e.popoverState?e:{...e,popoverState:1},2:(e,t)=>e.button===t.button?e:{...e,button:t.button},3:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},4:(e,t)=>e.panel===t.panel?e:{...e,panel:t.panel},5:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId}},K=(0,h.createContext)(null);function W(e){let t=(0,h.useContext)(K);if(null===t){let n=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(n,W),n}return t}K.displayName="PopoverContext";let G=(0,h.createContext)(null);function Y(e){let t=(0,h.useContext)(G);if(null===t){let n=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(n,Y),n}return t}G.displayName="PopoverAPIContext";let J=(0,h.createContext)(null);function Q(){return(0,h.useContext)(J)}J.displayName="PopoverGroupContext";let X=(0,h.createContext)(null);function ee(e,t){return(0,y.E)(t.type,q,e,t)}X.displayName="PopoverPanelContext";let et=(0,x.yV)(function(e,t){var n,r,o,a;let l;let i=(0,h.useRef)(null),u=(0,w.T)(t,(0,w.h)(e=>{i.current=e})),s=(0,h.useReducer)(ee,{popoverState:1,buttons:[],button:null,buttonId:null,panel:null,panelId:null,beforePanelSentinel:(0,h.createRef)(),afterPanelSentinel:(0,h.createRef)()}),[{popoverState:c,button:d,buttonId:f,panel:p,panelId:m,beforePanelSentinel:v,afterPanelSentinel:g},b]=s,E=_(null!=(n=i.current)?n:d),j=(0,h.useMemo)(()=>{if(!d||!p)return!1;for(let e of document.querySelectorAll("body > *"))if(Number(null==e?void 0:e.contains(d))^Number(null==e?void 0:e.contains(p)))return!0;let t=T(),n=t.indexOf(d),r=(n+t.length-1)%t.length,o=(n+1)%t.length,a=t[r],l=t[o];return!p.contains(a)&&!p.contains(l)},[d,p]),P=(0,I.E)(f),S=(0,I.E)(m),N=(0,h.useMemo)(()=>({buttonId:P,panelId:S,close:()=>b({type:1})}),[P,S,b]),O=Q(),C=null==O?void 0:O.registerPopover,M=(0,z.z)(()=>{var e;return null!=(e=null==O?void 0:O.isFocusWithinPopoverGroup())?e:(null==E?void 0:E.activeElement)&&((null==d?void 0:d.contains(E.activeElement))||(null==p?void 0:p.contains(E.activeElement)))});(0,h.useEffect)(()=>null==C?void 0:C(N),[C,N]),r=null==E?void 0:E.defaultView,o="focus",a=e=>{var t,n,r,o;0===c&&(M()||!d||!p||e.target!==window&&(null!=(n=null==(t=v.current)?void 0:t.contains)&&n.call(t,e.target)||null!=(o=null==(r=g.current)?void 0:r.contains)&&o.call(r,e.target)||b({type:1})))},l=(0,I.E)(a),(0,h.useEffect)(()=>{function e(e){l.current(e)}return(r=null!=r?r:window).addEventListener(o,e,!0),()=>r.removeEventListener(o,e,!0)},[r,o,!0]),function(e,t,n=!0){let r=(0,h.useRef)(!1);function o(n,o){if(!r.current||n.defaultPrevented)return;let a=function e(t){return"function"==typeof t?e(t()):Array.isArray(t)||t instanceof Set?t:[t]}(e),l=o(n);if(null!==l&&l.getRootNode().contains(l)){for(let i of a){if(null===i)continue;let u=i instanceof HTMLElement?i:i.current;if(null!=u&&u.contains(l)||n.composed&&n.composedPath().includes(u))return}return A(l,F.Loose)||-1===l.tabIndex||n.preventDefault(),t(n,l)}}(0,h.useEffect)(()=>{requestAnimationFrame(()=>{r.current=n})},[n]);let a=(0,h.useRef)(null);H("mousedown",e=>{var t,n;r.current&&(a.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target)},!0),H("click",e=>{a.current&&(o(e,()=>a.current),a.current=null)},!0),H("blur",e=>o(e,()=>window.document.activeElement instanceof HTMLIFrameElement?window.document.activeElement:null),!0)}([d,p],(e,t)=>{b({type:1}),A(t,F.Loose)||(e.preventDefault(),null==d||d.focus())},0===c);let R=(0,z.z)(e=>{b({type:1});let t=e?e instanceof HTMLElement?e:"current"in e&&e.current instanceof HTMLElement?e.current:d:d;null==t||t.focus()}),k=(0,h.useMemo)(()=>({close:R,isPortalled:j}),[R,j]),B=(0,h.useMemo)(()=>({open:0===c,close:R}),[c,R]);return h.createElement(K.Provider,{value:s},h.createElement(G.Provider,{value:k},h.createElement(L.up,{value:(0,y.E)(c,{0:L.ZM.Open,1:L.ZM.Closed})},(0,x.sY)({ourProps:{ref:u},theirProps:e,slot:B,defaultTag:"div",name:"Popover"}))))}),en=(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-button-${n}`,...o}=e,[a,l]=W("Popover.Button"),{isPortalled:i}=Y("Popover.Button"),u=(0,h.useRef)(null),s=`headlessui-focus-sentinel-${(0,E.M)()}`,c=Q(),d=null==c?void 0:c.closeOthers,f=(0,h.useContext)(X),p=null!==f&&f===a.panelId;(0,h.useEffect)(()=>{if(!p)return l({type:3,buttonId:r}),()=>{l({type:3,buttonId:null})}},[r,l]);let m=(0,w.T)(u,t,p?null:e=>{if(e)a.buttons.push(r);else{let t=a.buttons.indexOf(r);-1!==t&&a.buttons.splice(t,1)}a.buttons.length>1&&console.warn("You are already using a but only 1 is supported."),e&&l({type:2,button:e})}),v=(0,w.T)(u,t),g=_(u),b=(0,z.z)(e=>{var t,n,r;if(p){if(1===a.popoverState)return;switch(e.key){case j.R.Space:case j.R.Enter:e.preventDefault(),null==(n=(t=e.target).click)||n.call(t),l({type:1}),null==(r=a.button)||r.focus()}}else switch(e.key){case j.R.Space:case j.R.Enter:e.preventDefault(),e.stopPropagation(),1===a.popoverState&&(null==d||d(a.buttonId)),l({type:0});break;case j.R.Escape:if(0!==a.popoverState)return null==d?void 0:d(a.buttonId);if(!u.current||(null==g?void 0:g.activeElement)&&!u.current.contains(g.activeElement))return;e.preventDefault(),e.stopPropagation(),l({type:1})}}),S=(0,z.z)(e=>{p||e.key===j.R.Space&&e.preventDefault()}),N=(0,z.z)(t=>{var n,r;(0,P.P)(t.currentTarget)||e.disabled||(p?(l({type:1}),null==(n=a.button)||n.focus()):(t.preventDefault(),t.stopPropagation(),1===a.popoverState&&(null==d||d(a.buttonId)),l({type:0}),null==(r=a.button)||r.focus()))}),C=(0,z.z)(e=>{e.preventDefault(),e.stopPropagation()}),M=0===a.popoverState,T=(0,h.useMemo)(()=>({open:M}),[M]),F=(0,k.f)(e,u),A=p?{ref:v,type:F,onKeyDown:b,onClick:N}:{ref:m,id:a.buttonId,type:F,"aria-expanded":e.disabled?void 0:0===a.popoverState,"aria-controls":a.panel?a.panelId:void 0,onKeyDown:b,onKeyUp:S,onClick:N,onMouseDown:C},L=V(),I=(0,z.z)(()=>{let e=a.panel;e&&(0,y.E)(L.current,{[Z.Forwards]:()=>R(e,O.First),[Z.Backwards]:()=>R(e,O.Last)})});return h.createElement(h.Fragment,null,(0,x.sY)({ourProps:A,theirProps:o,slot:T,defaultTag:"button",name:"Popover.Button"}),M&&!p&&i&&h.createElement(D,{id:s,features:B.Focusable,as:"button",type:"button",onFocus:I}))}),er=x.AN.RenderStrategy|x.AN.Static,eo=(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-overlay-${n}`,...o}=e,[{popoverState:a},l]=W("Popover.Overlay"),i=(0,w.T)(t),u=(0,L.oJ)(),s=null!==u?u===L.ZM.Open:0===a,c=(0,z.z)(e=>{if((0,P.P)(e.currentTarget))return e.preventDefault();l({type:1})}),d=(0,h.useMemo)(()=>({open:0===a}),[a]);return(0,x.sY)({ourProps:{ref:i,id:r,"aria-hidden":!0,onClick:c},theirProps:o,slot:d,defaultTag:"div",features:er,visible:s,name:"Popover.Overlay"})}),ea=x.AN.RenderStrategy|x.AN.Static,el=Object.assign(et,{Button:en,Overlay:eo,Panel:(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-panel-${n}`,focus:o=!1,...a}=e,[l,i]=W("Popover.Panel"),{close:u,isPortalled:s}=Y("Popover.Panel"),c=`headlessui-focus-sentinel-before-${(0,E.M)()}`,d=`headlessui-focus-sentinel-after-${(0,E.M)()}`,f=(0,h.useRef)(null),p=(0,w.T)(f,t,e=>{i({type:4,panel:e})}),m=_(f);(0,h.useEffect)(()=>(i({type:5,panelId:r}),()=>{i({type:5,panelId:null})}),[r,i]);let v=(0,L.oJ)(),g=null!==v?v===L.ZM.Open:0===l.popoverState,b=(0,z.z)(e=>{var t;if(e.key===j.R.Escape){if(0!==l.popoverState||!f.current||(null==m?void 0:m.activeElement)&&!f.current.contains(m.activeElement))return;e.preventDefault(),e.stopPropagation(),i({type:1}),null==(t=l.button)||t.focus()}});(0,h.useEffect)(()=>{var t;e.static||1===l.popoverState&&(null==(t=e.unmount)||t)&&i({type:4,panel:null})},[l.popoverState,e.unmount,e.static,i]),(0,h.useEffect)(()=>{if(!o||0!==l.popoverState||!f.current)return;let e=null==m?void 0:m.activeElement;f.current.contains(e)||R(f.current,O.First)},[o,f,l.popoverState]);let P=(0,h.useMemo)(()=>({open:0===l.popoverState,close:u}),[l,u]),S={ref:p,id:l.panelId,onKeyDown:b,onBlur:o&&0===l.popoverState?e=>{var t,n,r,o,a;let u=e.relatedTarget;!u||!f.current||null!=(t=f.current)&&t.contains(u)||(i({type:1}),((null==(r=null==(n=l.beforePanelSentinel.current)?void 0:n.contains)?void 0:r.call(n,u))||(null==(a=null==(o=l.afterPanelSentinel.current)?void 0:o.contains)?void 0:a.call(o,u)))&&u.focus({preventScroll:!0}))}:void 0,tabIndex:-1},N=V(),C=(0,z.z)(()=>{let e=f.current;e&&(0,y.E)(N.current,{[Z.Forwards]:()=>{R(e,O.First)},[Z.Backwards]:()=>{var e;null==(e=l.button)||e.focus({preventScroll:!0})}})}),M=(0,z.z)(()=>{let e=f.current;e&&(0,y.E)(N.current,{[Z.Forwards]:()=>{var e,t,n;if(!l.button)return;let r=T(),o=r.indexOf(l.button),a=r.slice(0,o+1),i=[...r.slice(o+1),...a];for(let u of i.slice())if((null==(t=null==(e=null==u?void 0:u.id)?void 0:e.startsWith)?void 0:t.call(e,"headlessui-focus-sentinel-"))||(null==(n=l.panel)?void 0:n.contains(u))){let s=i.indexOf(u);-1!==s&&i.splice(s,1)}R(i,O.First,!1)},[Z.Backwards]:()=>R(e,O.Last)})});return h.createElement(X.Provider,{value:l.panelId},g&&s&&h.createElement(D,{id:c,ref:l.beforePanelSentinel,features:B.Focusable,as:"button",type:"button",onFocus:C}),(0,x.sY)({ourProps:S,theirProps:a,slot:P,defaultTag:"div",features:ea,visible:g,name:"Popover.Panel"}),g&&s&&h.createElement(D,{id:d,ref:l.afterPanelSentinel,features:B.Focusable,as:"button",type:"button",onFocus:M}))}),Group:(0,x.yV)(function(e,t){let n=(0,h.useRef)(null),r=(0,w.T)(n,t),[o,a]=(0,h.useState)([]),l=(0,z.z)(e=>{a(t=>{let n=t.indexOf(e);if(-1!==n){let r=t.slice();return r.splice(n,1),r}return t})}),i=(0,z.z)(e=>(a(t=>[...t,e]),()=>l(e))),u=(0,z.z)(()=>{var e;let t=(0,S.r)(n);if(!t)return!1;let r=t.activeElement;return!!(null!=(e=n.current)&&e.contains(r))||o.some(e=>{var n,o;return(null==(n=t.getElementById(e.buttonId.current))?void 0:n.contains(r))||(null==(o=t.getElementById(e.panelId.current))?void 0:o.contains(r))})}),s=(0,z.z)(e=>{for(let t of o)t.buttonId.current!==e&&t.close()}),c=(0,h.useMemo)(()=>({registerPopover:i,unregisterPopover:l,isFocusWithinPopoverGroup:u,closeOthers:s}),[i,l,u,s]),d=(0,h.useMemo)(()=>({}),[]);return h.createElement(J.Provider,{value:c},(0,x.sY)({ourProps:{ref:r},theirProps:e,slot:d,defaultTag:"div",name:"Popover.Group"}))})});var ei=n(6723);function eu(){let e=(0,h.useRef)(!1);return(0,ei.e)(()=>(e.current=!0,()=>{e.current=!1}),[]),e}var es=n(2180);function ec(){let e=[],t=[],n={enqueue(e){t.push(e)},addEventListener:(e,t,r,o)=>(e.addEventListener(t,r,o),n.add(()=>e.removeEventListener(t,r,o))),requestAnimationFrame(...e){let t=requestAnimationFrame(...e);return n.add(()=>cancelAnimationFrame(t))},nextFrame:(...e)=>n.requestAnimationFrame(()=>n.requestAnimationFrame(...e)),setTimeout(...e){let t=setTimeout(...e);return n.add(()=>clearTimeout(t))},microTask(...e){var t;let r={current:!0};return t=()=>{r.current&&e[0]()},"function"==typeof queueMicrotask?queueMicrotask(t):Promise.resolve().then(t).catch(e=>setTimeout(()=>{throw e})),n.add(()=>{r.current=!1})},add:t=>(e.push(t),()=>{let n=e.indexOf(t);if(n>=0){let[r]=e.splice(n,1);r()}}),dispose(){for(let t of e.splice(0))t()},async workQueue(){for(let e of t.splice(0))await e()}};return n}function ed(e,...t){e&&t.length>0&&e.classList.add(...t)}function ef(e,...t){e&&t.length>0&&e.classList.remove(...t)}function ep(){let[e]=(0,h.useState)(ec);return(0,h.useEffect)(()=>()=>e.dispose(),[e]),e}function em(e=""){return e.split(" ").filter(e=>e.trim().length>1)}let ev=(0,h.createContext)(null);ev.displayName="TransitionContext";var eh=((d=eh||{}).Visible="visible",d.Hidden="hidden",d);let eg=(0,h.createContext)(null);function eb(e){return"children"in e?eb(e.children):e.current.filter(({el:e})=>null!==e.current).filter(({state:e})=>"visible"===e).length>0}function ey(e,t){let n=(0,I.E)(e),r=(0,h.useRef)([]),o=eu(),a=ep(),l=(0,z.z)((e,t=x.l4.Hidden)=>{let l=r.current.findIndex(({el:t})=>t===e);-1!==l&&((0,y.E)(t,{[x.l4.Unmount](){r.current.splice(l,1)},[x.l4.Hidden](){r.current[l].state="hidden"}}),a.microTask(()=>{var e;!eb(r)&&o.current&&(null==(e=n.current)||e.call(n))}))}),i=(0,z.z)(e=>{let t=r.current.find(({el:t})=>t===e);return t?"visible"!==t.state&&(t.state="visible"):r.current.push({el:e,state:"visible"}),()=>l(e,x.l4.Unmount)}),u=(0,h.useRef)([]),s=(0,h.useRef)(Promise.resolve()),c=(0,h.useRef)({enter:[],leave:[],idle:[]}),d=(0,z.z)((e,n,r)=>{u.current.splice(0),t&&(t.chains.current[n]=t.chains.current[n].filter(([t])=>t!==e)),null==t||t.chains.current[n].push([e,new Promise(e=>{u.current.push(e)})]),null==t||t.chains.current[n].push([e,new Promise(e=>{Promise.all(c.current[n].map(([e,t])=>t)).then(()=>e())})]),"enter"===n?s.current=s.current.then(()=>null==t?void 0:t.wait.current).then(()=>r(n)):r(n)}),f=(0,z.z)((e,t,n)=>{Promise.all(c.current[t].splice(0).map(([e,t])=>t)).then(()=>{var e;null==(e=u.current.shift())||e()}).then(()=>n(t))});return(0,h.useMemo)(()=>({children:r,register:i,unregister:l,onStart:d,onStop:f,wait:s,chains:c}),[i,l,r,d,f,c,s])}function ex(){}eg.displayName="NestingContext";let ew=["beforeEnter","afterEnter","beforeLeave","afterLeave"];function eE(e){var t;let n={};for(let r of ew)n[r]=null!=(t=e[r])?t:ex;return n}let ej=x.AN.RenderStrategy,eP=(0,x.yV)(function(e,t){var n;let r,{beforeEnter:o,afterEnter:a,beforeLeave:l,afterLeave:i,enter:u,enterFrom:s,enterTo:c,entered:d,leave:f,leaveFrom:p,leaveTo:m,...v}=e,g=(0,h.useRef)(null),b=(0,w.T)(g,t),E=v.unmount?x.l4.Unmount:x.l4.Hidden,{show:j,appear:P,initial:S}=function(){let e=(0,h.useContext)(ev);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),[N,O]=(0,h.useState)(j?"visible":"hidden"),C=function(){let e=(0,h.useContext)(eg);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),{register:M,unregister:T}=C,F=(0,h.useRef)(null);(0,h.useEffect)(()=>M(g),[M,g]),(0,h.useEffect)(()=>{if(E===x.l4.Hidden&&g.current){if(j&&"visible"!==N){O("visible");return}return(0,y.E)(N,{hidden:()=>T(g),visible:()=>M(g)})}},[N,g,M,T,j,E]);let A=(0,I.E)({enter:em(u),enterFrom:em(s),enterTo:em(c),entered:em(d),leave:em(f),leaveFrom:em(p),leaveTo:em(m)}),R=(n={beforeEnter:o,afterEnter:a,beforeLeave:l,afterLeave:i},r=(0,h.useRef)(eE(n)),(0,h.useEffect)(()=>{r.current=eE(n)},[n]),r),k=(0,es.H)();(0,h.useEffect)(()=>{if(k&&"visible"===N&&null===g.current)throw Error("Did you forget to passthrough the `ref` to the actual DOM node?")},[g,N,k]);let H=S&&!P,_=!k||H||F.current===j?"idle":j?"enter":"leave",B=(0,z.z)(e=>(0,y.E)(e,{enter:()=>R.current.beforeEnter(),leave:()=>R.current.beforeLeave(),idle:()=>{}})),D=(0,z.z)(e=>(0,y.E)(e,{enter:()=>R.current.afterEnter(),leave:()=>R.current.afterLeave(),idle:()=>{}})),Z=ey(()=>{O("hidden"),T(g)},C);(function({container:e,direction:t,classes:n,onStart:r,onStop:o}){let a=eu(),l=ep(),i=(0,I.E)(t);(0,ei.e)(()=>{let t=ec();l.add(t.dispose);let u=e.current;if(u&&"idle"!==i.current&&a.current){var s,c,d,f;let p,m,v,h,g,b,x;return t.dispose(),r.current(i.current),t.add((s=u,c=n.current,d="enter"===i.current,f=()=>{t.dispose(),o.current(i.current)},m=d?"enter":"leave",v=ec(),h=void 0!==f?(p={called:!1},(...e)=>{if(!p.called)return p.called=!0,f(...e)}):()=>{},"enter"===m&&(s.removeAttribute("hidden"),s.style.display=""),g=(0,y.E)(m,{enter:()=>c.enter,leave:()=>c.leave}),b=(0,y.E)(m,{enter:()=>c.enterTo,leave:()=>c.leaveTo}),x=(0,y.E)(m,{enter:()=>c.enterFrom,leave:()=>c.leaveFrom}),ef(s,...c.enter,...c.enterTo,...c.enterFrom,...c.leave,...c.leaveFrom,...c.leaveTo,...c.entered),ed(s,...g,...x),v.nextFrame(()=>{ef(s,...x),ed(s,...b),function(e,t){let n=ec();if(!e)return n.dispose;let{transitionDuration:r,transitionDelay:o}=getComputedStyle(e),[a,l]=[r,o].map(e=>{let[t=0]=e.split(",").filter(Boolean).map(e=>e.includes("ms")?parseFloat(e):1e3*parseFloat(e)).sort((e,t)=>t-e);return t});if(a+l!==0){let i=n.addEventListener(e,"transitionend",e=>{e.target===e.currentTarget&&(t(),i())})}else t();n.add(()=>t()),n.dispose}(s,()=>(ef(s,...g),ed(s,...c.entered),h()))}),v.dispose)),t.dispose}},[t])})({container:g,classes:A,direction:_,onStart:(0,I.E)(e=>{Z.onStart(g,e,B)}),onStop:(0,I.E)(e=>{Z.onStop(g,e,D),"leave"!==e||eb(Z)||(O("hidden"),T(g))})}),(0,h.useEffect)(()=>{H&&(E===x.l4.Hidden?F.current=null:F.current=j)},[j,H,N]);let V=v;return P&&j&&("undefined"==typeof window||"undefined"==typeof document)&&(V={...V,className:function(...e){return e.filter(Boolean).join(" ")}(v.className,...A.current.enter,...A.current.enterFrom)}),h.createElement(eg.Provider,{value:Z},h.createElement(L.up,{value:(0,y.E)(N,{visible:L.ZM.Open,hidden:L.ZM.Closed})},(0,x.sY)({ourProps:{ref:b},theirProps:V,defaultTag:"div",features:ej,visible:"visible"===N,name:"Transition.Child"})))}),eS=(0,x.yV)(function(e,t){let{show:n,appear:r=!1,unmount:o,...a}=e,l=(0,h.useRef)(null),i=(0,w.T)(l,t);(0,es.H)();let u=(0,L.oJ)();if(void 0===n&&null!==u&&(n=(0,y.E)(u,{[L.ZM.Open]:!0,[L.ZM.Closed]:!1})),![!0,!1].includes(n))throw Error("A is used but it is missing a `show={true | false}` prop.");let[s,c]=(0,h.useState)(n?"visible":"hidden"),d=ey(()=>{c("hidden")}),[f,p]=(0,h.useState)(!0),m=(0,h.useRef)([n]);(0,ei.e)(()=>{!1!==f&&m.current[m.current.length-1]!==n&&(m.current.push(n),p(!1))},[m,n]);let v=(0,h.useMemo)(()=>({show:n,appear:r,initial:f}),[n,r,f]);(0,h.useEffect)(()=>{if(n)c("visible");else if(eb(d)){let e=l.current;if(!e)return;let t=e.getBoundingClientRect();0===t.x&&0===t.y&&0===t.width&&0===t.height&&c("hidden")}else c("hidden")},[n,d]);let g={unmount:o};return h.createElement(eg.Provider,{value:d},h.createElement(ev.Provider,{value:v},(0,x.sY)({ourProps:{...g,as:h.Fragment,children:h.createElement(eP,{ref:i,...g,...a})},theirProps:{},defaultTag:h.Fragment,features:ej,visible:"visible"===s,name:"Transition"})))}),eN=(0,x.yV)(function(e,t){let n=null!==(0,h.useContext)(ev),r=null!==(0,L.oJ)();return h.createElement(h.Fragment,null,!n&&r?h.createElement(eS,{ref:t,...e}):h.createElement(eP,{ref:t,...e}))}),eO=Object.assign(eS,{Child:eN,Root:eS}),eC=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"}))}),eM=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M6 18L18 6M6 6l12 12"}))}),eT=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{fillRule:"evenodd",d:"M5.23 7.21a.75.75 0 011.06.02L10 11.168l3.71-3.938a.75.75 0 111.08 1.04l-4.25 4.5a.75.75 0 01-1.08 0l-4.25-4.5a.75.75 0 01.02-1.06z",clipRule:"evenodd"}))});var eF=n(5995),eA=n(8097);let eR=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{fillRule:"evenodd",d:"M13.5 4.938a7 7 0 11-9.006 1.737c.202-.257.59-.218.793.039.278.352.594.672.943.954.332.269.786-.049.773-.476a5.977 5.977 0 01.572-2.759 6.026 6.026 0 012.486-2.665c.247-.14.55-.016.677.238A6.967 6.967 0 0013.5 4.938zM14 12a4 4 0 01-4 4c-1.913 0-3.52-1.398-3.91-3.182-.093-.429.44-.643.814-.413a4.043 4.043 0 001.601.564c.303.038.531-.24.51-.544a5.975 5.975 0 011.315-4.192.447.447 0 01.431-.16A4.001 4.001 0 0114 12z",clipRule:"evenodd"}))});var eL=n(3707),ek=[{name:"Faster E2E",description:"Keep your team focused by slashing CI times with bare-metal performance.",href:"/#solutions",icon:eR},{name:"Completely private CI",description:"Bring your own machines or cloud instances, for full privacy.",href:"/#solutions",icon:eF.Z},{name:"ARM from dev to production",description:"Build with Apple M1/M2 locally, and in CI.",href:"/#solutions",icon:eL.Z},{name:"Live debugging",description:"Debug slow and tricky builds via SSH.",href:"/#solutions",icon:eA.Z}],eI=[{name:"Blog",href:"/blog"},{name:"Announcement",href:"/blog/blazing-fast-ci-with-microvms"},{name:"Pricing",href:"/pricing"},{name:"Docs",href:"https://docs.actuated.dev/"}];function eH(){for(var e=arguments.length,t=Array(e),n=0;nt.current(...e),[t])}},9946:function(e,t,n){"use strict";n.d(t,{M:function(){return s}});var r,o=n(7294),a=n(6723),l=n(2180);let i=0;function u(){return++i}let s=null!=(r=o.useId)?r:function(){let e=(0,l.H)(),[t,n]=o.useState(e?u:null);return(0,a.e)(()=>{null===t&&n(u())},[t]),null!=t?""+t:void 0}},6723:function(e,t,n){"use strict";n.d(t,{e:function(){return o}});var r=n(7294);let o=n(3393).s?r.useEffect:r.useLayoutEffect},3855:function(e,t,n){"use strict";n.d(t,{E:function(){return a}});var r=n(7294),o=n(6723);function a(e){let t=(0,r.useRef)(e);return(0,o.e)(()=>{t.current=e},[e]),t}},4157:function(e,t,n){"use strict";n.d(t,{f:function(){return l}});var r=n(7294),o=n(6723);function a(e){var t;if(e.type)return e.type;let n=null!=(t=e.as)?t:"button";if("string"==typeof n&&"button"===n.toLowerCase())return"button"}function l(e,t){let[n,l]=(0,r.useState)(()=>a(e));return(0,o.e)(()=>{l(a(e))},[e.type,e.as]),(0,o.e)(()=>{n||!t.current||t.current instanceof HTMLButtonElement&&!t.current.hasAttribute("type")&&l("button")},[n,t]),n}},2180:function(e,t,n){"use strict";n.d(t,{H:function(){return a}});var r=n(7294);let o={serverHandoffComplete:!1};function a(){let[e,t]=(0,r.useState)(o.serverHandoffComplete);return(0,r.useEffect)(()=>{!0!==e&&t(!0)},[e]),(0,r.useEffect)(()=>{!1===o.serverHandoffComplete&&(o.serverHandoffComplete=!0)},[]),e}},3784:function(e,t,n){"use strict";n.d(t,{T:function(){return i},h:function(){return l}});var r=n(7294),o=n(3781);let a=Symbol();function l(e,t=!0){return Object.assign(e,{[a]:t})}function i(...e){let t=(0,r.useRef)(e);(0,r.useEffect)(()=>{t.current=e},[e]);let n=(0,o.z)(e=>{for(let n of t.current)null!=n&&("function"==typeof n?n(e):n.current=e)});return e.every(e=>null==e||(null==e?void 0:e[a]))?void 0:n}},6567:function(e,t,n){"use strict";n.d(t,{ZM:function(){return l},oJ:function(){return i},up:function(){return u}});var r,o=n(7294);let a=(0,o.createContext)(null);a.displayName="OpenClosedContext";var l=((r=l||{})[r.Open=0]="Open",r[r.Closed=1]="Closed",r);function i(){return(0,o.useContext)(a)}function u({value:e,children:t}){return o.createElement(a.Provider,{value:e},t)}},4103:function(e,t,n){"use strict";function r(e){let t=e.parentElement,n=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(n=t),t=t.parentElement;let r=(null==t?void 0:t.getAttribute("disabled"))==="";return!(r&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(n))&&r}n.d(t,{P:function(){return r}})},2984:function(e,t,n){"use strict";function r(e,t,...n){if(e in t){let o=t[e];return"function"==typeof o?o(...n):o}let a=Error(`Tried to handle "${e}" but there is no handler defined. Only defined handlers are: ${Object.keys(t).map(e=>`"${e}"`).join(", ")}.`);throw Error.captureStackTrace&&Error.captureStackTrace(a,r),a}n.d(t,{E:function(){return r}})},5466:function(e,t,n){"use strict";n.d(t,{r:function(){return o}});var r=n(3393);function o(e){return r.s?null:e instanceof Node?e.ownerDocument:null!=e&&e.hasOwnProperty("current")&&e.current instanceof Node?e.current.ownerDocument:document}},2351:function(e,t,n){"use strict";n.d(t,{AN:function(){return i},l4:function(){return u},sY:function(){return s},yV:function(){return f}});var r,o,a=n(7294),l=n(2984),i=((r=i||{})[r.None=0]="None",r[r.RenderStrategy=1]="RenderStrategy",r[r.Static=2]="Static",r),u=((o=u||{})[o.Unmount=0]="Unmount",o[o.Hidden=1]="Hidden",o);function s({ourProps:e,theirProps:t,slot:n,defaultTag:r,features:o,visible:a=!0,name:i}){let u=d(t,e);if(a)return c(u,n,r,i);let s=null!=o?o:0;if(2&s){let{static:f=!1,...p}=u;if(f)return c(p,n,r,i)}if(1&s){let{unmount:m=!0,...v}=u;return(0,l.E)(m?0:1,{0:()=>null,1:()=>c({...v,hidden:!0,style:{display:"none"}},n,r,i)})}return c(u,n,r,i)}function c(e,t={},n,r){let{as:o=n,children:l,refName:i="ref",...u}=m(e,["unmount","static"]),s=void 0!==e.ref?{[i]:e.ref}:{},c="function"==typeof l?l(t):l;u.className&&"function"==typeof u.className&&(u.className=u.className(t));let f={};if(t){let v=!1,h=[];for(let[g,b]of Object.entries(t))"boolean"==typeof b&&(v=!0),!0===b&&h.push(g);v&&(f["data-headlessui-state"]=h.join(" "))}if(o===a.Fragment&&Object.keys(p(u)).length>0){if(!(0,a.isValidElement)(c)||Array.isArray(c)&&c.length>1)throw Error(['Passing props on "Fragment"!',"",`The current component <${r} /> is rendering a "Fragment".`,"However we need to passthrough the following props:",Object.keys(u).map(e=>` - ${e}`).join(` +(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[888],{1118:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/_app",function(){return n(1613)}])},227:function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getDomainLocale=function(e,t,n,r){return!1},("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},1551:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(4941).Z;n(5753).default,Object.defineProperty(t,"__esModule",{value:!0}),t.default=void 0;var o=n(2648).Z,a=n(7273).Z,l=o(n(7294)),i=n(1003),u=n(7795),s=n(4465),c=n(2692),d=n(8245),f=n(9246),p=n(227),m=n(3468),v=new Set;function h(e,t,n,r){if(i.isLocalURL(t)){if(!r.bypassPrefetchedCheck){var o=t+"%"+n+"%"+(void 0!==r.locale?r.locale:"locale"in e?e.locale:void 0);if(v.has(o))return;v.add(o)}Promise.resolve(e.prefetch(t,n,r)).catch(function(e){})}}function g(e){return"string"==typeof e?e:u.formatUrl(e)}var b=l.default.forwardRef(function(e,t){var n,o,u=e.href,v=e.as,b=e.children,y=e.prefetch,x=e.passHref,w=e.replace,E=e.shallow,j=e.scroll,P=e.locale,S=e.onClick,N=e.onMouseEnter,O=e.onTouchStart,C=e.legacyBehavior,M=void 0!==C&&C,T=a(e,["href","as","children","prefetch","passHref","replace","shallow","scroll","locale","onClick","onMouseEnter","onTouchStart","legacyBehavior"]);n=b,M&&("string"==typeof n||"number"==typeof n)&&(n=l.default.createElement("a",null,n));var F=!1!==y,A=l.default.useContext(c.RouterContext),R=l.default.useContext(d.AppRouterContext),L=null!=A?A:R,k=!A,I=l.default.useMemo(function(){if(!A){var e=g(u);return{href:e,as:v?g(v):e}}var t=r(i.resolveHref(A,u,!0),2),n=t[0],o=t[1];return{href:n,as:v?i.resolveHref(A,v):o||n}},[A,u,v]),H=I.href,_=I.as,B=l.default.useRef(H),D=l.default.useRef(_);M&&(o=l.default.Children.only(n));var z=M?o&&"object"==typeof o&&o.ref:t,Z=r(f.useIntersection({rootMargin:"200px"}),3),V=Z[0],U=Z[1],$=Z[2],q=l.default.useCallback(function(e){(D.current!==_||B.current!==H)&&($(),D.current=_,B.current=H),V(e),z&&("function"==typeof z?z(e):"object"==typeof z&&(z.current=e))},[_,z,H,$,V]);l.default.useEffect(function(){L&&U&&F&&h(L,H,_,{locale:P})},[_,H,U,P,F,null==A?void 0:A.locale,L]);var K={ref:q,onClick:function(e){M||"function"!=typeof S||S(e),M&&o.props&&"function"==typeof o.props.onClick&&o.props.onClick(e),L&&!e.defaultPrevented&&function(e,t,n,r,o,a,u,s,c,d){if("A"!==e.currentTarget.nodeName.toUpperCase()||(!(p=(f=e).currentTarget.target)||"_self"===p)&&!f.metaKey&&!f.ctrlKey&&!f.shiftKey&&!f.altKey&&(!f.nativeEvent||2!==f.nativeEvent.which)&&i.isLocalURL(n)){e.preventDefault();var f,p,m=function(){"beforePopState"in t?t[o?"replace":"push"](n,r,{shallow:a,locale:s,scroll:u}):t[o?"replace":"push"](r||n,{forceOptimisticNavigation:!d})};c?l.default.startTransition(m):m()}}(e,L,H,_,w,E,j,P,k,F)},onMouseEnter:function(e){M||"function"!=typeof N||N(e),M&&o.props&&"function"==typeof o.props.onMouseEnter&&o.props.onMouseEnter(e),L&&(F||!k)&&h(L,H,_,{locale:P,priority:!0,bypassPrefetchedCheck:!0})},onTouchStart:function(e){M||"function"!=typeof O||O(e),M&&o.props&&"function"==typeof o.props.onTouchStart&&o.props.onTouchStart(e),L&&(F||!k)&&h(L,H,_,{locale:P,priority:!0,bypassPrefetchedCheck:!0})}};if(!M||x||"a"===o.type&&!("href"in o.props)){var W=void 0!==P?P:null==A?void 0:A.locale,G=(null==A?void 0:A.isLocaleDomain)&&p.getDomainLocale(_,W,null==A?void 0:A.locales,null==A?void 0:A.domainLocales);K.href=G||m.addBasePath(s.addLocale(_,W,null==A?void 0:A.defaultLocale))}return M?l.default.cloneElement(o,K):l.default.createElement("a",Object.assign({},T,K),n)});t.default=b,("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},9246:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(4941).Z;Object.defineProperty(t,"__esModule",{value:!0}),t.useIntersection=function(e){var t=e.rootRef,n=e.rootMargin,s=e.disabled||!l,c=r(o.useState(!1),2),d=c[0],f=c[1],p=r(o.useState(null),2),m=p[0],v=p[1];return o.useEffect(function(){if(l){if(!s&&!d&&m&&m.tagName){var e,r,o,c;return r=(e=function(e){var t,n={root:e.root||null,margin:e.rootMargin||""},r=u.find(function(e){return e.root===n.root&&e.margin===n.margin});if(r&&(t=i.get(r)))return t;var o=new Map;return t={id:n,observer:new IntersectionObserver(function(e){e.forEach(function(e){var t=o.get(e.target),n=e.isIntersecting||e.intersectionRatio>0;t&&n&&t(n)})},e),elements:o},u.push(n),i.set(n,t),t}({root:null==t?void 0:t.current,rootMargin:n})).id,o=e.observer,(c=e.elements).set(m,function(e){return e&&f(e)}),o.observe(m),function(){if(c.delete(m),o.unobserve(m),0===c.size){o.disconnect(),i.delete(r);var e=u.findIndex(function(e){return e.root===r.root&&e.margin===r.margin});e>-1&&u.splice(e,1)}}}}else if(!d){var p=a.requestIdleCallback(function(){return f(!0)});return function(){return a.cancelIdleCallback(p)}}},[m,s,n,t,d]),[v,d,o.useCallback(function(){f(!1)},[])]};var o=n(7294),a=n(4686),l="function"==typeof IntersectionObserver,i=new Map,u=[];("function"==typeof t.default||"object"==typeof t.default&&null!==t.default)&&void 0===t.default.__esModule&&(Object.defineProperty(t.default,"__esModule",{value:!0}),Object.assign(t.default,t),e.exports=t.default)},1613:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return e$},reportWebVitals:function(){return eU}});var r,o,a,l,i,u,s,c,d,f=n(1799),p=n(5893),m=n(9008),v=n.n(m),h=n(7294),g=n(1664),b=n.n(g),y=n(2984),x=n(2351),w=n(3784),E=n(9946),j=n(1363),P=n(4103),S=n(5466);let N=["[contentEditable=true]","[tabindex]","a[href]","area[href]","button:not([disabled])","iframe","input:not([disabled])","select:not([disabled])","textarea:not([disabled])"].map(e=>`${e}:not([tabindex='-1'])`).join(",");var O=((r=O||{})[r.First=1]="First",r[r.Previous=2]="Previous",r[r.Next=4]="Next",r[r.Last=8]="Last",r[r.WrapAround=16]="WrapAround",r[r.NoScroll=32]="NoScroll",r),C=((o=C||{})[o.Error=0]="Error",o[o.Overflow=1]="Overflow",o[o.Success=2]="Success",o[o.Underflow=3]="Underflow",o),M=((a=M||{})[a.Previous=-1]="Previous",a[a.Next=1]="Next",a);function T(e=document.body){return null==e?[]:Array.from(e.querySelectorAll(N))}var F=((l=F||{})[l.Strict=0]="Strict",l[l.Loose=1]="Loose",l);function A(e,t=0){var n;return e!==(null==(n=(0,S.r)(e))?void 0:n.body)&&(0,y.E)(t,{0:()=>e.matches(N),1(){let t=e;for(;null!==t;){if(t.matches(N))return!0;t=t.parentElement}return!1}})}function R(e,t,n=!0,r=null){var o,a,l;let i=Array.isArray(e)?e.length>0?e[0].ownerDocument:document:e.ownerDocument,u=Array.isArray(e)?n?function(e,t=e=>e){return e.slice().sort((e,n)=>{let r=t(e),o=t(n);if(null===r||null===o)return 0;let a=r.compareDocumentPosition(o);return a&Node.DOCUMENT_POSITION_FOLLOWING?-1:a&Node.DOCUMENT_POSITION_PRECEDING?1:0})}(e):e:T(e);r=null!=r?r:i.activeElement;let s=(()=>{if(5&t)return 1;if(10&t)return -1;throw Error("Missing Focus.First, Focus.Previous, Focus.Next or Focus.Last")})(),c=(()=>{if(1&t)return 0;if(2&t)return Math.max(0,u.indexOf(r))-1;if(4&t)return Math.max(0,u.indexOf(r))+1;if(8&t)return u.length-1;throw Error("Missing Focus.First, Focus.Previous, Focus.Next or Focus.Last")})(),d=32&t?{preventScroll:!0}:{},f=0,p=u.length,m;do{if(f>=p||f+p<=0)return 0;let v=c+f;if(16&t)v=(v+p)%p;else{if(v<0)return 3;if(v>=p)return 1}null==(m=u[v])||m.focus(d),f+=s}while(m!==i.activeElement);return 6&t&&null!=(l=null==(a=null==(o=m)?void 0:o.matches)?void 0:a.call(o,"textarea,input"))&&l&&m.select(),m.hasAttribute("tabindex")||m.setAttribute("tabindex","0"),2}var L=n(6567),k=n(4157),I=n(3855);function H(e,t,n){let r=(0,I.E)(t);(0,h.useEffect)(()=>{function t(e){r.current(e)}return document.addEventListener(e,t,n),()=>document.removeEventListener(e,t,n)},[e,n])}function _(...e){return(0,h.useMemo)(()=>(0,S.r)(...e),[...e])}var B=((i=B||{})[i.None=1]="None",i[i.Focusable=2]="Focusable",i[i.Hidden=4]="Hidden",i);let D=(0,x.yV)(function(e,t){let{features:n=1,...r}=e,o={ref:t,"aria-hidden":(2&n)==2||void 0,style:{position:"fixed",top:1,left:1,width:1,height:0,padding:0,margin:-1,overflow:"hidden",clip:"rect(0, 0, 0, 0)",whiteSpace:"nowrap",borderWidth:"0",...(4&n)==4&&(2&n)!=2&&{display:"none"}}};return(0,x.sY)({ourProps:o,theirProps:r,slot:{},defaultTag:"div",name:"Hidden"})});var z=n(3781),Z=((u=Z||{})[u.Forwards=0]="Forwards",u[u.Backwards=1]="Backwards",u);function V(){var e,t;let n,r=(0,h.useRef)(0);return e="keydown",t=e=>{"Tab"===e.key&&(r.current=e.shiftKey?1:0)},n=(0,I.E)(t),(0,h.useEffect)(()=>{function t(e){n.current(e)}return window.addEventListener(e,t,!0),()=>window.removeEventListener(e,t,!0)},[e,!0]),r}var U=((s=U||{})[s.Open=0]="Open",s[s.Closed=1]="Closed",s),$=((c=$||{})[c.TogglePopover=0]="TogglePopover",c[c.ClosePopover=1]="ClosePopover",c[c.SetButton=2]="SetButton",c[c.SetButtonId=3]="SetButtonId",c[c.SetPanel=4]="SetPanel",c[c.SetPanelId=5]="SetPanelId",c);let q={0:e=>({...e,popoverState:(0,y.E)(e.popoverState,{0:1,1:0})}),1:e=>1===e.popoverState?e:{...e,popoverState:1},2:(e,t)=>e.button===t.button?e:{...e,button:t.button},3:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},4:(e,t)=>e.panel===t.panel?e:{...e,panel:t.panel},5:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId}},K=(0,h.createContext)(null);function W(e){let t=(0,h.useContext)(K);if(null===t){let n=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(n,W),n}return t}K.displayName="PopoverContext";let G=(0,h.createContext)(null);function Y(e){let t=(0,h.useContext)(G);if(null===t){let n=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(n,Y),n}return t}G.displayName="PopoverAPIContext";let J=(0,h.createContext)(null);function Q(){return(0,h.useContext)(J)}J.displayName="PopoverGroupContext";let X=(0,h.createContext)(null);function ee(e,t){return(0,y.E)(t.type,q,e,t)}X.displayName="PopoverPanelContext";let et=(0,x.yV)(function(e,t){var n,r,o,a;let l;let i=(0,h.useRef)(null),u=(0,w.T)(t,(0,w.h)(e=>{i.current=e})),s=(0,h.useReducer)(ee,{popoverState:1,buttons:[],button:null,buttonId:null,panel:null,panelId:null,beforePanelSentinel:(0,h.createRef)(),afterPanelSentinel:(0,h.createRef)()}),[{popoverState:c,button:d,buttonId:f,panel:p,panelId:m,beforePanelSentinel:v,afterPanelSentinel:g},b]=s,E=_(null!=(n=i.current)?n:d),j=(0,h.useMemo)(()=>{if(!d||!p)return!1;for(let e of document.querySelectorAll("body > *"))if(Number(null==e?void 0:e.contains(d))^Number(null==e?void 0:e.contains(p)))return!0;let t=T(),n=t.indexOf(d),r=(n+t.length-1)%t.length,o=(n+1)%t.length,a=t[r],l=t[o];return!p.contains(a)&&!p.contains(l)},[d,p]),P=(0,I.E)(f),S=(0,I.E)(m),N=(0,h.useMemo)(()=>({buttonId:P,panelId:S,close:()=>b({type:1})}),[P,S,b]),O=Q(),C=null==O?void 0:O.registerPopover,M=(0,z.z)(()=>{var e;return null!=(e=null==O?void 0:O.isFocusWithinPopoverGroup())?e:(null==E?void 0:E.activeElement)&&((null==d?void 0:d.contains(E.activeElement))||(null==p?void 0:p.contains(E.activeElement)))});(0,h.useEffect)(()=>null==C?void 0:C(N),[C,N]),r=null==E?void 0:E.defaultView,o="focus",a=e=>{var t,n,r,o;0===c&&(M()||!d||!p||e.target!==window&&(null!=(n=null==(t=v.current)?void 0:t.contains)&&n.call(t,e.target)||null!=(o=null==(r=g.current)?void 0:r.contains)&&o.call(r,e.target)||b({type:1})))},l=(0,I.E)(a),(0,h.useEffect)(()=>{function e(e){l.current(e)}return(r=null!=r?r:window).addEventListener(o,e,!0),()=>r.removeEventListener(o,e,!0)},[r,o,!0]),function(e,t,n=!0){let r=(0,h.useRef)(!1);function o(n,o){if(!r.current||n.defaultPrevented)return;let a=function e(t){return"function"==typeof t?e(t()):Array.isArray(t)||t instanceof Set?t:[t]}(e),l=o(n);if(null!==l&&l.getRootNode().contains(l)){for(let i of a){if(null===i)continue;let u=i instanceof HTMLElement?i:i.current;if(null!=u&&u.contains(l)||n.composed&&n.composedPath().includes(u))return}return A(l,F.Loose)||-1===l.tabIndex||n.preventDefault(),t(n,l)}}(0,h.useEffect)(()=>{requestAnimationFrame(()=>{r.current=n})},[n]);let a=(0,h.useRef)(null);H("mousedown",e=>{var t,n;r.current&&(a.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target)},!0),H("click",e=>{a.current&&(o(e,()=>a.current),a.current=null)},!0),H("blur",e=>o(e,()=>window.document.activeElement instanceof HTMLIFrameElement?window.document.activeElement:null),!0)}([d,p],(e,t)=>{b({type:1}),A(t,F.Loose)||(e.preventDefault(),null==d||d.focus())},0===c);let R=(0,z.z)(e=>{b({type:1});let t=e?e instanceof HTMLElement?e:"current"in e&&e.current instanceof HTMLElement?e.current:d:d;null==t||t.focus()}),k=(0,h.useMemo)(()=>({close:R,isPortalled:j}),[R,j]),B=(0,h.useMemo)(()=>({open:0===c,close:R}),[c,R]);return h.createElement(K.Provider,{value:s},h.createElement(G.Provider,{value:k},h.createElement(L.up,{value:(0,y.E)(c,{0:L.ZM.Open,1:L.ZM.Closed})},(0,x.sY)({ourProps:{ref:u},theirProps:e,slot:B,defaultTag:"div",name:"Popover"}))))}),en=(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-button-${n}`,...o}=e,[a,l]=W("Popover.Button"),{isPortalled:i}=Y("Popover.Button"),u=(0,h.useRef)(null),s=`headlessui-focus-sentinel-${(0,E.M)()}`,c=Q(),d=null==c?void 0:c.closeOthers,f=(0,h.useContext)(X),p=null!==f&&f===a.panelId;(0,h.useEffect)(()=>{if(!p)return l({type:3,buttonId:r}),()=>{l({type:3,buttonId:null})}},[r,l]);let m=(0,w.T)(u,t,p?null:e=>{if(e)a.buttons.push(r);else{let t=a.buttons.indexOf(r);-1!==t&&a.buttons.splice(t,1)}a.buttons.length>1&&console.warn("You are already using a but only 1 is supported."),e&&l({type:2,button:e})}),v=(0,w.T)(u,t),g=_(u),b=(0,z.z)(e=>{var t,n,r;if(p){if(1===a.popoverState)return;switch(e.key){case j.R.Space:case j.R.Enter:e.preventDefault(),null==(n=(t=e.target).click)||n.call(t),l({type:1}),null==(r=a.button)||r.focus()}}else switch(e.key){case j.R.Space:case j.R.Enter:e.preventDefault(),e.stopPropagation(),1===a.popoverState&&(null==d||d(a.buttonId)),l({type:0});break;case j.R.Escape:if(0!==a.popoverState)return null==d?void 0:d(a.buttonId);if(!u.current||(null==g?void 0:g.activeElement)&&!u.current.contains(g.activeElement))return;e.preventDefault(),e.stopPropagation(),l({type:1})}}),S=(0,z.z)(e=>{p||e.key===j.R.Space&&e.preventDefault()}),N=(0,z.z)(t=>{var n,r;(0,P.P)(t.currentTarget)||e.disabled||(p?(l({type:1}),null==(n=a.button)||n.focus()):(t.preventDefault(),t.stopPropagation(),1===a.popoverState&&(null==d||d(a.buttonId)),l({type:0}),null==(r=a.button)||r.focus()))}),C=(0,z.z)(e=>{e.preventDefault(),e.stopPropagation()}),M=0===a.popoverState,T=(0,h.useMemo)(()=>({open:M}),[M]),F=(0,k.f)(e,u),A=p?{ref:v,type:F,onKeyDown:b,onClick:N}:{ref:m,id:a.buttonId,type:F,"aria-expanded":e.disabled?void 0:0===a.popoverState,"aria-controls":a.panel?a.panelId:void 0,onKeyDown:b,onKeyUp:S,onClick:N,onMouseDown:C},L=V(),I=(0,z.z)(()=>{let e=a.panel;e&&(0,y.E)(L.current,{[Z.Forwards]:()=>R(e,O.First),[Z.Backwards]:()=>R(e,O.Last)})});return h.createElement(h.Fragment,null,(0,x.sY)({ourProps:A,theirProps:o,slot:T,defaultTag:"button",name:"Popover.Button"}),M&&!p&&i&&h.createElement(D,{id:s,features:B.Focusable,as:"button",type:"button",onFocus:I}))}),er=x.AN.RenderStrategy|x.AN.Static,eo=(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-overlay-${n}`,...o}=e,[{popoverState:a},l]=W("Popover.Overlay"),i=(0,w.T)(t),u=(0,L.oJ)(),s=null!==u?u===L.ZM.Open:0===a,c=(0,z.z)(e=>{if((0,P.P)(e.currentTarget))return e.preventDefault();l({type:1})}),d=(0,h.useMemo)(()=>({open:0===a}),[a]);return(0,x.sY)({ourProps:{ref:i,id:r,"aria-hidden":!0,onClick:c},theirProps:o,slot:d,defaultTag:"div",features:er,visible:s,name:"Popover.Overlay"})}),ea=x.AN.RenderStrategy|x.AN.Static,el=Object.assign(et,{Button:en,Overlay:eo,Panel:(0,x.yV)(function(e,t){let n=(0,E.M)(),{id:r=`headlessui-popover-panel-${n}`,focus:o=!1,...a}=e,[l,i]=W("Popover.Panel"),{close:u,isPortalled:s}=Y("Popover.Panel"),c=`headlessui-focus-sentinel-before-${(0,E.M)()}`,d=`headlessui-focus-sentinel-after-${(0,E.M)()}`,f=(0,h.useRef)(null),p=(0,w.T)(f,t,e=>{i({type:4,panel:e})}),m=_(f);(0,h.useEffect)(()=>(i({type:5,panelId:r}),()=>{i({type:5,panelId:null})}),[r,i]);let v=(0,L.oJ)(),g=null!==v?v===L.ZM.Open:0===l.popoverState,b=(0,z.z)(e=>{var t;if(e.key===j.R.Escape){if(0!==l.popoverState||!f.current||(null==m?void 0:m.activeElement)&&!f.current.contains(m.activeElement))return;e.preventDefault(),e.stopPropagation(),i({type:1}),null==(t=l.button)||t.focus()}});(0,h.useEffect)(()=>{var t;e.static||1===l.popoverState&&(null==(t=e.unmount)||t)&&i({type:4,panel:null})},[l.popoverState,e.unmount,e.static,i]),(0,h.useEffect)(()=>{if(!o||0!==l.popoverState||!f.current)return;let e=null==m?void 0:m.activeElement;f.current.contains(e)||R(f.current,O.First)},[o,f,l.popoverState]);let P=(0,h.useMemo)(()=>({open:0===l.popoverState,close:u}),[l,u]),S={ref:p,id:l.panelId,onKeyDown:b,onBlur:o&&0===l.popoverState?e=>{var t,n,r,o,a;let u=e.relatedTarget;!u||!f.current||null!=(t=f.current)&&t.contains(u)||(i({type:1}),((null==(r=null==(n=l.beforePanelSentinel.current)?void 0:n.contains)?void 0:r.call(n,u))||(null==(a=null==(o=l.afterPanelSentinel.current)?void 0:o.contains)?void 0:a.call(o,u)))&&u.focus({preventScroll:!0}))}:void 0,tabIndex:-1},N=V(),C=(0,z.z)(()=>{let e=f.current;e&&(0,y.E)(N.current,{[Z.Forwards]:()=>{R(e,O.First)},[Z.Backwards]:()=>{var e;null==(e=l.button)||e.focus({preventScroll:!0})}})}),M=(0,z.z)(()=>{let e=f.current;e&&(0,y.E)(N.current,{[Z.Forwards]:()=>{var e,t,n;if(!l.button)return;let r=T(),o=r.indexOf(l.button),a=r.slice(0,o+1),i=[...r.slice(o+1),...a];for(let u of i.slice())if((null==(t=null==(e=null==u?void 0:u.id)?void 0:e.startsWith)?void 0:t.call(e,"headlessui-focus-sentinel-"))||(null==(n=l.panel)?void 0:n.contains(u))){let s=i.indexOf(u);-1!==s&&i.splice(s,1)}R(i,O.First,!1)},[Z.Backwards]:()=>R(e,O.Last)})});return h.createElement(X.Provider,{value:l.panelId},g&&s&&h.createElement(D,{id:c,ref:l.beforePanelSentinel,features:B.Focusable,as:"button",type:"button",onFocus:C}),(0,x.sY)({ourProps:S,theirProps:a,slot:P,defaultTag:"div",features:ea,visible:g,name:"Popover.Panel"}),g&&s&&h.createElement(D,{id:d,ref:l.afterPanelSentinel,features:B.Focusable,as:"button",type:"button",onFocus:M}))}),Group:(0,x.yV)(function(e,t){let n=(0,h.useRef)(null),r=(0,w.T)(n,t),[o,a]=(0,h.useState)([]),l=(0,z.z)(e=>{a(t=>{let n=t.indexOf(e);if(-1!==n){let r=t.slice();return r.splice(n,1),r}return t})}),i=(0,z.z)(e=>(a(t=>[...t,e]),()=>l(e))),u=(0,z.z)(()=>{var e;let t=(0,S.r)(n);if(!t)return!1;let r=t.activeElement;return!!(null!=(e=n.current)&&e.contains(r))||o.some(e=>{var n,o;return(null==(n=t.getElementById(e.buttonId.current))?void 0:n.contains(r))||(null==(o=t.getElementById(e.panelId.current))?void 0:o.contains(r))})}),s=(0,z.z)(e=>{for(let t of o)t.buttonId.current!==e&&t.close()}),c=(0,h.useMemo)(()=>({registerPopover:i,unregisterPopover:l,isFocusWithinPopoverGroup:u,closeOthers:s}),[i,l,u,s]),d=(0,h.useMemo)(()=>({}),[]);return h.createElement(J.Provider,{value:c},(0,x.sY)({ourProps:{ref:r},theirProps:e,slot:d,defaultTag:"div",name:"Popover.Group"}))})});var ei=n(6723);function eu(){let e=(0,h.useRef)(!1);return(0,ei.e)(()=>(e.current=!0,()=>{e.current=!1}),[]),e}var es=n(2180);function ec(){let e=[],t=[],n={enqueue(e){t.push(e)},addEventListener:(e,t,r,o)=>(e.addEventListener(t,r,o),n.add(()=>e.removeEventListener(t,r,o))),requestAnimationFrame(...e){let t=requestAnimationFrame(...e);return n.add(()=>cancelAnimationFrame(t))},nextFrame:(...e)=>n.requestAnimationFrame(()=>n.requestAnimationFrame(...e)),setTimeout(...e){let t=setTimeout(...e);return n.add(()=>clearTimeout(t))},microTask(...e){var t;let r={current:!0};return t=()=>{r.current&&e[0]()},"function"==typeof queueMicrotask?queueMicrotask(t):Promise.resolve().then(t).catch(e=>setTimeout(()=>{throw e})),n.add(()=>{r.current=!1})},add:t=>(e.push(t),()=>{let n=e.indexOf(t);if(n>=0){let[r]=e.splice(n,1);r()}}),dispose(){for(let t of e.splice(0))t()},async workQueue(){for(let e of t.splice(0))await e()}};return n}function ed(e,...t){e&&t.length>0&&e.classList.add(...t)}function ef(e,...t){e&&t.length>0&&e.classList.remove(...t)}function ep(){let[e]=(0,h.useState)(ec);return(0,h.useEffect)(()=>()=>e.dispose(),[e]),e}function em(e=""){return e.split(" ").filter(e=>e.trim().length>1)}let ev=(0,h.createContext)(null);ev.displayName="TransitionContext";var eh=((d=eh||{}).Visible="visible",d.Hidden="hidden",d);let eg=(0,h.createContext)(null);function eb(e){return"children"in e?eb(e.children):e.current.filter(({el:e})=>null!==e.current).filter(({state:e})=>"visible"===e).length>0}function ey(e,t){let n=(0,I.E)(e),r=(0,h.useRef)([]),o=eu(),a=ep(),l=(0,z.z)((e,t=x.l4.Hidden)=>{let l=r.current.findIndex(({el:t})=>t===e);-1!==l&&((0,y.E)(t,{[x.l4.Unmount](){r.current.splice(l,1)},[x.l4.Hidden](){r.current[l].state="hidden"}}),a.microTask(()=>{var e;!eb(r)&&o.current&&(null==(e=n.current)||e.call(n))}))}),i=(0,z.z)(e=>{let t=r.current.find(({el:t})=>t===e);return t?"visible"!==t.state&&(t.state="visible"):r.current.push({el:e,state:"visible"}),()=>l(e,x.l4.Unmount)}),u=(0,h.useRef)([]),s=(0,h.useRef)(Promise.resolve()),c=(0,h.useRef)({enter:[],leave:[],idle:[]}),d=(0,z.z)((e,n,r)=>{u.current.splice(0),t&&(t.chains.current[n]=t.chains.current[n].filter(([t])=>t!==e)),null==t||t.chains.current[n].push([e,new Promise(e=>{u.current.push(e)})]),null==t||t.chains.current[n].push([e,new Promise(e=>{Promise.all(c.current[n].map(([e,t])=>t)).then(()=>e())})]),"enter"===n?s.current=s.current.then(()=>null==t?void 0:t.wait.current).then(()=>r(n)):r(n)}),f=(0,z.z)((e,t,n)=>{Promise.all(c.current[t].splice(0).map(([e,t])=>t)).then(()=>{var e;null==(e=u.current.shift())||e()}).then(()=>n(t))});return(0,h.useMemo)(()=>({children:r,register:i,unregister:l,onStart:d,onStop:f,wait:s,chains:c}),[i,l,r,d,f,c,s])}function ex(){}eg.displayName="NestingContext";let ew=["beforeEnter","afterEnter","beforeLeave","afterLeave"];function eE(e){var t;let n={};for(let r of ew)n[r]=null!=(t=e[r])?t:ex;return n}let ej=x.AN.RenderStrategy,eP=(0,x.yV)(function(e,t){var n;let r,{beforeEnter:o,afterEnter:a,beforeLeave:l,afterLeave:i,enter:u,enterFrom:s,enterTo:c,entered:d,leave:f,leaveFrom:p,leaveTo:m,...v}=e,g=(0,h.useRef)(null),b=(0,w.T)(g,t),E=v.unmount?x.l4.Unmount:x.l4.Hidden,{show:j,appear:P,initial:S}=function(){let e=(0,h.useContext)(ev);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),[N,O]=(0,h.useState)(j?"visible":"hidden"),C=function(){let e=(0,h.useContext)(eg);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),{register:M,unregister:T}=C,F=(0,h.useRef)(null);(0,h.useEffect)(()=>M(g),[M,g]),(0,h.useEffect)(()=>{if(E===x.l4.Hidden&&g.current){if(j&&"visible"!==N){O("visible");return}return(0,y.E)(N,{hidden:()=>T(g),visible:()=>M(g)})}},[N,g,M,T,j,E]);let A=(0,I.E)({enter:em(u),enterFrom:em(s),enterTo:em(c),entered:em(d),leave:em(f),leaveFrom:em(p),leaveTo:em(m)}),R=(n={beforeEnter:o,afterEnter:a,beforeLeave:l,afterLeave:i},r=(0,h.useRef)(eE(n)),(0,h.useEffect)(()=>{r.current=eE(n)},[n]),r),k=(0,es.H)();(0,h.useEffect)(()=>{if(k&&"visible"===N&&null===g.current)throw Error("Did you forget to passthrough the `ref` to the actual DOM node?")},[g,N,k]);let H=S&&!P,_=!k||H||F.current===j?"idle":j?"enter":"leave",B=(0,z.z)(e=>(0,y.E)(e,{enter:()=>R.current.beforeEnter(),leave:()=>R.current.beforeLeave(),idle:()=>{}})),D=(0,z.z)(e=>(0,y.E)(e,{enter:()=>R.current.afterEnter(),leave:()=>R.current.afterLeave(),idle:()=>{}})),Z=ey(()=>{O("hidden"),T(g)},C);(function({container:e,direction:t,classes:n,onStart:r,onStop:o}){let a=eu(),l=ep(),i=(0,I.E)(t);(0,ei.e)(()=>{let t=ec();l.add(t.dispose);let u=e.current;if(u&&"idle"!==i.current&&a.current){var s,c,d,f;let p,m,v,h,g,b,x;return t.dispose(),r.current(i.current),t.add((s=u,c=n.current,d="enter"===i.current,f=()=>{t.dispose(),o.current(i.current)},m=d?"enter":"leave",v=ec(),h=void 0!==f?(p={called:!1},(...e)=>{if(!p.called)return p.called=!0,f(...e)}):()=>{},"enter"===m&&(s.removeAttribute("hidden"),s.style.display=""),g=(0,y.E)(m,{enter:()=>c.enter,leave:()=>c.leave}),b=(0,y.E)(m,{enter:()=>c.enterTo,leave:()=>c.leaveTo}),x=(0,y.E)(m,{enter:()=>c.enterFrom,leave:()=>c.leaveFrom}),ef(s,...c.enter,...c.enterTo,...c.enterFrom,...c.leave,...c.leaveFrom,...c.leaveTo,...c.entered),ed(s,...g,...x),v.nextFrame(()=>{ef(s,...x),ed(s,...b),function(e,t){let n=ec();if(!e)return n.dispose;let{transitionDuration:r,transitionDelay:o}=getComputedStyle(e),[a,l]=[r,o].map(e=>{let[t=0]=e.split(",").filter(Boolean).map(e=>e.includes("ms")?parseFloat(e):1e3*parseFloat(e)).sort((e,t)=>t-e);return t});if(a+l!==0){let i=n.addEventListener(e,"transitionend",e=>{e.target===e.currentTarget&&(t(),i())})}else t();n.add(()=>t()),n.dispose}(s,()=>(ef(s,...g),ed(s,...c.entered),h()))}),v.dispose)),t.dispose}},[t])})({container:g,classes:A,direction:_,onStart:(0,I.E)(e=>{Z.onStart(g,e,B)}),onStop:(0,I.E)(e=>{Z.onStop(g,e,D),"leave"!==e||eb(Z)||(O("hidden"),T(g))})}),(0,h.useEffect)(()=>{H&&(E===x.l4.Hidden?F.current=null:F.current=j)},[j,H,N]);let V=v;return P&&j&&("undefined"==typeof window||"undefined"==typeof document)&&(V={...V,className:function(...e){return e.filter(Boolean).join(" ")}(v.className,...A.current.enter,...A.current.enterFrom)}),h.createElement(eg.Provider,{value:Z},h.createElement(L.up,{value:(0,y.E)(N,{visible:L.ZM.Open,hidden:L.ZM.Closed})},(0,x.sY)({ourProps:{ref:b},theirProps:V,defaultTag:"div",features:ej,visible:"visible"===N,name:"Transition.Child"})))}),eS=(0,x.yV)(function(e,t){let{show:n,appear:r=!1,unmount:o,...a}=e,l=(0,h.useRef)(null),i=(0,w.T)(l,t);(0,es.H)();let u=(0,L.oJ)();if(void 0===n&&null!==u&&(n=(0,y.E)(u,{[L.ZM.Open]:!0,[L.ZM.Closed]:!1})),![!0,!1].includes(n))throw Error("A is used but it is missing a `show={true | false}` prop.");let[s,c]=(0,h.useState)(n?"visible":"hidden"),d=ey(()=>{c("hidden")}),[f,p]=(0,h.useState)(!0),m=(0,h.useRef)([n]);(0,ei.e)(()=>{!1!==f&&m.current[m.current.length-1]!==n&&(m.current.push(n),p(!1))},[m,n]);let v=(0,h.useMemo)(()=>({show:n,appear:r,initial:f}),[n,r,f]);(0,h.useEffect)(()=>{if(n)c("visible");else if(eb(d)){let e=l.current;if(!e)return;let t=e.getBoundingClientRect();0===t.x&&0===t.y&&0===t.width&&0===t.height&&c("hidden")}else c("hidden")},[n,d]);let g={unmount:o};return h.createElement(eg.Provider,{value:d},h.createElement(ev.Provider,{value:v},(0,x.sY)({ourProps:{...g,as:h.Fragment,children:h.createElement(eP,{ref:i,...g,...a})},theirProps:{},defaultTag:h.Fragment,features:ej,visible:"visible"===s,name:"Transition"})))}),eN=(0,x.yV)(function(e,t){let n=null!==(0,h.useContext)(ev),r=null!==(0,L.oJ)();return h.createElement(h.Fragment,null,!n&&r?h.createElement(eS,{ref:t,...e}):h.createElement(eP,{ref:t,...e}))}),eO=Object.assign(eS,{Child:eN,Root:eS}),eC=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"}))}),eM=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M6 18L18 6M6 6l12 12"}))}),eT=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{fillRule:"evenodd",d:"M5.23 7.21a.75.75 0 011.06.02L10 11.168l3.71-3.938a.75.75 0 111.08 1.04l-4.25 4.5a.75.75 0 01-1.08 0l-4.25-4.5a.75.75 0 01.02-1.06z",clipRule:"evenodd"}))});var eF=n(5995),eA=n(8097);let eR=h.forwardRef(function({title:e,titleId:t,...n},r){return h.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:r,"aria-labelledby":t},n),e?h.createElement("title",{id:t},e):null,h.createElement("path",{fillRule:"evenodd",d:"M13.5 4.938a7 7 0 11-9.006 1.737c.202-.257.59-.218.793.039.278.352.594.672.943.954.332.269.786-.049.773-.476a5.977 5.977 0 01.572-2.759 6.026 6.026 0 012.486-2.665c.247-.14.55-.016.677.238A6.967 6.967 0 0013.5 4.938zM14 12a4 4 0 01-4 4c-1.913 0-3.52-1.398-3.91-3.182-.093-.429.44-.643.814-.413a4.043 4.043 0 001.601.564c.303.038.531-.24.51-.544a5.975 5.975 0 011.315-4.192.447.447 0 01.431-.16A4.001 4.001 0 0114 12z",clipRule:"evenodd"}))});var eL=n(3707),ek=[{name:"Faster E2E",description:"Keep your team focused by slashing CI times with bare-metal performance.",href:"/#solutions",icon:eR},{name:"Completely private CI",description:"Bring your own machines or cloud instances, for full privacy.",href:"/#solutions",icon:eF.Z},{name:"ARM from dev to production",description:"Build with Apple M1/M2 locally, and in CI.",href:"/#solutions",icon:eL.Z},{name:"Live debugging",description:"Debug slow and tricky builds via SSH.",href:"/#solutions",icon:eA.Z}],eI=[{name:"Blog",href:"/blog"},{name:"Announcement",href:"/blog/blazing-fast-ci-with-microvms"},{name:"Pricing",href:"/pricing"},{name:"Docs",href:"https://docs.actuated.com/"}];function eH(){for(var e=arguments.length,t=Array(e),n=0;nt.current(...e),[t])}},9946:function(e,t,n){"use strict";n.d(t,{M:function(){return s}});var r,o=n(7294),a=n(6723),l=n(2180);let i=0;function u(){return++i}let s=null!=(r=o.useId)?r:function(){let e=(0,l.H)(),[t,n]=o.useState(e?u:null);return(0,a.e)(()=>{null===t&&n(u())},[t]),null!=t?""+t:void 0}},6723:function(e,t,n){"use strict";n.d(t,{e:function(){return o}});var r=n(7294);let o=n(3393).s?r.useEffect:r.useLayoutEffect},3855:function(e,t,n){"use strict";n.d(t,{E:function(){return a}});var r=n(7294),o=n(6723);function a(e){let t=(0,r.useRef)(e);return(0,o.e)(()=>{t.current=e},[e]),t}},4157:function(e,t,n){"use strict";n.d(t,{f:function(){return l}});var r=n(7294),o=n(6723);function a(e){var t;if(e.type)return e.type;let n=null!=(t=e.as)?t:"button";if("string"==typeof n&&"button"===n.toLowerCase())return"button"}function l(e,t){let[n,l]=(0,r.useState)(()=>a(e));return(0,o.e)(()=>{l(a(e))},[e.type,e.as]),(0,o.e)(()=>{n||!t.current||t.current instanceof HTMLButtonElement&&!t.current.hasAttribute("type")&&l("button")},[n,t]),n}},2180:function(e,t,n){"use strict";n.d(t,{H:function(){return a}});var r=n(7294);let o={serverHandoffComplete:!1};function a(){let[e,t]=(0,r.useState)(o.serverHandoffComplete);return(0,r.useEffect)(()=>{!0!==e&&t(!0)},[e]),(0,r.useEffect)(()=>{!1===o.serverHandoffComplete&&(o.serverHandoffComplete=!0)},[]),e}},3784:function(e,t,n){"use strict";n.d(t,{T:function(){return i},h:function(){return l}});var r=n(7294),o=n(3781);let a=Symbol();function l(e,t=!0){return Object.assign(e,{[a]:t})}function i(...e){let t=(0,r.useRef)(e);(0,r.useEffect)(()=>{t.current=e},[e]);let n=(0,o.z)(e=>{for(let n of t.current)null!=n&&("function"==typeof n?n(e):n.current=e)});return e.every(e=>null==e||(null==e?void 0:e[a]))?void 0:n}},6567:function(e,t,n){"use strict";n.d(t,{ZM:function(){return l},oJ:function(){return i},up:function(){return u}});var r,o=n(7294);let a=(0,o.createContext)(null);a.displayName="OpenClosedContext";var l=((r=l||{})[r.Open=0]="Open",r[r.Closed=1]="Closed",r);function i(){return(0,o.useContext)(a)}function u({value:e,children:t}){return o.createElement(a.Provider,{value:e},t)}},4103:function(e,t,n){"use strict";function r(e){let t=e.parentElement,n=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(n=t),t=t.parentElement;let r=(null==t?void 0:t.getAttribute("disabled"))==="";return!(r&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(n))&&r}n.d(t,{P:function(){return r}})},2984:function(e,t,n){"use strict";function r(e,t,...n){if(e in t){let o=t[e];return"function"==typeof o?o(...n):o}let a=Error(`Tried to handle "${e}" but there is no handler defined. Only defined handlers are: ${Object.keys(t).map(e=>`"${e}"`).join(", ")}.`);throw Error.captureStackTrace&&Error.captureStackTrace(a,r),a}n.d(t,{E:function(){return r}})},5466:function(e,t,n){"use strict";n.d(t,{r:function(){return o}});var r=n(3393);function o(e){return r.s?null:e instanceof Node?e.ownerDocument:null!=e&&e.hasOwnProperty("current")&&e.current instanceof Node?e.current.ownerDocument:document}},2351:function(e,t,n){"use strict";n.d(t,{AN:function(){return i},l4:function(){return u},sY:function(){return s},yV:function(){return f}});var r,o,a=n(7294),l=n(2984),i=((r=i||{})[r.None=0]="None",r[r.RenderStrategy=1]="RenderStrategy",r[r.Static=2]="Static",r),u=((o=u||{})[o.Unmount=0]="Unmount",o[o.Hidden=1]="Hidden",o);function s({ourProps:e,theirProps:t,slot:n,defaultTag:r,features:o,visible:a=!0,name:i}){let u=d(t,e);if(a)return c(u,n,r,i);let s=null!=o?o:0;if(2&s){let{static:f=!1,...p}=u;if(f)return c(p,n,r,i)}if(1&s){let{unmount:m=!0,...v}=u;return(0,l.E)(m?0:1,{0:()=>null,1:()=>c({...v,hidden:!0,style:{display:"none"}},n,r,i)})}return c(u,n,r,i)}function c(e,t={},n,r){let{as:o=n,children:l,refName:i="ref",...u}=m(e,["unmount","static"]),s=void 0!==e.ref?{[i]:e.ref}:{},c="function"==typeof l?l(t):l;u.className&&"function"==typeof u.className&&(u.className=u.className(t));let f={};if(t){let v=!1,h=[];for(let[g,b]of Object.entries(t))"boolean"==typeof b&&(v=!0),!0===b&&h.push(g);v&&(f["data-headlessui-state"]=h.join(" "))}if(o===a.Fragment&&Object.keys(p(u)).length>0){if(!(0,a.isValidElement)(c)||Array.isArray(c)&&c.length>1)throw Error(['Passing props on "Fragment"!',"",`The current component <${r} /> is rendering a "Fragment".`,"However we need to passthrough the following props:",Object.keys(u).map(e=>` - ${e}`).join(` `),"","You can apply a few solutions:",['Add an `as="..."` prop, to ensure that we render an actual element instead of a "Fragment".',"Render a single element as the child so that we can forward the props onto that element."].map(e=>` - ${e}`).join(` `)].join(` `));return(0,a.cloneElement)(c,Object.assign({},d(c.props,p(m(u,["ref"]))),f,s,function(...e){return{ref:e.every(e=>null==e)?void 0:t=>{for(let n of e)null!=n&&("function"==typeof n?n(t):n.current=t)}}}(c.ref,s.ref)))}return(0,a.createElement)(o,Object.assign({},m(u,["ref"]),o!==a.Fragment&&s,o!==a.Fragment&&f),c)}function d(...e){if(0===e.length)return{};if(1===e.length)return e[0];let t={},n={};for(let r of e)for(let o in r)o.startsWith("on")&&"function"==typeof r[o]?(null!=n[o]||(n[o]=[]),n[o].push(r[o])):t[o]=r[o];if(t.disabled||t["aria-disabled"])return Object.assign(t,Object.fromEntries(Object.keys(n).map(e=>[e,void 0])));for(let a in n)Object.assign(t,{[a](e,...t){for(let r of n[a]){if((e instanceof Event||(null==e?void 0:e.nativeEvent)instanceof Event)&&e.defaultPrevented)return;r(e,...t)}}});return t}function f(e){var t;return Object.assign((0,a.forwardRef)(e),{displayName:null!=(t=e.displayName)?t:e.name})}function p(e){let t=Object.assign({},e);for(let n in t)void 0===t[n]&&delete t[n];return t}function m(e,t=[]){let n=Object.assign({},e);for(let r of t)r in n&&delete n[r];return n}},3393:function(e,t,n){"use strict";n.d(t,{s:function(){return r}});let r="undefined"==typeof window||"undefined"==typeof document},8097:function(e,t,n){"use strict";var r=n(7294);let o=r.forwardRef(function({title:e,titleId:t,...n},o){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:o,"aria-labelledby":t},n),e?r.createElement("title",{id:t},e):null,r.createElement("path",{fillRule:"evenodd",d:"M3.25 3A2.25 2.25 0 001 5.25v9.5A2.25 2.25 0 003.25 17h13.5A2.25 2.25 0 0019 14.75v-9.5A2.25 2.25 0 0016.75 3H3.25zm.943 8.752a.75.75 0 01.055-1.06L6.128 9l-1.88-1.693a.75.75 0 111.004-1.114l2.5 2.25a.75.75 0 010 1.114l-2.5 2.25a.75.75 0 01-1.06-.055zM9.75 10.25a.75.75 0 000 1.5h2.5a.75.75 0 000-1.5h-2.5z",clipRule:"evenodd"}))});t.Z=o},3707:function(e,t,n){"use strict";var r=n(7294);let o=r.forwardRef(function({title:e,titleId:t,...n},o){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:o,"aria-labelledby":t},n),e?r.createElement("title",{id:t},e):null,r.createElement("path",{d:"M14 6H6v8h8V6z"}),r.createElement("path",{fillRule:"evenodd",d:"M9.25 3V1.75a.75.75 0 011.5 0V3h1.5V1.75a.75.75 0 011.5 0V3h.5A2.75 2.75 0 0117 5.75v.5h1.25a.75.75 0 010 1.5H17v1.5h1.25a.75.75 0 010 1.5H17v1.5h1.25a.75.75 0 010 1.5H17v.5A2.75 2.75 0 0114.25 17h-.5v1.25a.75.75 0 01-1.5 0V17h-1.5v1.25a.75.75 0 01-1.5 0V17h-1.5v1.25a.75.75 0 01-1.5 0V17h-.5A2.75 2.75 0 013 14.25v-.5H1.75a.75.75 0 010-1.5H3v-1.5H1.75a.75.75 0 010-1.5H3v-1.5H1.75a.75.75 0 010-1.5H3v-.5A2.75 2.75 0 015.75 3h.5V1.75a.75.75 0 011.5 0V3h1.5zM4.5 5.75c0-.69.56-1.25 1.25-1.25h8.5c.69 0 1.25.56 1.25 1.25v8.5c0 .69-.56 1.25-1.25 1.25h-8.5c-.69 0-1.25-.56-1.25-1.25v-8.5z",clipRule:"evenodd"}))});t.Z=o},5995:function(e,t,n){"use strict";var r=n(7294);let o=r.forwardRef(function({title:e,titleId:t,...n},o){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor","aria-hidden":"true",ref:o,"aria-labelledby":t},n),e?r.createElement("title",{id:t},e):null,r.createElement("path",{fillRule:"evenodd",d:"M8 7a5 5 0 113.61 4.804l-1.903 1.903A1 1 0 019 14H8v1a1 1 0 01-1 1H6v1a1 1 0 01-1 1H3a1 1 0 01-1-1v-2a1 1 0 01.293-.707L8.196 8.39A5.002 5.002 0 018 7zm5-3a.75.75 0 000 1.5A1.5 1.5 0 0114.5 7 .75.75 0 0016 7a3 3 0 00-3-3z",clipRule:"evenodd"}))});t.Z=o},1799:function(e,t,n){"use strict";function r(e){for(var t=1;t({...e,disclosureState:(0,d.E)(e.disclosureState,{0:1,1:0})}),1:e=>1===e.disclosureState?e:{...e,disclosureState:1},4:e=>!0===e.linkedPanel?e:{...e,linkedPanel:!0},5:e=>!1===e.linkedPanel?e:{...e,linkedPanel:!1},2:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},3:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId}},v=(0,o.createContext)(null);function N(e){let t=(0,o.useContext)(v);if(null===t){let a=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(a,N),a}return t}v.displayName="DisclosureContext";let k=(0,o.createContext)(null);k.displayName="DisclosureAPIContext";let S=(0,o.createContext)(null);function A(e,t){return(0,d.E)(t.type,j,e,t)}S.displayName="DisclosurePanelContext";let C=o.Fragment,I=(0,c.yV)(function(e,t){let{defaultOpen:a=!1,...s}=e,n=(0,o.useRef)(null),i=(0,m.T)(t,(0,m.h)(e=>{n.current=e},void 0===e.as||e.as===o.Fragment)),l=(0,o.useRef)(null),r=(0,o.useRef)(null),u=(0,o.useReducer)(A,{disclosureState:a?0:1,linkedPanel:!1,buttonRef:r,panelRef:l,buttonId:null,panelId:null}),[{disclosureState:x,buttonId:h},p]=u,y=(0,b.z)(e=>{p({type:1});let t=(0,f.r)(n);if(!t||!h)return;let a=e?e instanceof HTMLElement?e:e.current instanceof HTMLElement?e.current:t.getElementById(h):t.getElementById(h);null==a||a.focus()}),w=(0,o.useMemo)(()=>({close:y}),[y]),j=(0,o.useMemo)(()=>({open:0===x,close:y}),[x,y]);return o.createElement(v.Provider,{value:u},o.createElement(k.Provider,{value:w},o.createElement(g.up,{value:(0,d.E)(x,{0:g.ZM.Open,1:g.ZM.Closed})},(0,c.sY)({ourProps:{ref:i},theirProps:s,slot:j,defaultTag:C,name:"Disclosure"}))))}),E=(0,c.yV)(function(e,t){let a=(0,u.M)(),{id:s=`headlessui-disclosure-button-${a}`,...n}=e,[i,l]=N("Disclosure.Button"),r=(0,o.useContext)(S),d=null!==r&&r===i.panelId,g=(0,o.useRef)(null),f=(0,m.T)(g,t,d?null:i.buttonRef);(0,o.useEffect)(()=>{if(!d)return l({type:2,buttonId:s}),()=>{l({type:2,buttonId:null})}},[s,l,d]);let y=(0,b.z)(e=>{var t;if(d){if(1===i.disclosureState)return;switch(e.key){case x.R.Space:case x.R.Enter:e.preventDefault(),e.stopPropagation(),l({type:0}),null==(t=i.buttonRef.current)||t.focus()}}else switch(e.key){case x.R.Space:case x.R.Enter:e.preventDefault(),e.stopPropagation(),l({type:0})}}),w=(0,b.z)(e=>{e.key===x.R.Space&&e.preventDefault()}),j=(0,b.z)(t=>{var a;(0,h.P)(t.currentTarget)||e.disabled||(d?(l({type:0}),null==(a=i.buttonRef.current)||a.focus()):l({type:0}))}),v=(0,o.useMemo)(()=>({open:0===i.disclosureState}),[i]),k=(0,p.f)(e,g),A=d?{ref:f,type:k,onKeyDown:y,onClick:j}:{ref:f,id:s,type:k,"aria-expanded":e.disabled?void 0:0===i.disclosureState,"aria-controls":i.linkedPanel?i.panelId:void 0,onKeyDown:y,onKeyUp:w,onClick:j};return(0,c.sY)({ourProps:A,theirProps:n,slot:v,defaultTag:"button",name:"Disclosure.Button"})}),D=c.AN.RenderStrategy|c.AN.Static,M=Object.assign(I,{Button:E,Panel:(0,c.yV)(function(e,t){let a=(0,u.M)(),{id:s=`headlessui-disclosure-panel-${a}`,...n}=e,[i,l]=N("Disclosure.Panel"),{close:r}=function e(t){let a=(0,o.useContext)(k);if(null===a){let s=Error(`<${t} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(s,e),s}return a}("Disclosure.Panel"),d=(0,m.T)(t,i.panelRef,e=>{l({type:e?4:5})});(0,o.useEffect)(()=>(l({type:3,panelId:s}),()=>{l({type:3,panelId:null})}),[s,l]);let x=(0,g.oJ)(),h=null!==x?x===g.ZM.Open:0===i.disclosureState,p=(0,o.useMemo)(()=>({open:0===i.disclosureState,close:r}),[i,r]);return o.createElement(S.Provider,{value:i.panelId},(0,c.sY)({ourProps:{ref:d,id:s},theirProps:n,slot:p,defaultTag:"div",features:D,visible:h,name:"Disclosure.Panel"}))})}),H=o.forwardRef(function({title:e,titleId:t,...a},s){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:s,"aria-labelledby":t},a),e?o.createElement("title",{id:t},e):null,o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19.5 8.25l-7.5 7.5-7.5-7.5"}))});var P=[{question:"What is Firecracker? Why is it better than a container?",answer:"Generally, any use of containers for CI means bypassing security, so that any job can take over a host or the cluster. Actuated uses Firecracker, an open-source project built by AWS to fully isolate every job with an immutable microVM.",link:"https://www.youtube.com/watch?v=CYCsa5e2vqg",action:"Watch our Firecracker webinar"},{question:"Can we talk to you before signing up for a plan?",answer:"Just fill out the form for a call with our founder. If you think our solution is a good fit, you can be up and running on the same day.",link:"https://docs.google.com/forms/d/e/1FAIpQLScA12IGyVFrZtSAp2Oj24OdaSMloqARSwoxx3AZbQbs0wpGww/viewform",action:"Schedule a call"},{question:"How much does it cost? What is the right plan for our team?",answer:"We are offering unmetered billing with a flat-rate monthly subscription. You can use as many minutes as you like."},{question:"What kind of support do you offer?",answer:"For the pilot, every customer is invited to a Slack channel for collaboration and support. We have operational experience with GitHub Actions, Docker and Kubernetes and we're making time to help you tune your builds up to get the most out of them."},{question:"What kinds of servers do I need?",answer:"You can use your own physical servers, nested virtualisation with cloud VMs or rent instances paid by the hour.",link:"https://docs.actuated.dev/provision-server/",action:"Explore server options"},{question:"Doesn't GitHub already offer faster runners?",answer:"GitHub are in a beta phase for larger runners for their Team and Enterprises plans, these have an increased cost vs. standard runners and there is no Arm support. With actuated you get much faster speeds at a flat rate cost, with usage insights across your team and organisation."},{question:"What are the differences vs. Actions Runtime Controller?",answer:"actions-runtime-controller compromises the security of a Kubernetes cluster by using privileged containers or by mounting the Docker socket - both mean that code in a CI job can take over the host - or potentially the cluster.",link:"/blog/blazing-fast-ci-with-microvms"},{question:"How much faster is an Arm build than using hosted runners?",answer:"In our testing of the open source Parca project, we got the build time down from 33 minutes to 1 minute 26s simply by changing to an Arm runner instead of using QEMU. For Network Service Mesh, Dasch Swiss and Calyptia - their builds couldn't complete within 6 hours, all complete in 4-20 minutes with actuated.",link:"/blog/native-arm64-for-github-actions",action:"Read a case study"},{question:"How do I launch jobs in parallel?",answer:"Have a look at the examples in our docs for matrix builds. Each job within the workflow will be launched in a separate, parallel VM.",link:"https://docs.actuated.dev/"},{question:"How mature is actuated?",answer:"Actuated is built on battle tested technology that's run in production at huge scale by Amazon Web Services (AWS) and GitHub. Our solution has launched over 25k VMs for customers already, without issue."},{question:"Is GitLab supported?",answer:"We have a tech preview for self-hosted GitLab.",link:"/blog/secure-microvm-ci-gitlab",action:"Read the announcement"},{question:"Where can I find detailed information about actuated?",answer:"We cover most common questions in much more detail over in the FAQ in the docs.",link:"https://docs.actuated.dev/faq",action:"FAQ"}];function R(){return(0,i.jsx)("div",{className:"bg-gray-50",children:(0,i.jsx)("div",{className:"mx-auto max-w-7xl py-12 px-4 sm:py-16 sm:px-6 lg:px-8",children:(0,i.jsxs)("div",{className:"mx-auto max-w-3xl divide-y-2 divide-gray-200",children:[(0,i.jsx)("h2",{className:"text-center text-3xl font-bold tracking-tight text-gray-900 sm:text-4xl",children:"Frequently asked questions"}),(0,i.jsx)("dl",{className:"mt-6 space-y-6 divide-y divide-gray-200",children:P.map(function(e){return(0,i.jsx)(M,{as:"div",className:"pt-6",children:function(t){var a=t.open;return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)("dt",{className:"text-lg",children:(0,i.jsxs)(M.Button,{className:"flex w-full items-start justify-between text-left text-gray-400",children:[(0,i.jsx)("span",{className:"font-medium text-gray-900",children:e.question}),(0,i.jsx)("span",{className:"ml-6 flex h-7 items-center",children:(0,i.jsx)(H,{className:function(){for(var e=arguments.length,t=Array(e),a=0;a