Skip to content

Commit

Permalink
feat: Use Cached Kernel, fsync for latest (#1502)
Browse files Browse the repository at this point in the history
  • Loading branch information
m2Giles committed Jul 15, 2024
1 parent 059a167 commit 6cd71c1
Show file tree
Hide file tree
Showing 9 changed files with 141 additions and 98 deletions.
168 changes: 107 additions & 61 deletions .github/workflows/reusable-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,67 97,91 @@ jobs:
echo "AKMODS_FLAVOR=asus" >> $GITHUB_ENV
elif [[ "${{ matrix.image_flavor }}" =~ "surface" ]]; then
echo "AKMODS_FLAVOR=surface" >> $GITHUB_ENV
echo "KERNEL_SUFFIX=surface" >> $GITHUB_ENV
elif [[ "${{ matrix.fedora_version }}" == "stable" || \
"${{ matrix.fedora_version }}" == "gts" ]]; then
echo "AKMODS_FLAVOR=coreos" >> $GITHUB_ENV
elif [[ "${{ matrix.fedora_version }}" =~ stable|gts ]]; then
echo "AKMODS_FLAVOR=coreos-stable" >> $GITHUB_ENV
elif [[ "${{ matrix.fedora_version }}" == "latest" ]]; then
echo "AKMODS_FLAVOR=fsync" >> $GITHUB_ENV
else
echo "AKMODS_FLAVOR=main" >> $GITHUB_ENV
fi
# Env for matrix.image_flavor
if [[ "${{ matrix.image_flavor }}" == "nvidia" ]] && \
[[ "${{ matrix.fedora_version }}" == "stable" || \
"${{ matrix.fedora_version }}" == "gts" ]]; then
echo "image_flavor=main" >> $GITHUB_ENV
echo "coreos_type=nvidia" >> $GITHUB_ENV
elif [[ "${{ matrix.image_flavor }}" == "main" ]] && \
[[ "${{ matrix.fedora_version }}" == "stable" || \
"${{ matrix.fedora_version }}" == "gts" ]]; then
if [[ "${{ matrix.image_flavor }}" == "nvidia" && \
"${{ matrix.fedora_version }}" != "beta" ]]; then
echo "image_flavor=main" >> $GITHUB_ENV
echo "nvidia_type=nvidia" >> $GITHUB_ENV
elif [[ "${{ matrix.image_flavor }}" == "main" && \
"${{ matrix.fedora_version }}" != "beta" ]]; then
echo "image_flavor=${{ matrix.image_flavor }}" >> $GITHUB_ENV
echo "coreos_type=main" >> $GITHUB_ENV
echo "nvidia_type=main" >> $GITHUB_ENV
else
echo "image_flavor=${{ matrix.image_flavor }}" >> $GITHUB_ENV
fi
- name: Get Current Fedora Version
id: labels
shell: bash
run: |
set -eo pipefail
if [[ ${{ matrix.fedora_version }} == "stable" ]]; then
KERNEL_RELEASE=$(skopeo inspect docker://quay.io/fedora/fedora-coreos:stable | jq -r '.Labels["ostree.linux"] | split(".x86_64")[0]')
elif [[ ${{ matrix.fedora_version }} == "gts" ]]; then
coreos_kernel_release=$(skopeo inspect docker://quay.io/fedora/fedora-coreos:stable | jq -r '.Labels["ostree.linux"] | split(".x86_64")[0]')
major_minor_patch=$(echo "$coreos_kernel_release" | cut -d '-' -f 1)
coreos_fedora_version=$(echo $coreos_kernel_release | grep -oP 'fc\K[0-9] ')
KERNEL_RELEASE="${major_minor_patch}-200.fc$(($coreos_fedora_version - 1))"
else
KERNEL_RELEASE=$(skopeo inspect docker://ghcr.io/ublue-os/silverblue-${{ env.image_flavor }}:${{ matrix.fedora_version }} | jq -r '.Labels["ostree.linux"] | split(".x86_64")[0]')
fi
fedora_version=$(echo $KERNEL_RELEASE | grep -oP 'fc\K[0-9] ')
echo "kernel_release=$KERNEL_RELEASE" >> $GITHUB_OUTPUT
echo "fedora_version=$fedora_version" >> $GITHUB_OUTPUT
ver=$(skopeo inspect docker://ghcr.io/ublue-os/${{ env.BASE_IMAGE_NAME }}-${{ env.image_flavor }}:$fedora_version | jq -r '.Labels["org.opencontainers.image.version"]')
if [ -z "$ver" ] || [ "null" = "$ver" ]; then
echo "inspected image version must not be empty or null"
exit 1
fi
echo "VERSION=$ver" >> $GITHUB_OUTPUT
uses: Wandalen/[email protected]
with:
attempt_limit: 3
attempt_delay: 15000
command: |
set -eox pipefail
if [[ ${{ matrix.fedora_version }} == "stable" ]]; then
KERNEL_RELEASE=$(skopeo inspect docker://quay.io/fedora/fedora-coreos:stable | jq -r '.Labels["ostree.linux"]')
elif [[ ${{ matrix.fedora_version }} == "gts" && ${{ env.AKMODS_FLAVOR }} != "surface" ]]; then
coreos_kernel_release=$(skopeo inspect docker://quay.io/fedora/fedora-coreos:stable | jq -r '.Labels["ostree.linux"]')
major_minor_patch=$(echo "$coreos_kernel_release" | cut -d '-' -f 1)
coreos_fedora_version=$(echo $coreos_kernel_release | grep -oP 'fc\K[0-9] ')
KERNEL_RELEASE="${major_minor_patch}-200.fc$(($coreos_fedora_version - 1)).$(uname -m)"
else
base_kernel_release=$(skopeo inspect docker://ghcr.io/ublue-os/silverblue-${{ env.image_flavor }}:${{ matrix.fedora_version }} | jq -r '.Labels["ostree.linux"]')
base_fedora_version=$(echo $base_kernel_release | grep -oP 'fc\K[0-9] ')
KERNEL_RELEASE=$(skopeo inspect docker://ghcr.io/ublue-os/${{ env.AKMODS_FLAVOR }}-kernel:${base_fedora_version} | jq -r '.Labels["ostree.linux"]')
fi
fedora_version=$(echo $KERNEL_RELEASE | grep -oP 'fc\K[0-9] ')
echo "kernel_release=$KERNEL_RELEASE" >> $GITHUB_ENV
echo "fedora_version=$fedora_version" >> $GITHUB_ENV
ver=$(skopeo inspect docker://ghcr.io/ublue-os/${{ env.BASE_IMAGE_NAME }}-${{ env.image_flavor }}:$fedora_version | jq -r '.Labels["org.opencontainers.image.version"]')
if [ -z "$ver" ] || [ "null" = "$ver" ]; then
echo "inspected image version must not be empty or null"
exit 1
fi
echo "VERSION=$ver" >> $GITHUB_ENV
- name: Verify base image
uses: EyeCantCU/cosign-action/verify@58722a084c82190b57863002d494c91eabbe9e79 # v0.3.0
with:
containers: ${{ env.BASE_IMAGE_NAME}}-${{ env.image_flavor }}:${{ steps.labels.outputs.fedora_version }}
containers: ${{ env.BASE_IMAGE_NAME}}-${{ env.image_flavor }}:${{ env.fedora_version }}

- name: Verify Chainguard images
if: matrix.base_name != 'bluefin' && matrix.base_name != 'aurora'
uses: EyeCantCU/cosign-action/verify@58722a084c82190b57863002d494c91eabbe9e79 # v0.3.0
- name: Verify Akmods
uses: EyeCantCU/cosign-action/verify@11f8c114a5e67c7a663c9dfcaf76d85429d254bc # v0.2.2
with:
containers: akmods:${{ env.AKMODS_FLAVOR}}-${{ env.fedora_version }}

- name: Verify Nvidia
uses: EyeCantCU/cosign-action/verify@11f8c114a5e67c7a663c9dfcaf76d85429d254bc # v0.2.2
with:
containers: akmods-nvidia:${{ env.AKMODS_FLAVOR}}-${{ env.fedora_version }}

- name: Verify Kernel Cache
uses: EyeCantCU/cosign-action/verify@11f8c114a5e67c7a663c9dfcaf76d85429d254bc # v0.2.2
with:
containers: dive, flux, helm, ko, minio, kubectl
cert-identity: https://github.com/chainguard-images/images/.github/workflows/release.yaml@refs/heads/main
oidc-issuer: https://token.actions.githubusercontent.com
registry: cgr.dev/chainguard
containers: ${{ env.AKMODS_FLAVOR }}-kernel:${{ env.kernel_release }}

- name: Verify Kernel Verion Matches
uses: Wandalen/[email protected]
with:
attempt_limit: 3
attempt_delay: 15000
command: |
set -x
akmods_version=$(skopeo inspect docker://ghcr.io/ublue-os/akmods:${{ env.AKMODS_FLAVOR }}-${{ env.fedora_version }} | jq -r '.Labels["ostree.linux"]')
if [[ "${akmods_version}" == "${{ env.kernel_release }}" ]]; then
echo "Kernel Versions Match"
else
echo "Kernel Version do Not Match"
exit 1
fi
- name: Maximize build space
if: contains(matrix.base_name, '-dx') && (github.event_name == 'pull_request' && ( matrix.image_flavor == 'main' || matrix.image_flavor == 'nvidia' ) || github.event_name != 'pull_request')
Expand Down Expand Up @@ -212,14 236,14 @@ jobs:
if [[ ${{ matrix.fedora_version }} == "stable" ]]; then
BUILD_TAGS=("${FEDORA_VERSION}" "${FEDORA_VERSION}-${TIMESTAMP}")
else
BUILD_TAGS=("${{ steps.labels.outputs.fedora_version }}" "${{ steps.labels.outputs.fedora_version }}-${TIMESTAMP}")
BUILD_TAGS=("${{ env.fedora_version }}" "${{ env.fedora_version }}-${TIMESTAMP}")
fi
if [[ ${{ github.ref_name }} == "testing" ]]; then
if [[ ${{ matrix.fedora_version }} == "stable" ]]; then
BUILD_TAGS=("${FEDORA_VERSION}-testing" "${FEDORA_VERSION}-testing-${TIMESTAMP}")
else
BUILD_TAGS=("${{ steps.labels.outputs.fedora_version }}-testing" "${{ steps.labels.outputs.fedora_version }}-testing-${TIMESTAMP}")
BUILD_TAGS=("${{ env.fedora_version }}-testing" "${{ env.fedora_version }}-testing-${TIMESTAMP}")
fi
if [[ "$IS_LATEST_VERSION" == "true" ]] && \
[[ "$IS_STABLE_VERSION" == "true" ]]; then
Expand Down Expand Up @@ -269,12 293,26 @@ jobs:
${{ env.IMAGE_NAME }}
labels: |
org.opencontainers.image.title=${{ env.IMAGE_NAME }}
org.opencontainers.image.version=${{ steps.labels.outputs.VERSION }}
org.opencontainers.image.version=${{ env.VERSION }}
org.opencontainers.image.description=An interpretation of the Ubuntu spirit built on Fedora technology
ostree.linux=${{ steps.labels.outputs.kernel_release }}.x86_64
ostree.linux=${{ env.kernel_release }}
io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/bluefin/bluefin/README.md
io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4
- name: Pull images
if: github.event_name == 'pull_request' && ( matrix.image_flavor == 'main' || matrix.image_flavor == 'nvidia' ) || github.event_name != 'pull_request'
uses: Wandalen/[email protected]
with:
attempt_limit: 3
attempt_delay: 15000
command: |
# pull the base image used for FROM in containerfile so
# we can retry on that unfortunately common failure case
podman pull ${{ env.IMAGE_REGISTRY }}/${{ env.BASE_IMAGE_NAME }}-${{ env.image_flavor }}:${{ env.fedora_version }}
podman pull ${{ env.IMAGE_REGISTRY }}/akmods:${{ env.AKMODS_FLAVOR }}-${{ env.fedora_version }}
podman pull ${{ env.IMAGE_REGISTRY }}/akmods-nvidia:${{ env.AKMODS_FLAVOR }}-${{ env.fedora_version }}
podman pull ${{ env.IMAGE_REGISTRY }}/${{ env.AKMODS_FLAVOR }}-kernel:${{ env.kernel_release }}
# Build image using Buildah action
- name: Build Image
id: build_image
Expand All @@ -291,11 329,11 @@ jobs:
IMAGE_NAME=${{ env.IMAGE_NAME }}
IMAGE_FLAVOR=${{ env.image_flavor }}
IMAGE_VENDOR=${{ github.repository_owner }}
FEDORA_MAJOR_VERSION=${{ steps.labels.outputs.fedora_version }}
FEDORA_MAJOR_VERSION=${{ env.fedora_version }}
TARGET_BASE=${{ matrix.target_base }}
AKMODS_FLAVOR=${{ env.AKMODS_FLAVOR }}
COREOS_TYPE=${{ env.coreos_type }}
KERNEL=${{ steps.labels.outputs.kernel_release }}
NVIDIA_TYPE=${{ env.nvidia_type }}
KERNEL=${{ env.kernel_release }}
UBLUE_IMAGE_TAG=${{ matrix.fedora_version }}
labels: ${{ steps.meta.outputs.labels }}
oci: false
Expand All @@ -305,17 343,25 @@ jobs:
extra-args: |
--target=${{ env.TARGET_NAME }}
- name: Sign kernel
uses: ublue-os/kernel-signer@d7bee36277c13bd4072f452766392a3420d2bbdf # v0.2.4
if: github.event_name != 'pull_request'
with:
image: ${{ steps.build_image.outputs.image }}
default-tag: ${{ env.DEFAULT_TAG }}
privkey: ${{ secrets.AKMOD_PRIVKEY_20230518 }}
pubkey: /etc/pki/akmods/certs/akmods-ublue.der
tags: ${{ steps.build_image.outputs.tags }}
kernel_suffix: ${{ env.KERNEL_SUFFIX }}
strip: false
- name: Check Secureboot
if: github.event_name == 'pull_request' && ( matrix.image_flavor == 'main' || matrix.image_flavor == 'nvidia' ) || github.event_name != 'pull_request'
shell: bash
run: |
set -x
if [[ ! $(command -v sbverify) || ! $(command -v curl) || ! $(command -v openssl) ]]; then
sudo apt update
sudo apt install sbsigntool curl openssl
fi
podman run -d --rm --name ${{env.IMAGE_NAME}}-$(echo "${{ steps.generate-tags.outputs.alias_tags }}" | cut -d " " -f 1) "${{ env.IMAGE_NAME }}":$(echo "${{ steps.generate-tags.outputs.alias_tags }}" | cut -d " " -f 1) sleep 1000
podman cp ${{env.IMAGE_NAME}}-$(echo "${{ steps.generate-tags.outputs.alias_tags }}" | cut -d " " -f 1):/usr/lib/modules/${{ env.kernel_release }}/vmlinuz .
podman rm -f ${{env.IMAGE_NAME}}-$(echo "${{ steps.generate-tags.outputs.alias_tags }}" | cut -d " " -f 1)
sbverify --list vmlinuz
curl --retry 3 -Lo kernel-sign.der https://github.com/ublue-os/kernel-cache/raw/main/certs/public_key.der
curl --retry 3 -Lo akmods.der https://github.com/ublue-os/kernel-cache/raw/main/certs/public_key_2.der
openssl x509 -in kernel-sign.der -out kernel-sign.crt
openssl x509 -in akmods.der -out akmods.crt
sbverify --cert kernel-sign.crt vmlinuz || exit 1
sbverify --cert akmods.crt vmlinuz || exit 1
# Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
# https://github.com/macbre/push-to-ghcr/issues/12
Expand Down
22 changes: 12 additions & 10 deletions Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 5,17 @@ ARG SOURCE_IMAGE="${SOURCE_IMAGE:-${BASE_IMAGE_NAME}-${IMAGE_FLAVOR}}"
ARG BASE_IMAGE="ghcr.io/ublue-os/${SOURCE_IMAGE}"
ARG FEDORA_MAJOR_VERSION="${FEDORA_MAJOR_VERSION:-40}"
ARG TARGET_BASE="${TARGET_BASE:-bluefin}"
ARG COREOS_TYPE="${COREOS_TYPE:-}"
ARG KERNEL="${KERNEL:-}"
ARG NVIDIA_TYPE="${NVIDIA_TYPE:-}"
ARG KERNEL="${KERNEL:-6.9.7-200.fc40.x86_64}"
ARG UBLUE_IMAGE_TAG="${UBLUE_IMAGE_TAG:-latest}"

# FROM's for copying
ARG KMOD_SOURCE_COMMON="ghcr.io/ublue-os/akmods:${AKMODS_FLAVOR}-${FEDORA_MAJOR_VERSION}"
ARG COREOS_KMODS="ghcr.io/ublue-os/ucore-kmods:stable"
ARG COREOS_NVIDIA="ghcr.io/ublue-os/akmods-nvidia:coreos-${FEDORA_MAJOR_VERSION}"
ARG NVIDIA_CACHE="ghcr.io/ublue-os/akmods-nvidia:${AKMODS_FLAVOR}-${FEDORA_MAJOR_VERSION}"
ARG KERNEL_CACHE="ghcr.io/ublue-os/${AKMODS_FLAVOR}-kernel:${KERNEL}"
FROM ${KMOD_SOURCE_COMMON} AS akmods
FROM ${COREOS_NVIDIA} AS coreos_nvidia
FROM ${NVIDIA_CACHE} AS nvidia_cache
FROM ${KERNEL_CACHE} AS kernel_cache

## bluefin image section
FROM ${BASE_IMAGE}:${FEDORA_MAJOR_VERSION} AS base
Expand All @@ -25,8 26,8 @@ ARG IMAGE_FLAVOR="${IMAGE_FLAVOR}"
ARG AKMODS_FLAVOR="${AKMODS_FLAVOR}"
ARG BASE_IMAGE_NAME="${BASE_IMAGE_NAME}"
ARG FEDORA_MAJOR_VERSION="${FEDORA_MAJOR_VERSION}"
ARG COREOS_TYPE="${COREOS_TYPE:-}"
ARG KERNEL="${KERNEL:-}"
ARG NVIDIA_TYPE="${NVIDIA_TYPE:-}"
ARG KERNEL="${KERNEL:-6.9.7-200.fc40.x86_64}"
ARG UBLUE_IMAGE_TAG="${UBLUE_IMAGE_TAG:-latest}"

# COPY Build Files
Expand All @@ -39,7 40,8 @@ COPY packages.json /tmp/packages.json
COPY /system_files/shared/usr/etc/ublue-update/ublue-update.toml /tmp/ublue-update.toml
# COPY ublue kmods, add needed negativo17 repo and then immediately disable due to incompatibility with RPMFusion
COPY --from=akmods /rpms /tmp/akmods-rpms
COPY --from=coreos_nvidia /rpms /tmp/akmods-rpms
COPY --from=nvidia_cache /rpms /tmp/akmods-rpms
COPY --from=kernel_cache /tmp/rpms /tmp/kernel-rpms

# Build, cleanup, commit.
RUN rpm-ostree cliwrap install-to-root / && \
Expand All @@ -61,8 63,8 @@ ARG BASE_IMAGE_NAME="${BASE_IMAGE_NAME}"
ARG IMAGE_FLAVOR="${IMAGE_FLAVOR}"
ARG AKMODS_FLAVOR="${AKMODS_FLAVOR}"
ARG FEDORA_MAJOR_VERSION="${FEDORA_MAJOR_VERSION}"
ARG COREOS_TYPE="${COREOS_TYPE:-}"
ARG KERNEL="${KERNEL:-}"
ARG NVIDIA_TYPE="${NVIDIA_TYPE:-}"
ARG KERNEL="${KERNEL:-6.9.7-200.fc40.x86_64}"
ARG UBLUE_IMAGE_TAG="${UBLUE_IMAGE_TAG:-latest}"

# dx specific files come from the dx directory in this repo
Expand Down
2 changes: 1 addition & 1 deletion build_files/base/build-base.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 4,7 @@
set -ouex pipefail

. /tmp/build/firmware.sh
. /tmp/build/coreos_kernel.sh
. /tmp/build/cache_kernel.sh
. /tmp/build/copr-repos.sh
. /tmp/build/install-akmods.sh
. /tmp/build/packages.sh
Expand Down
10 changes: 10 additions & 0 deletions build_files/base/cache_kernel.sh
Original file line number Diff line number Diff line change
@@ -0,0 1,10 @@
#!/usr/bin/bash

set -eoux pipefail

if [[ -n "${NVIDIA_TYPE:-}" ]]; then
rpm-ostree override replace --experimental \
/tmp/kernel-rpms/kernel-[0-9]*.rpm \
/tmp/kernel-rpms/kernel-core-*.rpm \
/tmp/kernel-rpms/kernel-modules-*.rpm
fi
15 changes: 0 additions & 15 deletions build_files/base/coreos_kernel.sh

This file was deleted.

14 changes: 7 additions & 7 deletions build_files/base/install-akmods.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 2,13 @@

set -ouex pipefail

if [[ -n "${COREOS_TYPE:-}" ]]; then
curl -L -o /etc/yum.repos.d/fedora-coreos-pool.repo \
https://raw.githubusercontent.com/coreos/fedora-coreos-config/testing-devel/fedora-coreos-pool.repo
fi
# if [[ -n "${NVIDIA_TYPE:-}" ]]; then
# curl -L -o /etc/yum.repos.d/fedora-coreos-pool.repo \
# https://raw.githubusercontent.com/coreos/fedora-coreos-config/testing-devel/fedora-coreos-pool.repo
# fi

# Nvidia for gts/stable - nvidia
if [[ "${COREOS_TYPE}" == "nvidia" ]]; then
if [[ "${NVIDIA_TYPE}" == "nvidia" ]]; then
curl -Lo /tmp/nvidia-install.sh https://raw.githubusercontent.com/ublue-os/hwe/main/nvidia-install.sh && \
chmod x /tmp/nvidia-install.sh && \
IMAGE_NAME="${BASE_IMAGE_NAME}" RPMFUSION_MIRROR="" /tmp/nvidia-install.sh
Expand Down Expand Up @@ -36,9 36,9 @@ fi
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/negativo17-fedora-multimedia.repo

# ZFS for gts/stable
if [[ -n "${COREOS_TYPE:-}" ]]; then
if [[ ${AKMODS_FLAVOR} == "coreos" ]]; then
rpm-ostree install /tmp/akmods-rpms/kmods/zfs/*.rpm \
pv
depmod -a -v "${KERNEL}".x86_64
depmod -a -v "${KERNEL}"
echo "zfs" > /usr/lib/modules-load.d/zfs.conf
fi
2 changes: 1 addition & 1 deletion build_files/base/nvidia.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 3,7 @@
set -ouex pipefail

# Nvidia Configurations
if [[ "${IMAGE_FLAVOR}" =~ "nvidia" || "${COREOS_TYPE}" =~ "nvidia" ]]; then
if [[ "${IMAGE_FLAVOR}" =~ "nvidia" || "${NVIDIA_TYPE}" =~ "nvidia" ]]; then
# Restore x11 for Nvidia Images
if [[ "${BASE_IMAGE_NAME}" =~ "kinoite" && "${FEDORA_MAJOR_VERSION}" -gt "39" ]]; then
rpm-ostree install plasma-workspace-x11
Expand Down
2 changes: 1 addition & 1 deletion build_files/shared/image-info.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 8,7 @@ IMAGE_REF="ostree-image-signed:docker://ghcr.io/$IMAGE_VENDOR/$IMAGE_NAME"
#shellcheck disable=SC2153
image_flavor="${IMAGE_FLAVOR}"

if [[ "${COREOS_TYPE}" == "nvidia" ]]; then
if [[ "${NVIDIA_TYPE}" == "nvidia" ]]; then
image_flavor="nvidia"
fi

Expand Down
Loading

0 comments on commit 6cd71c1

Please sign in to comment.