ci-test/justfile

# This is a justfile. See https://github.com/casey/just
# This is only used for local development. The builds made on the Fedora
# infrastructure are run via Pungi in a Koji runroot.

# Set a default for some recipes
default_variant := "silverblue"
# Current default in Pungi
force_nocache := "true"

# Just doesn't have a native dict type, but quoted bash dictionary works fine
pretty_names := '(
    [silverblue]="Silverblue"
    [kinoite]="Kinoite"
    [kinoite-nightly]="Kinoite"
    [kinoite-beta]="Kinoite"
    [kinoite-mobile]="Kinoite"
    [sway-atomic]="Sway Atomic"
    [budgie-atomic]="Budgie Atomic"
    [xfce-atomic]="XFCE Atomic"
    [lxqt-atomic]="LXQt Atomic"
    [base-atomic]="Base Atomic"
)'

# Legacy names, used to keep some references stable
legacy_names := '(
    [base-atomic]="base"
    [budgie-atomic]="onyx"
    [lxqt-atomic]="lazurite"
    [sway-atomic]="sericea"
    [xfce-atomic]="vauxite"
)'

# subset of the map from https://pagure.io/pungi-fedora/blob/main/f/general.conf
volume_id_substitutions := '(
    [silverblue]="SB"
    [kinoite]="Kin"
    [kinoite-nightly]="Kin"
    [kinoite-beta]="Kin"
    [kinoite-mobile]="Kin"
    [sway-atomic]="SwA"
    [budgie-atomic]="BdA"
    [xfce-atomic]="XfA"
    [lxqt-atomic]="LxA"
    [base-atomic]="BsA"
)'

# Default is to only validate the manifests
all: validate

# Basic validation to make sure the manifests are not completely broken
validate:
    ./ci/validate

# Comps-sync, but without pulling latest
sync:
    #!/bin/bash
    set -euo pipefail

    if [[ ! -d fedora-comps ]]; then
        git clone https://pagure.io/fedora-comps.git
    fi

    default_variant={{default_variant}}
    version="$(rpm-ostree compose tree --print-only --repo=repo ${default_variant}.yaml | jq -r '."mutate-os-release"')"
    ./comps-sync.py --save fedora-comps/comps-f${version}.xml.in

# Sync the manifests with the content of the comps groups
comps-sync:
    #!/bin/bash
    set -euo pipefail

    if [[ ! -d fedora-comps ]]; then
        git clone https://pagure.io/fedora-comps.git
    else
        pushd fedora-comps > /dev/null || exit 1
        git fetch
        git reset --hard origin/main
        popd > /dev/null || exit 1
    fi

    default_variant={{default_variant}}
    version="$(rpm-ostree compose tree --print-only --repo=repo ${default_variant}.yaml | jq -r '."mutate-os-release"')"
    ./comps-sync.py --save fedora-comps/comps-f${version}.xml.in

# Check if the manifests are in sync with the content of the comps groups
comps-sync-check:
    #!/bin/bash
    set -euo pipefail

    if [[ ! -d fedora-comps ]]; then
        git clone https://pagure.io/fedora-comps.git
    else
        pushd fedora-comps > /dev/null || exit 1
        git fetch
        git reset --hard origin/main
        popd > /dev/null || exit 1
    fi

    default_variant={{default_variant}}
    version="$(rpm-ostree compose tree --print-only --repo=repo ${default_variant}.yaml | jq -r '."mutate-os-release"')"
    ./comps-sync.py fedora-comps/comps-f${version}.xml.in

# Output the processed manifest for a given variant (defaults to Silverblue)
manifest variant=default_variant:
    #!/bin/bash
    set -euo pipefail

    declare -A pretty_names={{pretty_names}}
    variant={{variant}}
    variant_pretty=${pretty_names[$variant]-}
    if [[ -z $variant_pretty ]]; then
        echo "Unknown variant"
        exit 1
    fi

    rpm-ostree compose tree --print-only --repo=repo {{variant}}.yaml

# Perform dependency resolution for a given variant (defaults to Silverblue)
compose-dry-run variant=default_variant:
    #!/bin/bash
    set -euxo pipefail

    mkdir -p repo cache logs
    if [[ ! -f "repo/config" ]]; then
        pushd repo > /dev/null || exit 1
        ostree init --repo . --mode=bare-user
        popd > /dev/null || exit 1
    fi

    rpm-ostree compose tree --unified-core --repo=repo --dry-run {{variant}}.yaml

# Alias/shortcut for compose-image command
compose variant=default_variant: (compose-image variant)

# Compose a variant using the legacy non container path (defaults to Silverblue)
compose-legacy variant=default_variant:
    #!/bin/bash
    set -euxo pipefail

    declare -A pretty_names={{pretty_names}}
    variant={{variant}}
    variant_pretty=${pretty_names[$variant]-}
    if [[ -z $variant_pretty ]]; then
        echo "Unknown variant"
        exit 1
    fi

    ./ci/validate > /dev/null || (echo "Failed manifest validation" && exit 1)

    mkdir -p repo cache logs
    if [[ ! -f "repo/config" ]]; then
        pushd repo > /dev/null || exit 1
        ostree init --repo . --mode=bare-user
        popd > /dev/null || exit 1
    fi
    # Set option to reduce fsync for transient builds
    ostree --repo=repo config set 'core.fsync' 'false'

    buildid="$(date '+%Y%m%d.0')"
    timestamp="$(date --iso-8601=sec)"
    echo "${buildid}" > .buildid

    version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')"
    echo "Composing ${variant_pretty} ${version}.${buildid} ..."

    ARGS="--repo=repo --cachedir=cache"
    ARGS+=" --unified-core"
    if [[ {{force_nocache}} == "true" ]]; then
        ARGS+=" --force-nocache"
    fi
    CMD="rpm-ostree"
    if [[ ${EUID} -ne 0 ]]; then
        CMD="sudo rpm-ostree"
    fi

    ${CMD} compose tree ${ARGS} \
        --add-metadata-string="version=${variant_pretty} ${version}.${buildid}" \
        "${variant}-ostree.yaml" \
            |& tee "logs/${variant}_${version}_${buildid}.${timestamp}.log"

    if [[ ${EUID} -ne 0 ]]; then
        sudo chown --recursive "$(id --user --name):$(id --group --name)" repo cache
    fi

    ostree summary --repo=repo --update

# Compose an Ostree Native Container OCI image
compose-image variant=default_variant:
    #!/bin/bash
    set -euxo pipefail

    declare -A pretty_names={{pretty_names}}
    variant={{variant}}
    variant_pretty=${pretty_names[$variant]-}
    if [[ -z $variant_pretty ]]; then
        echo "Unknown variant"
        exit 1
    fi

    ./ci/validate > /dev/null || (echo "Failed manifest validation" && exit 1)

    mkdir -p repo cache
    if [[ ! -f "repo/config" ]]; then
        pushd repo > /dev/null || exit 1
        ostree init --repo . --mode=bare-user
        popd > /dev/null || exit 1
    fi
    # Set option to reduce fsync for transient builds
    ostree --repo=repo config set 'core.fsync' 'false'

    buildid="$(date '+%Y%m%d.0')"
    timestamp="$(date --iso-8601=sec)"
    echo "${buildid}" > .buildid

    version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')"
    echo "Composing ${variant_pretty} ${version}.${buildid} ..."

    ARGS="--cachedir=cache --initialize"
    if [[ {{force_nocache}} == "true" ]]; then
        ARGS+=" --force-nocache"
    fi
    # To debug with gdb, use: gdb --args ...
    CMD="rpm-ostree"
    if [[ ${EUID} -ne 0 ]]; then
        CMD="sudo rpm-ostree"
    fi

    ${CMD} compose image ${ARGS} \
         --label="quay.expires-after=4w" \
        "${variant}.yaml" \
        "${variant}.ociarchive"

# Clean up everything
clean-all:
    just clean-repo
    just clean-cache

# Only clean the ostree repo
clean-repo:
    rm -rf ./repo

# Only clean the package and repo caches
clean-cache:
    rm -rf ./cache

# Run from inside a container
podman:
    podman run --rm -ti --volume $PWD:/srv:rw --workdir /srv --privileged quay.io/fedora-ostree-desktops/buildroot

# Update the container image
podman-pull:
    podman pull quay.io/fedora-ostree-desktops/buildroot

# Build an ISO
lorax variant=default_variant:
    #!/bin/bash
    set -euxo pipefail

    rm -rf iso
    # Do not create the iso directory or lorax will fail
    mkdir -p tmp cache/lorax

    declare -A pretty_names={{pretty_names}}
    declare -A volume_id_substitutions={{volume_id_substitutions}}
    variant={{variant}}
    variant_pretty=${pretty_names[$variant]-}
    volid_sub=${volume_id_substitutions[$variant]-}
    if [[ -z $variant_pretty ]] || [[ -z $volid_sub ]]; then
        echo "Unknown variant"
        exit 1
    fi

    if [[ ! -d fedora-lorax-templates ]]; then
        git clone https://pagure.io/fedora-lorax-templates.git
    else
        pushd fedora-lorax-templates > /dev/null || exit 1
        git fetch
        git reset --hard origin/main
        popd > /dev/null || exit 1
    fi

    version_number="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')"
    if [[ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]] || [[ -f "fedora-rawhide.repo" ]]; then
        version_pretty="Rawhide"
        version="rawhide"
    else
        version_pretty="${version_number}"
        version="${version_number}"
    fi
    source_url="https://kojipkgs.fedoraproject.org/compose/${version}/latest-Fedora-${version_pretty}/compose/Everything/x86_64/os/"
    volid="Fedora-${volid_sub}-x86_64-${version_pretty}"

    buildid=""
    if [[ -f ".buildid" ]]; then
        buildid="$(< .buildid)"
    else
        buildid="$(date '+%Y%m%d.0')"
        echo "${buildid}" > .buildid
    fi

    # Stick to the latest stable runtime available here
    # Only include a subset of Flatpaks here
    # Exhaustive list in https://pagure.io/pungi-fedora/blob/main/f/fedora.conf
    # flatpak_remote_refs="runtime/org.fedoraproject.Platform/x86_64/f39"
    # flatpak_apps=(
    #     "app/org.gnome.Calculator/x86_64/stable"
    #     "app/org.gnome.Calendar/x86_64/stable"
    #     "app/org.gnome.Extensions/x86_64/stable"
    #     "app/org.gnome.TextEditor/x86_64/stable"
    #     "app/org.gnome.clocks/x86_64/stable"
    #     "app/org.gnome.eog/x86_64/stable"
    # )
    # for ref in ${flatpak_refs[@]}; do
    #     flatpak_remote_refs+=" ${ref}"
    # done
    # FLATPAK_ARGS=""
    # FLATPAK_ARGS+=" --add-template=${pwd}/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl"
    # FLATPAK_ARGS+=" --add-template-var=flatpak_remote_name=fedora"
    # FLATPAK_ARGS+=" --add-template-var=flatpak_remote_url=oci+https://registry.fedoraproject.org"
    # FLATPAK_ARGS+=" --add-template-var=flatpak_remote_refs=${flatpak_remote_refs}"

    pwd="$(pwd)"

    lorax \
        --product=Fedora \
        --version=${version_pretty} \
        --release=${buildid} \
        --source="${source_url}" \
        --variant="${variant_pretty}" \
        --nomacboot \
        --isfinal \
        --buildarch=x86_64 \
        --volid="${volid}" \
        --logfile=${pwd}/logs/lorax.log \
        --tmp=${pwd}/tmp \
        --cachedir=cache/lorax \
        --rootfs-size=8 \
        --add-template=${pwd}/fedora-lorax-templates/ostree-based-installer/lorax-configure-repo.tmpl \
        --add-template=${pwd}/fedora-lorax-templates/ostree-based-installer/lorax-embed-repo.tmpl \
        --add-template-var=ostree_install_repo=file://${pwd}/repo \
        --add-template-var=ostree_update_repo=file://${pwd}/repo \
        --add-template-var=ostree_osname=fedora \
        --add-template-var=ostree_oskey=fedora-${version_number}-primary \
        --add-template-var=ostree_contenturl=mirrorlist=https://ostree.fedoraproject.org/mirrorlist \
        --add-template-var=ostree_install_ref=fedora/${version}/x86_64/${variant} \
        --add-template-var=ostree_update_ref=fedora/${version}/x86_64/${variant} \
        ${pwd}/iso/linux

# Upload the containers to a registry (Quay.io)
upload-container variant=default_variant:
    #!/bin/bash
    set -euxo pipefail

    variant={{variant}}

    declare -A pretty_names={{pretty_names}}
    variant_pretty=${pretty_names[$variant]-}
    if [[ -z $variant_pretty ]]; then
        echo "Unknown variant"
        exit 1
    fi

    if [[ -z ${CI_REGISTRY_USER+x} ]] || [[ -z ${CI_REGISTRY_PASSWORD+x} ]]; then
        echo "Skipping artifact archiving: Not in CI"
        exit 0
    fi
    if [[ "${CI}" != "true" ]]; then
        echo "Skipping artifact archiving: Not in CI"
        exit 0
    fi

    version=""
    if [[ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]] || [[ -f "fedora-rawhide.repo" ]]; then
        version="rawhide"
    else
        version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')"
    fi

    buildid=""
    if [[ -f ".buildid" ]]; then
        buildid="$(< .buildid)"
    else
        buildid="$(date '+%Y%m%d.0')"
        echo "${buildid}" > .buildid
    fi

    git_commit=""
    if [[ -n "${CI_COMMIT_SHORT_SHA}" ]]; then
        git_commit="${CI_COMMIT_SHORT_SHA}"
    else
        git_commit="$(git rev-parse --short HEAD)"
    fi

    # Login to the registry
    skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" quay.io

    # Copy to the new names
    image="quay.io/fedora-ostree-desktops/${variant}"

    # Use '--dest-compress-format zstd:chunked' only once 41 is released
    SKOPEO_ARGS="--retry-times 3 --dest-compress-format gzip"

    # Copy fully versioned tag (major version, build date/id, git commit)
    skopeo copy ${SKOPEO_ARGS} \
        "oci-archive:${variant}.ociarchive" \
        "docker://${image}:${version}.${buildid}.${git_commit}"

    # Update "un-versioned" tag (only major version)
    skopeo copy ${SKOPEO_ARGS} \
        "docker://${image}:${version}.${buildid}.${git_commit}" \
        "docker://${image}:${version}"

    if [[ "${variant}" == "kinoite-nightly" ]]; then
        # Update latest tag for kinoite-nightly only
        skopeo copy ${SKOPEO_ARGS} \
            "docker://${image}:${version}.${buildid}.${git_commit}" \
            "docker://${image}:latest"
    fi

    # Copy to legacy names if needed
    declare -A legacy_names={{legacy_names}}
    variant_legacy=${legacy_names[$variant]-}
    if [[ -n ${variant_legacy} ]]; then
        image="quay.io/fedora-ostree-desktops/${variant_legacy}"

        # Copy fully versioned tag (major version, build date/id, git commit)
        skopeo copy ${SKOPEO_ARGS} \
            "oci-archive:${variant}.ociarchive" \
            "docker://${image}:${version}.${buildid}.${git_commit}"

        # Update "un-versioned" tag (only major version)
        skopeo copy ${SKOPEO_ARGS} \
            "docker://${image}:${version}.${buildid}.${git_commit}" \
            "docker://${image}:${version}"
    fi