Move and rename repos, upgrade to Catalyst 4, support SDK on arm64

.github/workflows/common.sh

@@ -14,9 +14,8 @@ if [[ ! -d "${WORK_SCRIPTS_DIR:-}" ]]; then
 fi
 
 readonly SDK_OUTER_TOPDIR="${WORK_SCRIPTS_DIR}"
-readonly SDK_OUTER_OVERLAY="${SDK_OUTER_TOPDIR}/sdk_container/src/third_party/coreos-overlay"
-readonly SDK_INNER_SRCDIR="/mnt/host/source/src"
-readonly SDK_INNER_OVERLAY="${SDK_INNER_SRCDIR}/third_party/coreos-overlay"
+readonly SDK_OUTER_OVERLAY="${SDK_OUTER_TOPDIR}/repos/flatcar-overlay"
+readonly SDK_INNER_OVERLAY="/mnt/host/source/src/scripts/repos/flatcar-overlay"
 
 readonly BUILDBOT_USERNAME="Flatcar Buildbot"
 readonly BUILDBOT_USEREMAIL="[email protected]"

.github/workflows/image_changes.sh

@@ -25,8 +25,8 @@ function github_ricj_callback() {
     show_changes_env+=(
         # Override the default locations of repositories.
         "SCRIPTS_REPO=."
-        "COREOS_OVERLAY_REPO=../coreos-overlay"
-        "PORTAGE_STABLE_REPO=../portage-stable"
+        "FLATCAR_OVERLAY_REPO=../flatcar-overlay"
+        "GENTOO_SUBSET_REPO=../gentoo-subset"
     )
     show_changes_params+=(
         # We may not have a tag handy, so we tell show-changes

.github/workflows/update-portage-stable-packages-from-list.yaml → .github/workflows/update-gentoo-subset-packages-from-list.yaml

@@ -1,4 +1,4 @@
-name: Keep portage-stable packages updated
+name: Keep gentoo-subset packages updated
 on:
   schedule:
     - cron:  '0 7 * * 1'
@@ -35,10 +35,10 @@ jobs:
           git config --global user.name "Flatcar Buildbot"
           git config --global user.email "[email protected]"
           old_head=$(git -C scripts rev-parse HEAD)
-          packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list)
+          packages_list=$(realpath scripts/.github/workflows/gentoo-subset-packages-list)
           gentoo_repo=$(realpath gentoo)
           build_scripts=$(realpath flatcar-build-scripts)
-          pushd scripts/sdk_container/src/third_party/portage-stable
+          pushd scripts/repos/gentoo-subset
           while read -r package; do
               if [[ ! -e "${package}" ]]; then
                   # If this happens, it means that the package was moved to overlay
@@ -50,7 +50,7 @@ jobs:
                   # If this happens, it means that the package was obsoleted or moved
                   # in Gentoo. The obsoletion needs to be handled in the case-by-case
                   # manner, while move should be handled by doing the same move
-                  # in portage-stable. The build should not break because of the move,
+                  # in gentoo-subset. The build should not break because of the move,
                   # because most likely it's already reflected in the profiles/updates
                   # directory.
                   echo "::warning title=${package}::Obsolete or moved package"
@@ -73,10 +73,10 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: scripts
-          branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
+          branch: buildbot/weekly-gentoo-subset-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
           delete-branch: true
           base: main
-          title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
+          title: Weekly gentoo-subset package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
           body: |
             CI: TODO
 

.github/workflows/update-metadata-glsa.yaml

@@ -15,7 +15,7 @@ jobs:
       - name: Update GLSA metadata
         id: update-glsa-metadata
         run: |
-          glsa_dir='sdk_container/src/third_party/portage-stable/metadata/glsa'
+          glsa_dir='repos/gentoo-subset/metadata/glsa'
           rm -rf "${glsa_dir}"
           mkdir --parents "${glsa_dir}"
           rsync --archive rsync://rsync.gentoo.org/gentoo-portage/metadata/glsa/* "${glsa_dir}"
@@ -30,6 +30,6 @@ jobs:
           base: main
           title: Monthly GLSA metadata ${{steps.update-glsa-metadata.outputs.TODAYDATE }}
           body: Updated GLSA metadata
-          commit-message: "portage-stable/metadata: Monthly GLSA metadata updates"
+          commit-message: "gentoo-subset/metadata: Monthly GLSA metadata updates"
           author: Flatcar Buildbot <[email protected]>
           labels: main

PREFIX.md

@@ -35,7 +35,7 @@ Prefix uses a _staging environment_ to build binary packages, then installs thes
 The _staging environment_ contains toolchains and all build tools required to create binary packages (a full `@system`).
 The _final environment_ only contains run-time dependencies.
 
-Packages are built from ebuilds in coreos-overlay, portage-stable, and prefix-overlay.
+Packages are built from ebuilds in flatcar-overlay, gentoo-subset, and prefix-overlay.
 
 A QoL `emerge` wrapper is included to install packages to the prefix.
 

README.md

@@ -14,20 +14,18 @@ The SDK can be used to
 # Using the scripts repository
 
 The repository is meant to be the entry point for Flatcar builds and development.
-Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/):
+Ebuilds for all packages reside in one of 2 subdirectories - [flatcar-overlay](repos/flatcar-overlay) and [gentoo-subset](repos/gentoo-subset/):
 ```
 scripts
-   +--sdk_container
-          +---------src
-                     +--third_party
-                             +------coreos-overlay
-                             +------portage-stable
+   +--repos
+          +--flatcar-overlay
+          +--gentoo-subset
 ```
 
-`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
+`gentoo-subset` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
 Consider it a small sub-set of Gentoo.
 
-`coreos-overlay` contains significantly modified or even entirely self-written ebuilds.
+`flatcar-overlay` contains significantly modified or even entirely self-written ebuilds.
 
 The `scripts` repository makes ample use of tags to mark releases.
 Sometimes, local and origin tags can diverge (e.g. when re-tagging something locally to test a build).
@@ -52,7 +50,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready
 
 The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo:
 * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container.
-* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
+* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the gentoo-subset and flatcar-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
 
 ## Standalone mode
 
@@ -82,7 +80,7 @@ To start a container in privileged mode with `/dev` available use:
 
 This is the preferred mode of working with the SDK.
 Interaction with the container happens via wrapper scripts from the scripts repository.
-Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly.
+Both the host's scripts repo as well as the ebuild paths (gentoo-subset and flatcar-overlay) are made available in the container, allowing for work on these directly.
 The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency.
 
 To clone the scripts repo and pick a version:

bootstrap_sdk

@@ -7,11 +7,11 @@
 # This uses Gentoo's catalyst for very thoroughly building images from
 # scratch. Using images based on this will eliminate some of the hackery
 # in make_chroot.sh for building up the sdk from a stock stage3 tarball.
-# 
+#
 # For reference the procedure it performs is this:
-# 
-# 1. snapshot: Grab a snapshot of the portage-stable repo from
-#     the current SDK's /var/lib/gentoo/repos/gentoo.
+#
+# 1. snapshot: Grab a snapshot of the gentoo-subset repo from
+#     the current SDK's /var/gentoo/repos/gentoo-subset.
 #     Alternatively, check out a git ref specified via --portage-ref.
 #
 # 2. stage1: Using a "seed" tarball as a build environment, build a
@@ -22,20 +22,14 @@
 #      will be linked against the SEED SDK libraries, NOT against libraries
 #      built in stage 1. See "stage_repo()" documentation further below for more.
 #     This stage uses:
-#     - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
+#     - gentoo-subset from the SDK's /var/gentoo/repos/gentoo-subset
 #        or a custom path via --stage1_portage_path command line option
-#     - coreos-overlay  from the SDK's /var/lib/gentoo/repos/coreos-overlay
+#     - flatcar-overlay  from the SDK's /var/gentoo/repos/flatcar-overlay
 #        or a custom path via --stage1_overlay_path command line option
 #     Command line option refs need caution though, since
 #     stage1 must not contain updated ebuilds (see build_stage1 below).
 #
-# 3. stage2: Run portage-stable/scripts/bootstrap.sh
-#     This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
-#     to or otherwise influenced by whatever was in the "seed" tarball.
-#     The toolchain rebuild may contain updated package ebuilds from
-#     third_party/(portage-stable|coreos-overlay).
-#     This and all following stages use portage-stable and coreos-overlay
-#     from third_party/... (see 1.)
+# 3. stage2: This is skipped as recommended by upstream Gentoo.
 #
 # 4. stage3: Run emerge -e system to rebuild everything using the fresh updated
 #     toolchain from 3., using the normal USE flags provided by the profile. This
@@ -45,7 +39,7 @@
 # 5. stage4: Install any extra packages or other desired tweaks. For the
 #     sdk we just install all the packages normally make_chroot.sh does.
 #
-# Usage: bootstrap_sdk [stage1 stage2 etc]
+# Usage: bootstrap_sdk [stage1 stage3 etc]
 # By default all four stages will be built using the latest stage4 as a seed.
 
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
@@ -59,24 +53,17 @@ TYPE="flatcar-sdk"
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
 
-DEFINE_string stage1_portage_path "" \
-  "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-DEFINE_string stage1_overlay_path "" \
-  "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-
-
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
-target: stage4
 pkgcache_path: $BINPKGS
 stage4/packages: coreos-devel/sdk-depends
 stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
 stage4/root_overlay: ${ROOT_OVERLAY}
-stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
+stage4/empty: /root /var/cache/edb
 stage4/rm: /etc/machine-id /etc/resolv.conf
 EOF
-catalyst_stage_default
+catalyst_stage_default 4
 }
 
 # Switch to HTTP because early boostrap stages do not have SSL support.
@@ -91,16 +78,7 @@ ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
 if [[ "$STAGES" =~ stage4 ]]; then
     info "Setting release to ${FLATCAR_VERSION}"
     rm -rf "${ROOT_OVERLAY}"
-    # need to setup the lib->lib64 symlink correctly
-    libdir=$(get_sdk_libdir)
-    mkdir -p "${ROOT_OVERLAY}/usr/${libdir}"
-    if [[ "${libdir}" != lib ]]; then
-        if [[ "$(get_sdk_symlink_lib)" == "yes" ]]; then
-            ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib"
-        else
-            mkdir -p "${ROOT_OVERLAY}/usr/lib"
-        fi
-    fi
+    mkdir -p "${ROOT_OVERLAY}"
     "${BUILD_LIBRARY_DIR}/set_lsb_release" \
         --root "${ROOT_OVERLAY}"
 fi
@@ -113,125 +91,36 @@ chmod 1777 "${ROOT_OVERLAY}/tmp"
 cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 
 
-# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable)
-#  to build a minimal toolchain (USE="-*") for stage 2.
+# Stage 1 uses "known-good" ebuilds (from both flatcar-overlay and gentoo-subset)
+#  to build a minimal toolchain (USE="-*") for stage 3.
 #
-# No package updates must happen in stage 1, so we use the portage-stable and
-#  coreos-overlay paths included with the current SDK (from the SDK chroot's
-#  /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
+# No package updates must happen in stage 1, so we use the gentoo-subset and
+#  flatcar-overlay paths included with the current SDK (from the SDK chroot's
+#  /var/gentoo/repos/). "Current SDK" refers to the SDK we entered with
 #  'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
 #
 # Using ebuilds from the above mentioned sources will ensure that stage 1 builds
-#  a minimal stage 2 from known-good ebuild versions - the same ebuild versions
+#  a stage 3 from known-good ebuild versions - the same ebuild versions
 #  that were used to build the very SDK we run ./bootstrap_sdk in.
 #
 # DANGER ZONE
 #
 # Stage 1 lacks proper isolation and will link all packages built for
-#  stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
-#  installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot).
+#  stage 3 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
+#  installed into the FS root of the stage 3 seed ("/tmp/stage1root" in the catalyst chroot).
 #  This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use
 #  "known good" ebuild repo versions outlined above.
 #
 # In special circumstances it may be required to circumvent this and use custom paths
 #  for either (or both) portage and overlay. The command line options
 #  --stage1-portage-path and --stage1-overlay-path may be used to specify
 #  a repo path known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
-#  will be updated prior to starting to build stage 2.
+#  will be updated prior to starting to build stage 3.
 #  NOTE that this should never be used to introduce library updates in stage 1. All binaries
 #  produced in stage 1 are linked against libraries in the seed tarball, NOT libraries produced
-#  by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
+#  by stage 1. Therefore, these binaries will cease to work in stage 3 when linked against
 #  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
 
-stage_repo() {
-    local repo=${1}
-    local path=${2}
-    local dest=${3}
-    local update_seed_file=${4}
-    local gitname="$repo"
-
-    if [ "$gitname" = "gentoo" ] ; then
-        gitname="portage-stable"
-    fi
-
-    if [ -z "$path" ]; then
-        cp -R "/var/gentoo/repos/${repo}" "$dest"
-        info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
-    else
-        mkdir "$dest/$repo"
-        cp -R "${path}/"* "$dest/${repo}/"
-        info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
-        info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
-    fi
-    (
-        set -euo pipefail
-        local repo_var hook name
-
-        # FLAGS_coreos_overlay for gitname coreos-overlay
-        repo_var="FLAGS_${gitname//-/_}"
-        shopt -s nullglob
-        for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
-            name=${hook##*/}
-            name=${name%"-${gitname}.sh"}
-            info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
-            "${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
-        done
-    )
-}
-
-build_stage1() {
-    # First, write out the default 4-stage catalyst configuration files
-    write_configs
-
-    # Prepare local copies of both the "known-good" portage-stable and the
-    #  "known-good" coreos-overlay ebuild repos
-    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
-    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
-    rm -rf "$stage1_repos"
-    mkdir "$stage1_repos"
-
-    # If the file exists and is not empty, seed will be updated.
-    # Stage1 hooks may decide that the seed SDK needs updating.
-    local update_seed_file="${TEMPDIR}/update_seed"
-
-    # prepare ebuild repos for stage 1, either from the local SDK (default)
-    #  or from custom paths specified via command line flags
-    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
-    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"
-
-    # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
-    #  This requires us to create a custom catalyst config to point it to the
-    #  repo copy we just created, for snapshotting.
-    catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
-    sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
-         "$TEMPDIR/catalyst-stage1.conf"
-    # take the "portage directory" (portage-stable copy) snapshot
-    build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"
-
-    # Update the stage 1 spec to use the "known-good" portage-stable snapshot
-    #  and coreos-overlay copy repository versions from above.
-    sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
-           -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
-        "$TEMPDIR/stage1.spec"
-
-    # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
-    if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
-        sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
-        echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
-            >>"$TEMPDIR/stage1.spec"
-    fi
-    rm -f "${update_seed_file}"
-
-    # Finally, build stage 1
-    build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
-}
-
-if [[ "$STAGES" =~ stage1 ]]; then
-    build_stage1
-    STAGES="${STAGES/stage1/}"
-    SEED="${TYPE}/stage1-${ARCH}-latest"
-fi
-
 catalyst_build
 
 if [[ "$STAGES" =~ stage4 ]]; then