From: Don Zickus <dzickus@xxxxxxxxxx>
Add new os-build targets: rt-devel and automotive-devel
This is an attempt to automate the rt and automotive devel branches
using the upstream linux-rt-devel tree as the base combined with
os-build.
The overall process isn't too complicated but there are enough steps to
make it compilicated.
Steps:
* map upstream linux-rt-devel to master-rt branch
* update os-build
* grab linux version from master-rt and os-build
* if version the same then merge os-build and master-rt to
os-build-rt-automated
* else merge tag kernel-N.V.0-0 and master-rt to os-build-rt-automated until
master-rt is update to os-build
* merge os-build-rt-automated into os-build-rt-devel
* merge os-build-rt-automated into os-build-automotive-devel
* run the generate pending-rhel config scripts on rt-devel and
automotive-devel
The script has beginning logic to handle rebasing if necessary when the
rt-devel branch transitions from os-build-stable (linux-stable) to linux
master again.
NOTE: The script uses kernel-N.V.0-0 which is rebased after os-build
linux-x.y GA is created but before linux-x.y+1 pre-rc1 merges happen.
The reason for this is because linux-stable-x.y doesn't exist until
linux-x.y+1-rc1 exists thus leaving linux-stable-x.y-1 in the meantime.
An awkward gap exists, use kernel-N.V.0-0 as the base.
The script has no effect on the day-to-day operations of os-build. They
are designed to be run from a gitlab cron job and update in the
background. Once they are deemed stable, adding ARK MRs that target
either os-build-rt-devel or os-build-automotive-devel will be possible
and those branches can start proper parallel developement with os-build.
Cleanup namespace pollution because shellcheck doesn't like 'local'.
Signed-off-by: Don Zickus <dzickus@xxxxxxxxxx>
diff --git a/redhat/scripts/ci/ark-ci-env.sh b/redhat/scripts/ci/ark-ci-env.sh
index blahblah..blahblah 100644
--- a/redhat/scripts/ci/ark-ci-env.sh
+++ b/redhat/scripts/ci/ark-ci-env.sh
@@ -8,24 +8,188 @@ die()
ci_pre_check()
{
- if test -n "${TO_PUSH}"; then
- if test -z "${GITLAB_PROJECT_URL}" || test -z "$GITLAB_PROJECT_PUSHURL"; then
- echo "To enable git-push, please run:"
- echo "git remote add gitlab <url>"
- echo "git remote set-url --push gitlab <pushurl>"
+ if test -z "${GITLAB_PROJECT_URL}" || test -z "$GITLAB_PROJECT_PUSHURL"; then
+ echo "To enable git-push, please run:"
+ echo "git remote add gitlab <url>"
+ echo "git remote set-url --push gitlab <pushurl>"
+ if test -n "${TO_PUSH}"; then
die "Misconfigured 'gitlab' entry for git"
fi
fi
git diff-index --quiet HEAD || die "Dirty tree, please clean before merging."
}
+# wrapper around branches that may not be exist yet
+ark_git_branch()
+{
+ _target_branch="$1"
+ _source_branch="$2"
+
+ # switch to branch if it exists otherwise create and set to source
+ # branch
+ git show-ref -q --heads "$_target_branch" || \
+ git branch "$_target_branch" "$_source_branch"
+}
+
+# GitLab can only mirror one project at a time. This wrapper function does
+# the mirroring for any other branches.
+ark_git_mirror()
+{
+ target_branch="$1"
+ upstream_tree="$2"
+ source_branch="$3"
+ reset_branch="$4"
+
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ remote_branch="$upstream_tree/$source_branch"
+ ark_git_branch "$target_branch" "$remote_branch"
+ git checkout "$target_branch"
+ git fetch "$upstream_tree" "$source_branch"
+ if test -z "$reset_branch"; then
+ git merge "$remote_branch" || die "git merge $remote_branch failed"
+ else
+ git reset --hard "$remote_branch" || die "git reset $remote_branch failed"
+ fi
+ git checkout "$prev_branch"
+}
+
+# Merge wrapper in case issues arise
+ark_git_merge()
+{
+ source_branch="$1"
+ target_branch="$2"
+ reset_branch="$3"
+
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ ark_git_branch "$target_branch" "$source_branch"
+ git checkout "$target_branch"
+ if test -n "$reset_branch"; then
+ # there are cases when the initial merge is a reset
+ git reset --hard "$source_branch" || die "git reset $source_branch failed"
+ elif ! git merge -m "Merge '$source_branch' into '$target_branch'" "$source_branch"; then
+ git merge --abort
+ printf "Merge conflict; halting!\n"
+ printf "To reproduce:\n"
+ printf "* git checkout %s\n" "${target_branch}"
+ printf "* git merge %s\n" "${source_branch}"
+ die "Merge conflicts"
+ fi
+
+ git checkout "$prev_branch"
+ return 0
+}
+
+ark_git_rebase()
+{
+ rebase_branch="$1"
+ _upstream="$2"
+ _base="$3"
+
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ git checkout "${rebase_branch}"
+ if ! git rebase --onto "$_base" "$_upstream"; then
+ git rebase --abort
+ printf "Rebase conflict; halting!\n"
+ printf "To reproduce:\n"
+ printf "* git checkout %s\n" "${rebase_branch}"
+ printf "* git rebase --onto %s %s\n" "${_base}" "${_upstream}"
+ die "Rebase conflicts"
+ fi
+ git checkout "$prev_branch"
+ return 0
+}
+
+ark_update_configs()
+{
+ config_branch="$1"
+ skip_configs="$2"
+
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ git checkout "${config_branch}"
+
+ # Generates and commits all the pending configs
+ make -j FLAVOR=fedora dist-configs-commit
+ # Skip executing gen_config_patches.sh for new Fedora configs
+
+ old_head="$(git rev-parse HEAD)"
+ make -j FLAVOR=rhel dist-configs-commit
+ new_head="$(git rev-parse HEAD)"
+
+
+ # Converts each new pending config from above into its finalized git
+ # configs/<date>/<config> config_branch. These commits are used for Merge
+ # Requests.
+ [ "$old_head" != "$new_head" ] && CONFIGS_ADDED="1" || CONFIGS_ADDED=""
+
+ if test "$CONFIGS_ADDED"; then
+ if test -z "$skip_configs"; then
+ git checkout "$prev_branch"
+ ./redhat/scripts/genspec/gen_config_patches.sh "$config_branch"
+ fi
+ else
+ printf "No new configuration values exposed from "
+ printf "merging %s into $BRANCH\n" "$UPSTREAM_REF"
+ fi
+
+ git checkout "$prev_branch"
+ test -z "$CONFIGS_ADDED" && return 0 || return 1
+}
+
+ark_push_changes()
+{
+ push_branch="$1"
+ skip_configs="$2"
+
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ git checkout "${push_branch}"
+
+ TMPFILE=".push-warnings"
+ touch "$TMPFILE"
+
+ test "$TO_PUSH" && PUSH_VERB="Pushing" || PUSH_VERB="To push"
+ PUSH_STR="branch ${push_branch} to ${GITLAB_URL}"
+ PUSH_CMD="git push gitlab ${push_branch}"
+ PUSH_CONFIG_STR="config update branches"
+ PUSH_CONFIG_CMD="for conf_branch in \$(git branch | grep configs/${push_branch}/\"\$(date +%F)\"); do
+ git push \\
+ -o merge_request.create \\
+ -o merge_request.target=\"$push_branch\" \\
+ -o merge_request.remove_source_branch \\
+ gitlab \"\$conf_branch\" 2>&1 | tee -a $TMPFILE
+ done
+ "
+
+ #Push push_branch
+ echo "# $PUSH_VERB $PUSH_STR"
+ echo "$PUSH_CMD"
+ test "$TO_PUSH" && eval "$PUSH_CMD"
+
+ #Push config branches if created
+ if test -z "$skip_configs"; then
+ echo
+ echo "# $PUSH_VERB $PUSH_CONFIG_STR"
+ echo "$PUSH_CONFIG_CMD"
+ test "$TO_PUSH" && eval "$PUSH_CONFIG_CMD"
+ fi
+
+ # GitLab server side warnings do not fail git-push but leave verbose
+ # WARNING messages. Grep for those and consider it a script
+ # failure. Make sure all push_branches are pushed first as follow up
+ # git-pushes may succeed.
+ grep -q "remote:[ ]* WARNINGS" "$TMPFILE" && die "Server side warnings"
+
+ rm "$TMPFILE"
+ git checkout "$prev_branch"
+ return 0
+}
+
# Common variables for all CI scripts
UPSTREAM_REF=${1:-"master"}
BRANCH=${2:-"os-build"}
PROJECT_ID=${PROJECT_ID:-"13604247"}
TO_PUSH=${DIST_PUSH:-""}
-GITLAB_PROJECT_URL="$(git remote get-url gitlab 2>/dev/null)"
-GITLAB_PROJECT_PUSHURL="$(git config --get remote.gitlab.pushurl 2>/dev/null)"
+GITLAB_PROJECT_URL="$(git remote get-url gitlab 2>/dev/null)" || true
+GITLAB_PROJECT_PUSHURL="$(git config --get remote.gitlab.pushurl 2>/dev/null)" || true
ci_pre_check
diff --git a/redhat/scripts/ci/ark-merge-rt.sh b/redhat/scripts/ci/ark-merge-rt.sh
new file mode 100755
index blahblah..blahblah 100755
--- /dev/null
+++ b/redhat/scripts/ci/ark-merge-rt.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+#
+# This script is intended to sync up the RT and automotive branch (derivative
+# of RT). It adds the extra twist of detecting the right upstream rt branch
+# to sync with depending on the existance of the next branch. Sometimes the
+# rt-devel branch waits until -rc1/2 to create new branches.
+# Finally the code handles the rebases in those cases where newer branches
+# are available.
+#
+# Why the complexity?
+# Development branches will need to be periodically rebased unfortunately.
+# Using 'git rebase --onto <new head> <old_head>' only works with one common
+# branch to rebase from not two. Meanwhile, the -devel branches are formed
+# from two upstream branches, os-build and linux-rt-devel. The idea is
+# to merge the two branches into a single throwaway branch that can be
+# recreated from scratch anytime then use that as the base for -devel.
+
+set -e
+
+# source common CI functions and variables
+# shellcheck disable=SC1091
+. "$(dirname "$0")"/ark-ci-env.sh
+
+#Upstream RT tree git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git
+UPSTREAM_RT_TREE_URL="git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git"
+UPSTREAM_RT_TREE_NAME="linux-rt-devel"
+DOWNSTREAM_RT_BRANCH="master-rt-devel"
+RT_AUTOMATED_BRANCH="os-build-rt-automated"
+RT_DEVEL_BRANCH="os-build-rt-devel"
+AUTOMOTIVE_DEVEL_BRANCH="os-build-automotive-devel"
+
+# verify git remote rt is setup
+if ! git remote get-url "$UPSTREAM_RT_TREE_NAME" 2>/dev/null; then
+ die "Please 'git remote add linux-rt-devel $UPSTREAM_RT_TREE_URL'"
+fi
+
+# grab the os-build base branches
+ark_git_mirror "os-build" "origin" "os-build"
+ark_git_mirror "master" "origin" "master"
+
+# make sure tags are available for git-describe to correctly work
+git fetch -t origin
+
+# upstream -rt devel branches are aligned with version numbers and are not
+# always up to date with master. Figure out which branch to mirror based on
+# version number and existance. We may have to trigger a rebase.
+
+# what are the current versions of rt-devel and os-build (use 'master' to
+# avoid fedora tagging of kernel-X.Y.0.0.....)
+# use git tags which are always 'vX.Y-rcZ-aaa-gbbbbb' or 'vX.Y-aaa-gbbbbb'
+# where X.Y is the version number that maps to linux-rt's branch of
+# 'linux-X.Y.y'
+get_upstream_version()
+{
+ branch=$1
+ upstream="master"
+
+ # Thanks to pre-rc1 merging, we have a 2 week window with an
+ # incorrect version number. Detect and correct.
+ mergebase="$(git merge-base "$branch" "$upstream")"
+ raw_version="$(git describe "$mergebase")"
+ version="$(git show "$branch":Makefile | sed -ne '/^VERSION\ =\ /{s///;p;q}')"
+ patchlevel="$(git show "$branch":Makefile | sed -ne '/^PATCHLEVEL\ =\ /{s///;p;q}')"
+ kver="${version}.${patchlevel}"
+
+ #-rc indicates no tricks necessary, return version
+ if echo "${raw_version}" | grep -q -- "-rc"; then
+ echo "$kver"
+ return
+ fi
+
+ #if -gXXX is _not_ there, must be a GA release, use version
+ if ! echo "${raw_version}" | grep -q -- "-g"; then
+ echo "$kver"
+ return
+ fi
+
+ #must be a post tag release with -g but not -rcX, IOW an rc0.
+ #Add a 1 to the version number
+ echo "${version}.$((patchlevel + 1))"
+}
+
+# To handle missing branches, precalculate previous kernel versions to fetch
+get_prev_version()
+{
+ version_str=$1
+
+ version="$(echo "$version_str" | cut -c1)"
+ patchlevel="$(echo "$version_str" | cut -c3)"
+
+ echo "${version}.$((patchlevel - 1))"
+}
+
+OS_BUILD_VER="$(get_upstream_version os-build)"
+OS_BUILD_VER_prev="$(get_prev_version "$OS_BUILD_VER")"
+
+# check latest upstream RT branch
+if git fetch -q "$UPSTREAM_RT_TREE_NAME" "linux-${OS_BUILD_VER}.y-rt"; then
+ UPSTREAM_RT_DEVEL_VER="${OS_BUILD_VER}"
+elif git fetch -q "$UPSTREAM_RT_TREE_NAME" "linux-${OS_BUILD_VER_prev}.y-rt"; then
+ UPSTREAM_RT_DEVEL_VER="${OS_BUILD_VER_prev}"
+else
+ die "Neither version ($OS_BUILD_VER, $OS_BUILD_VER_prev) in upstream tree: $UPSTREAM_RT_TREE_NAME"
+fi
+
+# verify the core branches exist or use provided defaults
+UPSTREAM_RT_DEVEL_BRANCH="linux-${UPSTREAM_RT_DEVEL_VER}.y-rt"
+ark_git_branch "$DOWNSTREAM_RT_BRANCH" "$UPSTREAM_RT_TREE_NAME/$UPSTREAM_RT_DEVEL_BRANCH"
+ark_git_branch "$RT_AUTOMATED_BRANCH" "$UPSTREAM_RT_TREE_NAME/$UPSTREAM_RT_DEVEL_BRANCH"
+ark_git_branch "$RT_DEVEL_BRANCH" "$UPSTREAM_RT_TREE_NAME/$UPSTREAM_RT_DEVEL_BRANCH"
+ark_git_branch "$AUTOMOTIVE_DEVEL_BRANCH" "$UPSTREAM_RT_TREE_NAME/$UPSTREAM_RT_DEVEL_BRANCH"
+
+MASTER_RT_DEVEL_VER="$(get_upstream_version "$DOWNSTREAM_RT_BRANCH")"
+RT_AUTOMATED_VER="$(get_upstream_version $RT_AUTOMATED_BRANCH)"
+RT_DEVEL_VER="$(get_upstream_version $RT_DEVEL_BRANCH)"
+AUTOMOTIVE_DEVEL_VER="$(get_upstream_version $AUTOMOTIVE_DEVEL_BRANCH)"
+
+OS_BUILD_BASE_BRANCH="os-build"
+RT_REBASE=""
+
+if test "$UPSTREAM_RT_DEVEL_VER" != "$OS_BUILD_VER"; then
+ # no newer upstream branch to rebase onto, continue with an
+ # os-build stable tag
+ OS_BUILD_BASE_BRANCH="kernel-${MASTER_RT_DEVEL_VER}.0-0"
+fi
+
+# sanity check, sometimes broken scripts leave a mess
+if test "$MASTER_RT_DEVEL_VER" != "$UPSTREAM_RT_DEVEL_VER" -o \
+ "$MASTER_RT_DEVEL_VER" != "$RT_AUTOMATED_VER" -o \
+ "$MASTER_RT_DEVEL_VER" != "$RT_DEVEL_VER" -o \
+ "$MASTER_RT_DEVEL_VER" != "$AUTOMOTIVE_DEVEL_VER"; then
+ # rebase time
+ RT_REBASE="yes"
+fi
+
+## PREP the upstream branches
+# on a rebase, propogate all the git resets
+# fetch the determined rt-devel branch
+ark_git_mirror "$DOWNSTREAM_RT_BRANCH" "$UPSTREAM_RT_TREE_NAME" "$UPSTREAM_RT_DEVEL_BRANCH" "$RT_REBASE"
+# finally merge the two correct branches
+ark_git_merge "$OS_BUILD_BASE_BRANCH" "$RT_AUTOMATED_BRANCH" "$RT_REBASE"
+ark_git_merge "$DOWNSTREAM_RT_BRANCH" "$RT_AUTOMATED_BRANCH"
+
+## MERGE the upstream branches to the development branches
+if test -n "$RT_REBASE"; then
+ # handle the rebase
+ # rebases usually go from prev version to new version
+ # rebuild the prev merge base in case the previous automated one is
+ # corrupted.
+ prev_branch="$(git rev-parse --abbrev-ref HEAD)"
+ temp_branch="_temp_rt_devel_$(date +%F)"
+ git branch -D "$temp_branch" 2>/dev/null
+ git checkout -b "$temp_branch" "kernel-${OS_BUILD_VER_prev}.0-0"
+ git merge "$UPSTREAM_RT_TREE_NAME/linux-${OS_BUILD_VER_prev}.y-rt"
+ git checkout "$prev_branch"
+ ark_git_rebase "$RT_DEVEL_BRANCH" "$temp_branch" "$RT_AUTOMATED_BRANCH"
+ ark_git_rebase "$AUTOMOTIVE_DEVEL_BRANCH" "$temp_branch" "$RT_AUTOMATED_BRANCH"
+ git branch -D "$temp_branch"
+fi
+
+## Build -rt-devel branch, generate pending-rhel configs
+ark_git_merge "$RT_AUTOMATED_BRANCH" "$RT_DEVEL_BRANCH"
+# don't care if configs were added or not hence '|| true'
+ark_update_configs "$RT_DEVEL_BRANCH" || true
+# skip pushing config update MRs, keep them in pending-rhel
+ark_push_changes "$RT_DEVEL_BRANCH" "skip"
+
+## Build -automotive-devel branch, generate pending-rhel configs
+ark_git_merge "$RT_AUTOMATED_BRANCH" "$AUTOMOTIVE_DEVEL_BRANCH"
+# don't care if configs were added or not hence '|| true'
+ark_update_configs "$AUTOMOTIVE_DEVEL_BRANCH" || true
+# skip pushing config update MRs, keep them in pending-rhel
+ark_push_changes "$AUTOMOTIVE_DEVEL_BRANCH" "skip"
--
https://gitlab.com/cki-project/kernel-ark/-/merge_requests/2732
--
_______________________________________________
kernel mailing list -- kernel@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to kernel-leave@xxxxxxxxxxxxxxxxxxxxxxx
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/kernel@xxxxxxxxxxxxxxxxxxxxxxx
Do not reply to spam, report it: https://pagure.io/fedora-infrastructure/new_issue
