mirror: Adjust sync-kernel.sh for bpftool
Copied from libbpf in the previous commit, the sync-kernel.sh script
will help us synchronise the source files in this repository with the
original ones, in the kernel repository. But it needs a few adjustments
to work with bpftool. These include:
- Adding the reference commits in CHECKPOINT-COMMIT and
BPF-CHECKPOINT-COMMIT
- Making it possible to pass BPF_NEXT_COMMIT_TIP and BPF_COMMIT_TIP to
the script, so we can synchronise with libbpf's checkpoints instead of
current tips of kernel branches
- Updating the list of files to synchronise, as well as the relevant
filters
- Add some processing for copying or moving around source files into
different subdirectories
- Removing the synchronisation step for bpf_helper_defs.h
- Fix variables/argument parsing order to get usage() and not an error
from "cat" if no argument is provided
- Skip cherry-picking for commits on branch bpf that are already in
bpf-next (see note below).
- Add a few info messages before cherry-picking commits for each branch
- Add color (auto mode) on the final diff to check consistency
- Renaming most references to "libbpf" into "bpftool"
Note on cherry-picking:
We add a check to skip cherry-picking for commits on branch bpf that
are already on branch bpf-next (commits that are ancestors to the
selected bpf-next tip). The rationale is that all commits from
bpf-next are already dealt with when we process bpf: the initial files
were added from bpf-next, and then we always cherry-pick from bpf-next
before we do bpf. This goes faster, and it avoids conflicts during the
early life of the mirror, when commits in bpf are already accounted
for but when the relevant signatures are not yet in the mirror's
history (because the commits in bpf-next are anterior to the initial
import of the files). The latter point should not be an issue for
long, given that synchronisation with bpf-next will make signatures of
new commits available in the history.
Signed-off-by: Quentin Monnet <[email protected]>
diff --git a/scripts/sync-kernel.sh b/scripts/sync-kernel.sh
index b5fa2aa..ab87e79 100755
--- a/scripts/sync-kernel.sh
+++ b/scripts/sync-kernel.sh
@@ -1,30 +1,34 @@
-#!/bin/bash
+#!/usr/bin/env bash
usage () {
- echo "USAGE: ./sync-kernel.sh <libbpf-repo> <kernel-repo> <bpf-branch>"
+ echo "USAGE: ./sync-kernel.sh <bpftool-repo> <kernel-repo> <bpf-branch>"
echo ""
- echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <libbpf-repo>/CHECKPOINT-COMMIT."
- echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <libbpf-repo>/BPF-CHECKPOINT-COMMIT."
+ echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <bpftool-repo>/CHECKPOINT-COMMIT."
+ echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <bpftool-repo>/BPF-CHECKPOINT-COMMIT."
+ echo "Set BPF_NEXT_TIP_COMMIT to override bpf-next tree target commit, otherwise use HEAD from current branch in <kernel-repo>."
+ echo "Set BPF_TIP_COMMIT to override bpf tree target commit, otherwise use tip from <bpf-branch> in <kernel-repo>."
echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits."
exit 1
}
set -eu
-LIBBPF_REPO=${1-""}
+BPFTOOL_REPO=${1-""}
LINUX_REPO=${2-""}
BPF_BRANCH=${3-""}
-BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)}
-BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)}
-if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then
- echo "Error: libbpf or linux repos are not specified"
+if [ -z "${BPFTOOL_REPO}" ] || [ -z "${LINUX_REPO}" ]; then
+ echo "Error: bpftool or linux repos are not specified"
usage
fi
if [ -z "${BPF_BRANCH}" ]; then
echo "Error: linux's bpf tree branch is not specified"
usage
fi
+
+BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${BPFTOOL_REPO}/CHECKPOINT-COMMIT)}
+BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${BPFTOOL_REPO}/BPF-CHECKPOINT-COMMIT)}
+
if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then
echo "Error: bpf or bpf-next baseline commits are not provided"
usage
@@ -36,30 +40,49 @@
trap "cd ${WORKDIR}; exit" INT TERM EXIT
+BPFTOOL_SRC_DIR="tools/bpf/bpftool"
+
declare -A PATH_MAP
PATH_MAP=( \
- [tools/lib/bpf]=src \
+ [${BPFTOOL_SRC_DIR}]=src \
+ [${BPFTOOL_SRC_DIR}/bash-completion]=bash-completion \
+ [${BPFTOOL_SRC_DIR}/Documentation]=docs \
+ [kernel/bpf/disasm.c]=src/kernel/bpf/disasm.c \
+ [kernel/bpf/disasm.h]=src/kernel/bpf/disasm.h \
+ [tools/include/uapi/asm-generic/bitsperlong.h]=include/uapi/asm-generic/bitsperlong.h \
[tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \
[tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \
[tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \
- [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \
- [tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h \
- [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \
+ [tools/include/uapi/linux/const.h]=include/uapi/linux/const.h \
+ [tools/include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \
[tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \
[tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \
- [Documentation/bpf/libbpf]=docs \
+ [tools/include/uapi/linux/tc_act/tc_bpf.h]=include/uapi/linux/tc_act/tc_bpf.h \
)
-LIBBPF_PATHS="${!PATH_MAP[@]} :^tools/lib/bpf/Makefile :^tools/lib/bpf/Build :^tools/lib/bpf/.gitignore :^tools/include/tools/libc_compat.h"
-LIBBPF_VIEW_PATHS="${PATH_MAP[@]}"
-LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*'
-LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$'
+BPFTOOL_PATHS="${!PATH_MAP[@]}"
+BPFTOOL_VIEW_PATHS="${PATH_MAP[@]}"
+BPFTOOL_VIEW_EXCLUDE_REGEX='^src/Makefile\.(feature|include)$'
+LINUX_VIEW_EXCLUDE_REGEX='^$'
-LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n'
+# Deal with tools/bpf/bpftool first, because once we've mkdir-ed src/, command
+# "git mv" doesn't move bpftool _as_ src but _into_ src/.
+BPFTOOL_TREE_FILTER="mkdir __bpftool && "$'\\\n'
+BPFTOOL_TREE_FILTER+="git mv -kf ${BPFTOOL_SRC_DIR} __bpftool/${PATH_MAP[${BPFTOOL_SRC_DIR}]} && "$'\\\n'
+
+# Extract bash-completion and Documentation from src/.
+BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/bash-completion __bpftool/bash-completion && "$'\\\n'
+BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/Documentation __bpftool/docs && "$'\\\n'
+
+BPFTOOL_TREE_FILTER+="mkdir -p __bpftool/include/uapi/asm-generic __bpftool/include/uapi/linux/tc_act __bpftool/src/kernel/bpf && "$'\\\n'
for p in "${!PATH_MAP[@]}"; do
- LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n'
+ case ${p} in
+ ${BPFTOOL_SRC_DIR}*)
+ continue;;
+ esac
+ BPFTOOL_TREE_FILTER+="git mv -kf ${p} __bpftool/${PATH_MAP[${p}]} && "$'\\\n'
done
-LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null"
+BPFTOOL_TREE_FILTER+="true >/dev/null"
cd_to()
{
@@ -87,7 +110,7 @@
git show --pretty='("%s")|%aI|%b' --shortstat $1 -- ${2-.} | tr '\n' '|'
}
-# Cherry-pick commits touching libbpf-related files
+# Cherry-pick commits touching bpftool-related files
# $1 - baseline_tag
# $2 - tip_tag
cherry_pick_commits()
@@ -100,20 +123,26 @@
local should_skip
local synced_cnt
local manual_check
- local libbpf_conflict_cnt
+ local bpftool_conflict_cnt
local desc
- new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} ${LIBBPF_PATHS[@]})
+ new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} ${BPFTOOL_PATHS[@]})
for new_commit in ${new_commits}; do
+ if [[ "${baseline_tag}" == "${BPF_BASELINE_TAG}" ]]; then
+ if git merge-base --is-ancestor "${new_commit}" "${BASELINE_COMMIT}"; then
+ echo "Commit ${new_commit::12} from bpf is already in bpf-next branch, skipping."
+ continue
+ fi
+ fi
desc="$(commit_desc ${new_commit})"
- signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")"
- synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l)
+ signature="$(commit_signature ${new_commit} "${BPFTOOL_PATHS[@]}")"
+ synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/bpftool_commits.txt | wc -l)
manual_check=0
if ((${synced_cnt} > 0)); then
- # commit with the same subject is already in libbpf, but it's
+ # commit with the same subject is already in bpftool, but it's
# not 100% the same commit, so check with user
- echo "Commit '${desc}' is synced into libbpf as:"
- grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \
+ echo "Commit '${desc}' is synced into bpftool as:"
+ grep -F "${signature}" ${TMP_DIR}/bpftool_commits.txt | \
cut -d'|' -f1 | sed -e 's/^/- /'
if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then
echo "Skipping '${desc}' due to unique match..."
@@ -133,16 +162,16 @@
;;
esac
fi
- # commit hasn't been synced into libbpf yet
+ # commit hasn't been synced into bpftool yet
echo "Picking '${desc}'..."
if ! git cherry-pick ${new_commit} &>/dev/null; then
- echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..."
- libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- ${LIBBPF_PATHS[@]} | wc -l)
+ echo "Warning! Cherry-picking '${desc} failed, checking if it's non-bpftool files causing problems..."
+ bpftool_conflict_cnt=$(git diff --name-only --diff-filter=U -- ${BPFTOOL_PATHS[@]} | wc -l)
conflict_cnt=$(git diff --name-only | wc -l)
prompt_resolution=1
- if ((${libbpf_conflict_cnt} == 0)); then
- echo "Looks like only non-libbpf files have conflicts, ignoring..."
+ if ((${bpftool_conflict_cnt} == 0)); then
+ echo "Looks like only non-bpftool files have conflicts, ignoring..."
if ((${conflict_cnt} == 0)); then
echo "Empty cherry-pick, skipping it..."
git cherry-pick --abort
@@ -166,9 +195,9 @@
# Append signature of just cherry-picked commit to avoid
# potentially cherry-picking the same commit twice later when
# processing bpf tree commits. At this point we don't know yet
- # the final commit sha in libbpf repo, so we record Linux SHA
+ # the final commit sha in bpftool repo, so we record Linux SHA
# instead as LINUX_<sha>.
- echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt
+ echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/bpftool_commits.txt
done
}
@@ -186,34 +215,34 @@
}
-cd_to ${LIBBPF_REPO}
+cd_to ${BPFTOOL_REPO}
GITHUB_ABS_DIR=$(pwd)
-echo "Dumping existing libbpf commit signatures..."
+echo "Dumping existing bpftool commit signatures..."
for h in $(git log --pretty='%h' -n500); do
- echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt
+ echo $h "$(commit_signature $h)" >> ${TMP_DIR}/bpftool_commits.txt
done
# Use current kernel repo HEAD as a source of patches
cd_to ${LINUX_REPO}
LINUX_ABS_DIR=$(pwd)
TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD)
-TIP_COMMIT=$(git rev-parse HEAD)
-BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH})
-BASELINE_TAG=libbpf-baseline-${SUFFIX}
-TIP_TAG=libbpf-tip-${SUFFIX}
-BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX}
-BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX}
-VIEW_TAG=libbpf-view-${SUFFIX}
-LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX}
+TIP_COMMIT=${BPF_NEXT_TIP_COMMIT:-$(git rev-parse HEAD)}
+BPF_TIP_COMMIT=${BPF_TIP_COMMIT:-$(git rev-parse ${BPF_BRANCH})}
+BASELINE_TAG=bpftool-baseline-${SUFFIX}
+TIP_TAG=bpftool-tip-${SUFFIX}
+BPF_BASELINE_TAG=bpftool-bpf-baseline-${SUFFIX}
+BPF_TIP_TAG=bpftool-bpf-tip-${SUFFIX}
+VIEW_TAG=bpftool-view-${SUFFIX}
+BPFTOOL_SYNC_TAG=bpftool-sync-${SUFFIX}
# Squash state of kernel repo at baseline into single commit
-SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX}
-SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX}
+SQUASH_BASE_TAG=bpftool-squash-base-${SUFFIX}
+SQUASH_TIP_TAG=bpftool-squash-tip-${SUFFIX}
SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}")
echo "WORKDIR: ${WORKDIR}"
echo "LINUX REPO: ${LINUX_REPO}"
-echo "LIBBPF REPO: ${LIBBPF_REPO}"
+echo "BPFTOOL REPO: ${BPFTOOL_REPO}"
echo "TEMP DIR: ${TMP_DIR}"
echo "SUFFIX: ${SUFFIX}"
echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'"
@@ -228,7 +257,7 @@
echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}"
echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}"
echo "VIEW TAG: ${VIEW_TAG}"
-echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}"
+echo "BPFTOOL SYNC TAG: ${BPFTOOL_SYNC_TAG}"
echo "PATCHES: ${TMP_DIR}/patches"
git branch ${BASELINE_TAG} ${BASELINE_COMMIT}
@@ -239,15 +268,17 @@
git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT}
# Cherry-pick new commits onto squashed baseline commit
+echo "Cherry-pick for bpf-next..."
cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG}
+echo "Cherry-pick for bpf..."
cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG}
-# Move all libbpf files into __libbpf directory.
-FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
-# Make __libbpf a new root directory
-FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
+# Move all bpftool files into __bpftool directory.
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${BPFTOOL_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
+# Make __bpftool a new root directory
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __bpftool ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
-# If there are no new commits with libbpf-related changes, bail out
+# If there are no new commits with bpftool-related changes, bail out
COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG})
if ((${COMMIT_CNT} <= 0)); then
echo "No new changes to apply, we are done!"
@@ -258,9 +289,9 @@
# Exclude baseline commit and generate nice cover letter with summary
git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches
-# Now is time to re-apply libbpf-related linux patches to libbpf repo
-cd_to ${LIBBPF_REPO}
-git checkout -b ${LIBBPF_SYNC_TAG}
+# Now is time to re-apply bpftool-related linux patches to bpftool repo
+cd_to ${BPFTOOL_REPO}
+git checkout -b ${BPFTOOL_SYNC_TAG}
for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do
if ! git am --3way --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then
@@ -268,24 +299,6 @@
fi
done
-# Generate bpf_helper_defs.h and commit, if anything changed
-# restore Linux tip to use bpf_doc.py
-cd_to ${LINUX_REPO}
-git checkout ${TIP_TAG}
-# re-generate bpf_helper_defs.h
-cd_to ${LIBBPF_REPO}
-"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header \
- --file include/uapi/linux/bpf.h > src/bpf_helper_defs.h
-# if anything changed, commit it
-helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l)
-if ((${helpers_changes} == 1)); then
- git add src/bpf_helper_defs.h
- git commit -m "sync: auto-generate latest BPF helpers
-
-Latest changes to BPF helper definitions.
-" -- src/bpf_helper_defs.h
-fi
-
# Use generated cover-letter as a template for "sync commit" with
# baseline and checkpoint commits from kernel repo (and leave summary
# from cover letter intact, of course)
@@ -295,9 +308,9 @@
git add BPF-CHECKPOINT-COMMIT && \
awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \
sed "s/\*\*\* BLURB HERE \*\*\*/\
-sync: latest libbpf changes from kernel\n\
+sync: Pull latest bpftool changes from kernel\n\
\n\
-Syncing latest libbpf commits from kernel repository.\n\
+Syncing latest bpftool commits from kernel repository.\n\
Baseline bpf-next commit: ${BASELINE_COMMIT}\n\
Checkpoint bpf-next commit: ${TIP_COMMIT}\n\
Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\
@@ -306,23 +319,23 @@
echo "SUCCESS! ${COMMIT_CNT} commits synced."
-echo "Verifying Linux's and Github's libbpf state"
+echo "Verifying Linux's and Github's bpftool state"
cd_to ${LINUX_REPO}
git checkout -b ${VIEW_TAG} ${TIP_COMMIT}
-FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG}
-FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG}
-git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${BPFTOOL_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG}
+FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __bpftool ${VIEW_TAG}^..${VIEW_TAG}
+git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls
-cd_to ${LIBBPF_REPO}
-git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls
+cd_to ${BPFTOOL_REPO}
+git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${BPFTOOL_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls
echo "Comparing list of files..."
diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls
echo "Comparing file contents..."
CONSISTENT=1
for F in $(cat ${TMP_DIR}/linux-view.ls); do
- if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then
+ if ! diff -u --color "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then
echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!"
CONSISTENT=0
fi
