1#!/usr/bin/env bash 2 3usage () { 4 echo "USAGE: ./sync-kernel.sh <bpftool-repo> <kernel-repo>" 5 echo "" 6 echo "This script synchronizes the mirror with upstream bpftool sources from the kernel repository." 7 echo "It performs the following steps:" 8 echo " - Update the libbpf submodule, commit, and use its new checkpoints as target commits for bpftool." 9 echo " - Cherry-pick commits from the bpf-next branch, up to the bpf-next target commit." 10 echo " - Cherry-pick commits from the bpf branch, up to the bpf target commit." 11 echo " - Create a new commit with the updated version and checkpoints." 12 echo " - Check consistency." 13 echo "" 14 echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <bpftool-repo>/CHECKPOINT-COMMIT." 15 echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <bpftool-repo>/BPF-CHECKPOINT-COMMIT." 16 echo "Set BPF_NEXT_TIP_COMMIT to override bpf-next tree target commit, otherwise read from <bpftool-repo>/libbpf/CHECKPOINT-COMMIT, after libbpf update." 17 echo "Set BPF_TIP_COMMIT to override bpf tree target commit, otherwise read from <bpftool-repo>/libbpf/BPF-CHECKPOINT-COMMIT, after libbpf update." 18 echo "Set SKIP_LIBBPF_UPDATE to 1 to avoid updating libbpf automatically." 19 echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commit." 20 exit 1 21} 22 23set -eu 24 25BPFTOOL_REPO=${1-""} 26LINUX_REPO=${2-""} 27 28if [ -z "${BPFTOOL_REPO}" ] || [ -z "${LINUX_REPO}" ]; then 29 echo "Error: bpftool or linux repos are not specified" 30 usage 31fi 32 33BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat "${BPFTOOL_REPO}"/CHECKPOINT-COMMIT)} 34BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat "${BPFTOOL_REPO}"/BPF-CHECKPOINT-COMMIT)} 35 36if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then 37 echo "Error: bpf or bpf-next baseline commits are not provided" 38 usage 39fi 40 41SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) 42WORKDIR=$(pwd) 43TMP_DIR=$(mktemp -d) 44 45# shellcheck disable=SC2064 46trap "cd ${WORKDIR}; exit" INT TERM EXIT 47 48BPFTOOL_SRC_DIR="tools/bpf/bpftool" 49 50declare -A PATH_MAP 51PATH_MAP=( \ 52 [${BPFTOOL_SRC_DIR}]=src \ 53 [${BPFTOOL_SRC_DIR}/bash-completion]=bash-completion \ 54 [${BPFTOOL_SRC_DIR}/Documentation]=docs \ 55 [kernel/bpf/disasm.c]=src/kernel/bpf/disasm.c \ 56 [kernel/bpf/disasm.h]=src/kernel/bpf/disasm.h \ 57 [tools/include/tools/dis-asm-compat.h]=include/tools/dis-asm-compat.h \ 58 [tools/include/uapi/asm-generic/bitsperlong.h]=include/uapi/asm-generic/bitsperlong.h \ 59 [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ 60 [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ 61 [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ 62 [tools/include/uapi/linux/const.h]=include/uapi/linux/const.h \ 63 [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ 64 [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ 65 [tools/include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ 66 [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ 67 [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ 68 [tools/include/uapi/linux/tc_act/tc_bpf.h]=include/uapi/linux/tc_act/tc_bpf.h \ 69) 70 71BPFTOOL_PATHS=( "${!PATH_MAP[@]}" ) 72BPFTOOL_VIEW_PATHS=( "${PATH_MAP[@]}" ) 73BPFTOOL_VIEW_EXCLUDE_REGEX='^(docs/\.gitignore|src/Makefile\.(feature|include))$' 74LINUX_VIEW_EXCLUDE_REGEX='^$' 75 76# Deal with tools/bpf/bpftool first, because once we've mkdir-ed src/, command 77# "git mv" doesn't move bpftool _as_ src but _into_ src/. 78BPFTOOL_TREE_FILTER="mkdir __bpftool && "$'\\\n' 79BPFTOOL_TREE_FILTER+="git mv -kf ${BPFTOOL_SRC_DIR} __bpftool/${PATH_MAP[${BPFTOOL_SRC_DIR}]} && "$'\\\n' 80 81# Extract bash-completion and Documentation from src/. 82BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/bash-completion __bpftool/bash-completion && "$'\\\n' 83BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/Documentation __bpftool/docs && "$'\\\n' 84 85BPFTOOL_TREE_FILTER+="mkdir -p __bpftool/include/tools __bpftool/include/uapi/asm-generic __bpftool/include/uapi/linux/tc_act __bpftool/src/kernel/bpf && "$'\\\n' 86for p in "${!PATH_MAP[@]}"; do 87 case ${p} in 88 ${BPFTOOL_SRC_DIR}*) 89 continue;; 90 esac 91 BPFTOOL_TREE_FILTER+="git mv -kf ${p} __bpftool/${PATH_MAP[${p}]} && "$'\\\n' 92done 93BPFTOOL_TREE_FILTER+="true >/dev/null" 94 95cd_to() 96{ 97 cd "${WORKDIR}" && cd "$1" 98} 99 100# Output brief single-line commit description 101# $1 - commit ref 102commit_desc() 103{ 104 git log -n1 --pretty='%h ("%s")' "$1" 105} 106 107# Create commit single-line signature, which consists of: 108# - full commit subject 109# - author date in ISO8601 format 110# - full commit body with newlines replaced with vertical bars (|) 111# - shortstat appended at the end 112# The idea is that this single-line signature is good enough to make final 113# decision about whether two commits are the same, across different repos. 114# $1 - commit ref 115# $2 - paths filter 116commit_signature() 117{ 118 # shellcheck disable=SC2086 119 git show --pretty='("%s")|%aI|%b' --shortstat "$1" -- ${2-.} | tr '\n' '|' 120} 121 122# Cherry-pick commits touching bpftool-related files 123# $1 - baseline_tag 124# $2 - tip_tag 125cherry_pick_commits() 126{ 127 local manual_mode=${MANUAL_MODE:-0} 128 local baseline_tag=$1 129 local tip_tag=$2 130 local new_commits 131 local signature 132 local should_skip 133 local synced_cnt 134 local manual_check 135 local bpftool_conflict_cnt 136 local desc 137 138 # shellcheck disable=SC2068 139 new_commits=$(git rev-list --no-merges --topo-order --reverse "${baseline_tag}".."${tip_tag}" ${BPFTOOL_PATHS[@]}) 140 for new_commit in ${new_commits}; do 141 if [[ "${baseline_tag}" == "${BPF_BASELINE_TAG}" ]]; then 142 if git merge-base --is-ancestor "${new_commit}" "${BASELINE_COMMIT}"; then 143 echo "Commit ${new_commit::12} from bpf is already in bpf-next branch, skipping." 144 continue 145 fi 146 fi 147 desc="$(commit_desc "${new_commit}")" 148 signature="$(commit_signature "${new_commit}" "${BPFTOOL_PATHS[*]}")" 149 # shellcheck disable=SC2126 150 synced_cnt=$(grep -F "${signature}" "${TMP_DIR}"/bpftool_commits.txt | wc -l) 151 manual_check=0 152 if (("${synced_cnt}" > 0)); then 153 # commit with the same subject is already in bpftool, but it's 154 # not 100% the same commit, so check with user 155 echo "Commit '${desc}' is synced into bpftool as:" 156 grep -F "${signature}" "${TMP_DIR}"/bpftool_commits.txt | \ 157 cut -d'|' -f1 | sed -e 's/^/- /' 158 if (("${manual_mode}" != 1 && "${synced_cnt}" == 1)); then 159 echo "Skipping '${desc}' due to unique match..." 160 continue 161 fi 162 if (("${synced_cnt}" > 1)); then 163 echo "'${desc} matches multiple commits, please, double-check!" 164 manual_check=1 165 fi 166 fi 167 if (("${manual_mode}" == 1 || "${manual_check}" == 1)); then 168 read -rp "Do you want to skip '${desc}'? [y/N]: " should_skip 169 case "${should_skip}" in 170 "y" | "Y") 171 echo "Skipping '${desc}'..." 172 continue 173 ;; 174 esac 175 fi 176 # commit hasn't been synced into bpftool yet 177 echo "Picking '${desc}'..." 178 if ! git cherry-pick "${new_commit}" &>/dev/null; then 179 echo "Warning! Cherry-picking '${desc} failed, checking if it's non-bpftool files causing problems..." 180 # shellcheck disable=SC2068 181 bpftool_conflict_cnt=$(git diff --name-only --diff-filter=U -- ${BPFTOOL_PATHS[@]} | wc -l) 182 conflict_cnt=$(git diff --name-only | wc -l) 183 prompt_resolution=1 184 185 if (("${bpftool_conflict_cnt}" == 0)); then 186 echo "Looks like only non-bpftool files have conflicts, ignoring..." 187 if (("${conflict_cnt}" == 0)); then 188 echo "Empty cherry-pick, skipping it..." 189 git cherry-pick --abort 190 continue 191 fi 192 193 git add . 194 # GIT_EDITOR=true to avoid editor popping up to edit commit message 195 if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then 196 echo "Error! That still failed! Please resolve manually." 197 else 198 echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" 199 prompt_resolution=0 200 fi 201 fi 202 203 if (("${prompt_resolution}" == 1)); then 204 read -rp "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..." 205 fi 206 fi 207 # Append signature of just cherry-picked commit to avoid 208 # potentially cherry-picking the same commit twice later when 209 # processing bpf tree commits. At this point we don't know yet 210 # the final commit sha in bpftool repo, so we record Linux SHA 211 # instead as LINUX_<sha>. 212 echo "LINUX_$(git log --pretty='%h' -n1) ${signature}" >> "${TMP_DIR}"/bpftool_commits.txt 213 done 214} 215 216cleanup() 217{ 218 echo "Cleaning up..." 219 rm -r -- "${TMP_DIR}" 220 cd_to "${LINUX_REPO}" 221 git checkout "${TIP_SYM_REF}" 222 git branch -D "${BASELINE_TAG}" "${TIP_TAG}" "${BPF_BASELINE_TAG}" "${BPF_TIP_TAG}" \ 223 "${SQUASH_BASE_TAG}" "${SQUASH_TIP_TAG}" || true 224 # shellcheck disable=SC2015 225 git show-ref --verify --quiet refs/heads/"${VIEW_TAG}" && \ 226 git branch -D "${VIEW_TAG}" || true 227 228 cd_to . 229 echo "DONE." 230} 231 232cd_to "${BPFTOOL_REPO}" 233BPFTOOL_SYNC_TAG="bpftool-sync-${SUFFIX}" 234git checkout -b "${BPFTOOL_SYNC_TAG}" 235 236# Update libbpf 237if [[ "${SKIP_LIBBPF_UPDATE:-0}" -ne 1 ]]; then 238 cd_to "${BPFTOOL_REPO}"/libbpf 239 git pull origin master 240 LIBBPF_VERSION=$(grep -oE '^LIBBPF_([0-9.]+)' src/libbpf.map | sort -rV | head -n1 | cut -d'_' -f2) 241 LIBBPF_COMMIT=$(git rev-parse HEAD) 242 cd_to "${BPFTOOL_REPO}" 243 if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then 244 git add libbpf 245 git commit --signoff -m 'sync: Update libbpf submodule' \ 246 -m "\ 247Pull latest libbpf from mirror. 248Libbpf version: ${LIBBPF_VERSION} 249Libbpf commit: ${LIBBPF_COMMIT}" \ 250 -- libbpf 251 fi 252fi 253 254# Use libbpf's new checkpoints as tips 255TIP_COMMIT=${BPF_NEXT_TIP_COMMIT:-$(cat "${BPFTOOL_REPO}"/libbpf/CHECKPOINT-COMMIT)} 256BPF_TIP_COMMIT=${BPF_TIP_COMMIT:-$(cat "${BPFTOOL_REPO}"/libbpf/BPF-CHECKPOINT-COMMIT)} 257if [ -z "${TIP_COMMIT}" ] || [ -z "${BPF_TIP_COMMIT}" ]; then 258 echo "Error: bpf or bpf-next tip commits are not provided" 259 usage 260fi 261 262cd_to "${BPFTOOL_REPO}" 263GITHUB_ABS_DIR=$(pwd) 264echo "Dumping existing bpftool commit signatures..." 265for h in $(git log --pretty='%h' -n500); do 266 echo "$h" "$(commit_signature "$h")" >> "${TMP_DIR}"/bpftool_commits.txt 267done 268 269# Use current kernel repo HEAD as a source of patches 270cd_to "${LINUX_REPO}" 271LINUX_ABS_DIR=$(pwd) 272TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) 273BASELINE_TAG="bpftool-baseline-${SUFFIX}" 274TIP_TAG="bpftool-tip-${SUFFIX}" 275BPF_BASELINE_TAG="bpftool-bpf-baseline-${SUFFIX}" 276BPF_TIP_TAG="bpftool-bpf-tip-${SUFFIX}" 277VIEW_TAG="bpftool-view-${SUFFIX}" 278 279# Squash state of kernel repo at baseline into single commit 280SQUASH_BASE_TAG="bpftool-squash-base-${SUFFIX}" 281SQUASH_TIP_TAG="bpftool-squash-tip-${SUFFIX}" 282SQUASH_COMMIT=$(git commit-tree "${BASELINE_COMMIT}^{tree}" -m "BASELINE SQUASH ${BASELINE_COMMIT}") 283 284echo "WORKDIR: ${WORKDIR}" 285echo "LINUX REPO: ${LINUX_REPO}" 286echo "BPFTOOL REPO: ${BPFTOOL_REPO}" 287echo "TEMP DIR: ${TMP_DIR}" 288echo "SUFFIX: ${SUFFIX}" 289echo "BASE COMMIT: '$(commit_desc "${BASELINE_COMMIT}")'" 290echo "TIP COMMIT: '$(commit_desc "${TIP_COMMIT}")'" 291echo "BPF BASE COMMIT: '$(commit_desc "${BPF_BASELINE_COMMIT}")'" 292echo "BPF TIP COMMIT: '$(commit_desc "${BPF_TIP_COMMIT}")'" 293echo "SQUASH COMMIT: ${SQUASH_COMMIT}" 294echo "BASELINE TAG: ${BASELINE_TAG}" 295echo "TIP TAG: ${TIP_TAG}" 296echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" 297echo "BPF TIP TAG: ${BPF_TIP_TAG}" 298echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" 299echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" 300echo "VIEW TAG: ${VIEW_TAG}" 301echo "BPFTOOL SYNC TAG: ${BPFTOOL_SYNC_TAG}" 302echo "PATCHES: ${TMP_DIR}/patches" 303 304git branch "${BASELINE_TAG}" "${BASELINE_COMMIT}" 305git branch "${TIP_TAG}" "${TIP_COMMIT}" 306git branch "${BPF_BASELINE_TAG}" "${BPF_BASELINE_COMMIT}" 307git branch "${BPF_TIP_TAG}" "${BPF_TIP_COMMIT}" 308git branch "${SQUASH_BASE_TAG}" "${SQUASH_COMMIT}" 309git checkout -b "${SQUASH_TIP_TAG}" "${SQUASH_COMMIT}" 310 311# Cherry-pick new commits onto squashed baseline commit 312echo "Cherry-pick for bpf-next..." 313cherry_pick_commits "${BASELINE_TAG}" "${TIP_TAG}" 314echo "Cherry-pick for bpf..." 315cherry_pick_commits "${BPF_BASELINE_TAG}" "${BPF_TIP_TAG}" 316 317# Move all bpftool files into __bpftool directory. 318FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${BPFTOOL_TREE_FILTER}" "${SQUASH_TIP_TAG}" "${SQUASH_BASE_TAG}" 319# Make __bpftool a new root directory 320FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __bpftool "${SQUASH_TIP_TAG}" "${SQUASH_BASE_TAG}" 321 322# If there are no new commits with bpftool-related changes, bail out 323COMMIT_CNT=$(git rev-list --count "${SQUASH_BASE_TAG}".."${SQUASH_TIP_TAG}") 324if (("${COMMIT_CNT}" <= 0)); then 325 echo "No new changes to apply, we are done!" 326 cleanup 327 exit 2 328fi 329 330# Exclude baseline commit and generate nice cover letter with summary 331git format-patch --no-signature "${SQUASH_BASE_TAG}".."${SQUASH_TIP_TAG}" --cover-letter -o "${TMP_DIR}"/patches 332 333# Now is time to re-apply bpftool-related linux patches to bpftool repo 334cd_to "${BPFTOOL_REPO}" 335 336# shellcheck disable=SC2012 337for patch in $(ls -1 "${TMP_DIR}"/patches | tail -n +2); do 338 if ! git am --3way --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then 339 read -rp "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..." 340 fi 341done 342 343# Use generated cover-letter as a template for "sync commit" with 344# baseline and checkpoint commits from kernel repo (and leave summary 345# from cover letter intact, of course) 346echo "${TIP_COMMIT}" > CHECKPOINT-COMMIT && \ 347echo "${BPF_TIP_COMMIT}" > BPF-CHECKPOINT-COMMIT && \ 348git add CHECKPOINT-COMMIT && \ 349git add BPF-CHECKPOINT-COMMIT && \ 350awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' "${TMP_DIR}"/patches/0000-cover-letter.patch | \ 351sed "s/\*\*\* BLURB HERE \*\*\*/\ 352sync: Pull latest bpftool changes from kernel\n\ 353\n\ 354Syncing latest bpftool commits from kernel repository.\n\ 355Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ 356Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ 357Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ 358Checkpoint bpf commit: ${BPF_TIP_COMMIT}/" | \ 359git commit --signoff --file=- 360 361echo "SUCCESS! ${COMMIT_CNT} commits synced." 362 363echo "Verifying Linux's and Github's bpftool state" 364 365cd_to "${LINUX_REPO}" 366git checkout -b "${VIEW_TAG}" "${TIP_COMMIT}" 367FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${BPFTOOL_TREE_FILTER}" "${VIEW_TAG}"^.."${VIEW_TAG}" 368FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __bpftool "${VIEW_TAG}"^.."${VIEW_TAG}" 369# shellcheck disable=SC2068 370git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > "${TMP_DIR}"/linux-view.ls 371# Before we compare each file, try to apply to the mirror a patch containing the 372# expected differences between the two repositories; this is to avoid checking 373# "known" differences visually and taking the risk of missing a new, relevant 374# differences. 375echo "Patching to account for expected differences..." 376patch -d "${LINUX_ABS_DIR}" -p0 -f --reject-file=- --no-backup-if-mismatch < "${GITHUB_ABS_DIR}/scripts/sync-kernel-expected-diff.patch" || true 377git add -u 378git commit -m 'tmp: apply expected differences to compare github/kernel repos' || true 379 380cd_to "${BPFTOOL_REPO}" 381# shellcheck disable=SC2068 382git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${BPFTOOL_VIEW_EXCLUDE_REGEX}" > "${TMP_DIR}"/github-view.ls 383 384echo "Comparing list of files..." 385diff -u "${TMP_DIR}"/linux-view.ls "${TMP_DIR}"/github-view.ls 386echo "Comparing file contents..." 387CONSISTENT=1 388while IFS= read -r F; do 389 if ! diff -u --color "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then 390 echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" 391 CONSISTENT=0 392 fi 393done < "${TMP_DIR}"/linux-view.ls 394echo "" 395if (("${CONSISTENT}" == 1)); then 396 echo "Great! Content is identical!" 397else 398 ignore_inconsistency=n 399 echo "Unfortunately, there are some inconsistencies, please double check." 400 echo "Some of them may come from patches in bpf tree but absent from bpf-next." 401 echo "Note: I applied scripts/sync-kernel-expected-diff.patch before checking," 402 echo "to account for expected changes. If this patch needs an update," 403 echo "you can do it now with:" 404 echo "------" 405 echo " (cd \"${LINUX_ABS_DIR}\" && git -c advice.detachedHead=false checkout HEAD~)" 406 echo " for f in \$(cat \"${TMP_DIR}/linux-view.ls\"); do" 407 echo " diff -u --label \"\${f}\" --label \"\${f}\" \\" 408 echo " \"${LINUX_ABS_DIR}/\${f}\" \\" 409 echo " \"${GITHUB_ABS_DIR}/\${f}\"" 410 echo " done > \"${GITHUB_ABS_DIR}/scripts/sync-kernel-expected-diff.patch\"" 411 echo "------" 412 read -rp "Does everything look good? [y/N]: " ignore_inconsistency 413 case "${ignore_inconsistency}" in 414 "y" | "Y") 415 echo "Ok, proceeding..." 416 ;; 417 *) 418 echo "Oops, exiting with error..." 419 exit 4 420 esac 421fi 422 423cleanup 424