XLibre/xserver
1
1
Fork 0
mirror of https://github.com/X11Libre/xserver.git synced 2026-03-24 05:54:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

.github: use a smarter cloning script

Browse Source 
Several sites we need to clone our dependencies from (eg. freedesktop.org)
have pretty unreliable servers, so our CI jobs often fail just because
of temporary clone failure.

Therefore adding a separate cloning script, which is more clever with automatic
retries, but it also tries to keep the traffic low (eg. trying shallow clones
if possible) and automatically detecting whether we're pulling a ref or a
direct commit.

Signed-off-by: Enrico Weigelt, metux IT consult <info@metux.net>
This commit is contained in:
Enrico Weigelt, metux IT consult
2025-08-26 14:14:44 +02:00
parent 91cd4e2115
commit 7a2103fc17
3 changed files with 280 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
.github/scripts/git-smart-checkout.sh vendored Executable file
View File

@@ -0,0 +1,267 @@
#!/usr/bin/env bash
# git-smart-checkout: clone/reuse a repo and force-checkout a given ref (branch/tag/commit),
# with mirror URLs, minimal transfer, and automatic retries.
# Robustness
set -euo pipefail
IFS=$'\n\t'
# ---- Logging helpers --------------------------------------------------------
log_info() { printf '[INFO] %s\n' "$*" >&2; }
log_warn() { printf '[WARN] %s\n' "$*" >&2; }
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
# ---- Usage ------------------------------------------------------------------
usage() {
cat >&2 <<'USAGE'
Usage:
git-smart-checkout.sh -n <local-dir> -r <ref|commit> -u <url> [-u <url2> ...] [options]
Required:
-n, --name <dir> local target directory (repo / path).
-r, --ref <ref> branch, tag or Commit-SHA.
-u, --url <url> repo URL (multiple for mirror fallbacks)
Options:
--retries <N> amount of retries (default: 3).
--sleep <SEC> wait seconds between retries (default: 3).
--no-clean no hard working dir cleanup after checkout
--full-if-needed if commit not reachable with shallow fetch, retry again without --depth
--keep-remote dont change remote config (otherwise set 'origin' to first URL).
--depth <N> shallow depth for fetch (default: 1). only useful for branch/tag
--no-partial disable partial clone/filter (otherwise load blobs on-demand).
--no-tags dont fetch tags by default (default).
--allow-tags fetch tags (overwrites --no-tags).
Examples:
git-smart-checkout.sh -n myrepo -r main -u https://github.com/org/proj.git -u https://gitmirror/org/proj.git
git-smart-checkout.sh -n myrepo -r v2.1.0 -u ssh://git@example.com/proj.git
git-smart-checkout.sh -n myrepo -r 1a2b3c4d5e --full-if-needed -u https://github.com/org/proj.git
USAGE
exit 2
}
# ---- Defaults ---------------------------------------------------------------
NAME=""
REF=""
URLS=()
RETRIES=3
SLEEP_SEC=3
CLEAN=1
FULL_IF_NEEDED=0
KEEP_REMOTE=0
DEPTH=1
USE_PARTIAL=1
FETCH_TAGS=0 # 0 => --no-tags, 1 => allow tags
# ---- Parse args -------------------------------------------------------------
if [[ $# -eq 0 ]]; then usage; fi
while (( "$#" )); do
case "$1" in
-n|--name) NAME="${2:-}"; shift 2 ;;
-r|--ref) REF="${2:-}"; shift 2 ;;
-u|--url) URLS+=("${2:-}"); shift 2 ;;
--retries) RETRIES="${2:-}"; shift 2 ;;
--sleep) SLEEP_SEC="${2:-}"; shift 2 ;;
--no-clean) CLEAN=0; shift ;;
--full-if-needed) FULL_IF_NEEDED=1; shift ;;
--keep-remote) KEEP_REMOTE=1; shift ;;
--depth) DEPTH="${2:-}"; shift 2 ;;
--no-partial) USE_PARTIAL=0; shift ;;
--no-tags) FETCH_TAGS=0; shift ;;
--allow-tags) FETCH_TAGS=1; shift ;;
-h|--help) usage ;;
*) log_error "Unknown option: $1"; usage ;;
esac
done
# ---- Validate args ----------------------------------------------------------
if [[ -z "$NAME" ]]; then log_error "Missing --name"; usage; fi
if [[ -z "$REF" ]]; then log_error "Missing --ref"; usage; fi
if [[ ${#URLS[@]} -eq 0 ]]; then log_error "At least one --url is required"; usage; fi
if ! command -v git >/dev/null 2>&1; then log_error "'git' not found"; exit 127; fi
# ---- Helpers ----------------------------------------------------------------
is_git_dir() {
[[ -d "$1/.git" ]] || git -C "$1" rev-parse --git-dir >/dev/null 2>&1
}
is_sha_like() {
[[ "$1" =~ ^[0-9a-fA-F]{7,40}$ ]]
}
sleep_backoff() {
local attempt="$1"
local base="$SLEEP_SEC"
local delay=$(( base * attempt ))
[[ $delay -lt 1 ]] && delay=1
sleep "$delay"
}
git_retry() {
# git_retry <desc> <cmd...>
local desc="$1"; shift
local attempt=1
local rc=0
while :; do
if "$@"; then
return 0
fi
rc=$?
if (( attempt >= RETRIES )); then
log_error "$desc: failed after $RETRIES tries (rc=$rc)"
return "$rc"
fi
log_warn "$desc: try $attempt failed (rc=$rc). retrying ..."
sleep_backoff "$attempt"
attempt=$(( attempt + 1 ))
done
}
# Compose common fetch flags
fetch_flags_common=()
(( USE_PARTIAL )) && fetch_flags_common+=( "--filter=blob:none" )
(( FETCH_TAGS == 0 )) && fetch_flags_common+=( "--no-tags" )
fetch_flags_common+=( "--prune" "--prune-tags" "--update-head-ok" "--force" )
# With or without depth
fetch_flags_shallow=( "${fetch_flags_common[@]}" "--depth=$DEPTH" )
fetch_flags_full=( "${fetch_flags_common[@]}" )
# ---- Clone or reuse existing ------------------------------------------------
if [[ -d "$NAME" ]]; then
if is_git_dir "$NAME"; then
log_info "detected existing git repository: $NAME"
if (( KEEP_REMOTE == 0 )); then
log_info "setting 'origin' to first URL: ${URLS[0]}"
if git -C "$NAME" remote get-url origin >/dev/null 2>&1; then
git -C "$NAME" remote set-url origin "${URLS[0]}"
else
git -C "$NAME" remote add origin "${URLS[0]}"
fi
else
log_info "--keep-remote: remote config unchanged."
fi
else
log_error "directory '$NAME' exists, but not a git repository."
exit 1
fi
else
# fresh clone: minimal & no-checkout
mkdir -p "$(dirname -- "$NAME")"
clone_ok=0
for u in "${URLS[@]}"; do
log_info "Clone von $u -> $NAME (minimaler Transfer, no-checkout)"
clone_cmd=( git clone "--no-checkout" "--origin=origin" "$u" "$NAME" )
(( USE_PARTIAL )) && clone_cmd+=( "--filter=blob:none" )
(( FETCH_TAGS == 0 )) && clone_cmd+=( "--no-tags" )
if git_retry "clone $u" "${clone_cmd[@]}"; then
clone_ok=1
break
fi
done
if (( clone_ok == 0 )); then
log_error "clone failed on all URLs."
exit 1
fi
fi
# ---- Prepare partial clone config (for init/clone without --filter at fetch time) ----
if (( USE_PARTIAL )); then
# Best effort: ensure repo knows it's a partial clone (helps späteres lazy-fetch)
git -C "$NAME" config --local remote.origin.promisor true || true
git -C "$NAME" config --local remote.origin.partialclonefilter "blob:none" || true
git -C "$NAME" config --local extensions.partialClone "origin" || true
fi
# ---- Fetch target ref with minimal transfer ---------------------------------
# We'll try URLs in order; for each, attempt specific refspecs.
# Strategy:
# - If SHA-like: try shallow fetch of the object; fallback to full if enabled.
# - Else (name): try heads/<REF>, tags/<REF>, and plain <REF> shallow; fallback to full if needed.
enter_repo() { cd "$NAME"; } # for readability
enter_repo
fetch_from_url_for_ref() {
# fetch_from_url_for_ref <url> <ref> <shallow_or_full>
local url="$1" ref="$2" mode="$3" ; shift 3
local -a flags=()
if [[ "$mode" == "shallow" ]]; then
flags=( "${fetch_flags_shallow[@]}" )
else
flags=( "${fetch_flags_full[@]}" )
fi
if is_sha_like "$ref"; then
# Try fetching the specific commit object
# FETCH_HEAD will point to the fetched object on success.
git fetch "${flags[@]}" "$url" "$ref"
return $?
else
# Try specific namespaces to avoid extra refs
local try=(
"refs/heads/$ref:refs/tmp/$ref"
"refs/tags/$ref:refs/tmp/$ref"
"$ref:refs/tmp/$ref"
)
local t
for t in "${try[@]}"; do
if git fetch "${flags[@]}" "$url" "$t"; then
return 0
fi
done
return 1
fi
}
fetch_ok=0
for u in "${URLS[@]}"; do
# 1) Try shallow
if git_retry "fetch url: $u $REF" fetch_from_url_for_ref "$u" "$REF" "shallow"; then
fetch_ok=1
break
fi
# 2) Optional fallback to full (still filtered blobs)
if (( FULL_IF_NEEDED )); then
log_warn "shallow fetch insufficient. trying without --depth from $u ..."
if git_retry "fetch url: $u $REF" fetch_from_url_for_ref "$u" "$REF" "full"; then
fetch_ok=1
break
fi
fi
done
if (( fetch_ok == 0 )); then
log_error "failed fetching reference '$REF' on all URLs."
exit 1
fi
# Resolve fetched commit
TARGET_COMMIT="$(git rev-parse --verify --quiet FETCH_HEAD || true)"
if [[ -z "$TARGET_COMMIT" ]]; then
log_error "couldnt resolve FETCH_HEAD."
exit 1
fi
log_info "fetched: $TARGET_COMMIT"
# ---- Forced checkout --------------------------------------------------------
# We checkout detached at the exact commit to be fully deterministic.
# (No implicit branch tracking; avoids surprises if remote refs move.)
if (( CLEAN )); then
log_info "cleaning working dir (reset --hard, clean -fdx)"
git reset --hard
git clean -fdx
fi
log_info "forced checkout (detached) auf $TARGET_COMMIT"
# Ensure no stray temp ref prevents checkout
git update-ref -d "refs/tmp/$REF" >/dev/null 2>&1 || true
git checkout --force --detach "$TARGET_COMMIT"
# Small summary
log_info "working dir:"
git --no-pager log -1 --oneline
git --no-pager status --short --branch
log_info "Done."

7
.github/scripts/install-prereq.sh vendored
View File

@@ -9,17 +9,16 @@ cd $X11_BUILD_DIR
build_meson rendercheck https://gitlab.freedesktop.org/xorg/test/rendercheck rendercheck-1.6
if [ "$X11_OS" = "Linux" ]; then
build_meson drm https://gitlab.freedesktop.org/mesa/drm libdrm-2.4.121 "" \
-Domap=enabled
build_meson drm https://gitlab.freedesktop.org/mesa/drm libdrm-2.4.121 -Domap=enabled
fi
build_meson libxcvt https://gitlab.freedesktop.org/xorg/lib/libxcvt libxcvt-0.1.0
build_ac xorgproto https://gitlab.freedesktop.org/xorg/proto/xorgproto xorgproto-2024.1
if [ "$X11_OS" = "Darwin" ]; then
build_ac xset https://gitlab.freedesktop.org/xorg/app/xset xset-1.2.5
fi
build_ac_xts xts https://gitlab.freedesktop.org/xorg/test/xts master 12a887c2c72c4258962b56ced7b0aec782f1ffed
build_ac_xts xts https://gitlab.freedesktop.org/xorg/test/xts 12a887c2c72c4258962b56ced7b0aec782f1ffed
clone_source piglit https://gitlab.freedesktop.org/mesa/piglit main 265896c86f90cb72e8f218ba6a3617fca8b9a1e3
clone_source piglit https://gitlab.freedesktop.org/mesa/piglit 265896c86f90cb72e8f218ba6a3617fca8b9a1e3
echo '[xts]' > piglit/piglit.conf
echo "path=$X11_BUILD_DIR/xts" >> piglit/piglit.conf

37
.github/scripts/util.sh vendored
View File

@@ -1,41 +1,30 @@
. .github/scripts/conf.sh
SOURCE_DIR=`pwd`
clone_source() {
local pkgname="$1"
local url="$2"
local ref="$3"
local commit="$4"
if [ ! -f $pkgname/.git/config ]; then
echo "need to clone $pkgname"
if [ "$commit" ]; then
git clone $url $pkgname --branch=$ref
else
git clone $url $pkgname --branch=$ref --depth 1
fi
else
echo "already cloned $pkgname"
fi
if [ "$commit" ]; then
( cd $pkgname && git checkout -f "$commit" )
fi
$SOURCE_DIR/.github/scripts/git-smart-checkout.sh \
--name "$pkgname" \
--url "$url" \
--ref "$ref"
}
build_meson() {
local pkgname="$1"
local url="$2"
local ref="$3"
local commit="$4"
shift
shift
shift
shift || true
if [ -f $X11_PREFIX/$pkgname.DONE ]; then
echo "package $pkgname already built"
else
clone_source "$pkgname" "$url" "$ref" "$commit"
clone_source "$pkgname" "$url" "$ref"
(
cd $pkgname
meson "$@" build -Dprefix=$X11_PREFIX
@@ -49,15 +38,13 @@ build_ac() {
local pkgname="$1"
local url="$2"
local ref="$3"
local commit="$4"
shift
shift
shift
shift || true
if [ -f $X11_PREFIX/$pkgname.DONE ]; then
echo "package $pkgname already built"
else
clone_source "$pkgname" "$url" "$ref" "$commit"
clone_source "$pkgname" "$url" "$ref"
(
cd $pkgname
./autogen.sh --prefix=$X11_PREFIX
@@ -71,12 +58,10 @@ build_drv_ac() {
local pkgname="$1"
local url="$2"
local ref="$3"
local commit="$4"
shift
shift
shift
shift || true
clone_source "$pkgname" "$url" "$ref" "$commit"
clone_source "$pkgname" "$url" "$ref"
(
cd $pkgname
./autogen.sh # --prefix=$X11_PREFIX
@@ -88,15 +73,13 @@ build_ac_xts() {
local pkgname="$1"
local url="$2"
local ref="$3"
local commit="$4"
shift
shift
shift
shift || true
if [ -f $X11_PREFIX/$pkgname.DONE ]; then
echo "package $pkgname already built"
else
clone_source "$pkgname" "$url" "$ref" "$commit"
clone_source "$pkgname" "$url" "$ref"
(
cd $pkgname
CFLAGS='-fcommon'