diff --git a/.github/scripts/git-smart-checkout.sh b/.github/scripts/git-smart-checkout.sh new file mode 100755 index 0000000000..20206d0bf1 --- /dev/null +++ b/.github/scripts/git-smart-checkout.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash +# git-smart-checkout: clone/reuse a repo and force-checkout a given ref (branch/tag/commit), +# with mirror URLs, minimal transfer, and automatic retries. + +# Robustness +set -euo pipefail +IFS=$'\n\t' + +# ---- Logging helpers -------------------------------------------------------- +log_info() { printf '[INFO] %s\n' "$*" >&2; } +log_warn() { printf '[WARN] %s\n' "$*" >&2; } +log_error() { printf '[ERROR] %s\n' "$*" >&2; } + +# ---- Usage ------------------------------------------------------------------ +usage() { + cat >&2 <<'USAGE' +Usage: + git-smart-checkout.sh -n -r -u [-u ...] [options] + +Required: + -n, --name local target directory (repo / path). + -r, --ref branch, tag or Commit-SHA. + -u, --url repo URL (multiple for mirror fallbacks) + +Options: + --retries amount of retries (default: 3). + --sleep wait seconds between retries (default: 3). + --no-clean no hard working dir cleanup after checkout + --full-if-needed if commit not reachable with shallow fetch, retry again without --depth + --keep-remote dont change remote config (otherwise set 'origin' to first URL). + --depth shallow depth for fetch (default: 1). only useful for branch/tag + --no-partial disable partial clone/filter (otherwise load blobs on-demand). + --no-tags dont fetch tags by default (default). + --allow-tags fetch tags (overwrites --no-tags). + +Examples: + git-smart-checkout.sh -n myrepo -r main -u https://github.com/org/proj.git -u https://gitmirror/org/proj.git + git-smart-checkout.sh -n myrepo -r v2.1.0 -u ssh://git@example.com/proj.git + git-smart-checkout.sh -n myrepo -r 1a2b3c4d5e --full-if-needed -u https://github.com/org/proj.git +USAGE + exit 2 +} + +# ---- Defaults --------------------------------------------------------------- +NAME="" +REF="" +URLS=() +RETRIES=3 +SLEEP_SEC=3 +CLEAN=1 +FULL_IF_NEEDED=0 +KEEP_REMOTE=0 +DEPTH=1 +USE_PARTIAL=1 +FETCH_TAGS=0 # 0 => --no-tags, 1 => allow tags + +# ---- Parse args ------------------------------------------------------------- +if [[ $# -eq 0 ]]; then usage; fi + +while (( "$#" )); do + case "$1" in + -n|--name) NAME="${2:-}"; shift 2 ;; + -r|--ref) REF="${2:-}"; shift 2 ;; + -u|--url) URLS+=("${2:-}"); shift 2 ;; + --retries) RETRIES="${2:-}"; shift 2 ;; + --sleep) SLEEP_SEC="${2:-}"; shift 2 ;; + --no-clean) CLEAN=0; shift ;; + --full-if-needed) FULL_IF_NEEDED=1; shift ;; + --keep-remote) KEEP_REMOTE=1; shift ;; + --depth) DEPTH="${2:-}"; shift 2 ;; + --no-partial) USE_PARTIAL=0; shift ;; + --no-tags) FETCH_TAGS=0; shift ;; + --allow-tags) FETCH_TAGS=1; shift ;; + -h|--help) usage ;; + *) log_error "Unknown option: $1"; usage ;; + esac +done + +# ---- Validate args ---------------------------------------------------------- +if [[ -z "$NAME" ]]; then log_error "Missing --name"; usage; fi +if [[ -z "$REF" ]]; then log_error "Missing --ref"; usage; fi +if [[ ${#URLS[@]} -eq 0 ]]; then log_error "At least one --url is required"; usage; fi +if ! command -v git >/dev/null 2>&1; then log_error "'git' not found"; exit 127; fi + +# ---- Helpers ---------------------------------------------------------------- +is_git_dir() { + [[ -d "$1/.git" ]] || git -C "$1" rev-parse --git-dir >/dev/null 2>&1 +} + +is_sha_like() { + [[ "$1" =~ ^[0-9a-fA-F]{7,40}$ ]] +} + +sleep_backoff() { + local attempt="$1" + local base="$SLEEP_SEC" + local delay=$(( base * attempt )) + [[ $delay -lt 1 ]] && delay=1 + sleep "$delay" +} + +git_retry() { + # git_retry + local desc="$1"; shift + local attempt=1 + local rc=0 + while :; do + if "$@"; then + return 0 + fi + rc=$? + if (( attempt >= RETRIES )); then + log_error "$desc: failed after $RETRIES tries (rc=$rc)" + return "$rc" + fi + log_warn "$desc: try $attempt failed (rc=$rc). retrying ..." + sleep_backoff "$attempt" + attempt=$(( attempt + 1 )) + done +} + +# Compose common fetch flags +fetch_flags_common=() +(( USE_PARTIAL )) && fetch_flags_common+=( "--filter=blob:none" ) +(( FETCH_TAGS == 0 )) && fetch_flags_common+=( "--no-tags" ) +fetch_flags_common+=( "--prune" "--prune-tags" "--update-head-ok" "--force" ) + +# With or without depth +fetch_flags_shallow=( "${fetch_flags_common[@]}" "--depth=$DEPTH" ) +fetch_flags_full=( "${fetch_flags_common[@]}" ) + +# ---- Clone or reuse existing ------------------------------------------------ +if [[ -d "$NAME" ]]; then + if is_git_dir "$NAME"; then + log_info "detected existing git repository: $NAME" + if (( KEEP_REMOTE == 0 )); then + log_info "setting 'origin' to first URL: ${URLS[0]}" + if git -C "$NAME" remote get-url origin >/dev/null 2>&1; then + git -C "$NAME" remote set-url origin "${URLS[0]}" + else + git -C "$NAME" remote add origin "${URLS[0]}" + fi + else + log_info "--keep-remote: remote config unchanged." + fi + else + log_error "directory '$NAME' exists, but not a git repository." + exit 1 + fi +else + # fresh clone: minimal & no-checkout + mkdir -p "$(dirname -- "$NAME")" + clone_ok=0 + for u in "${URLS[@]}"; do + log_info "Clone von $u -> $NAME (minimaler Transfer, no-checkout)" + clone_cmd=( git clone "--no-checkout" "--origin=origin" "$u" "$NAME" ) + (( USE_PARTIAL )) && clone_cmd+=( "--filter=blob:none" ) + (( FETCH_TAGS == 0 )) && clone_cmd+=( "--no-tags" ) + if git_retry "clone $u" "${clone_cmd[@]}"; then + clone_ok=1 + break + fi + done + if (( clone_ok == 0 )); then + log_error "clone failed on all URLs." + exit 1 + fi +fi + +# ---- Prepare partial clone config (for init/clone without --filter at fetch time) ---- +if (( USE_PARTIAL )); then + # Best effort: ensure repo knows it's a partial clone (helps späteres lazy-fetch) + git -C "$NAME" config --local remote.origin.promisor true || true + git -C "$NAME" config --local remote.origin.partialclonefilter "blob:none" || true + git -C "$NAME" config --local extensions.partialClone "origin" || true +fi + +# ---- Fetch target ref with minimal transfer --------------------------------- +# We'll try URLs in order; for each, attempt specific refspecs. +# Strategy: +# - If SHA-like: try shallow fetch of the object; fallback to full if enabled. +# - Else (name): try heads/, tags/, and plain shallow; fallback to full if needed. +enter_repo() { cd "$NAME"; } # for readability +enter_repo + +fetch_from_url_for_ref() { + # fetch_from_url_for_ref + local url="$1" ref="$2" mode="$3" ; shift 3 + local -a flags=() + if [[ "$mode" == "shallow" ]]; then + flags=( "${fetch_flags_shallow[@]}" ) + else + flags=( "${fetch_flags_full[@]}" ) + fi + + if is_sha_like "$ref"; then + # Try fetching the specific commit object + # FETCH_HEAD will point to the fetched object on success. + git fetch "${flags[@]}" "$url" "$ref" + return $? + else + # Try specific namespaces to avoid extra refs + local try=( + "refs/heads/$ref:refs/tmp/$ref" + "refs/tags/$ref:refs/tmp/$ref" + "$ref:refs/tmp/$ref" + ) + local t + for t in "${try[@]}"; do + if git fetch "${flags[@]}" "$url" "$t"; then + return 0 + fi + done + return 1 + fi +} + +fetch_ok=0 +for u in "${URLS[@]}"; do + # 1) Try shallow + if git_retry "fetch url: $u $REF" fetch_from_url_for_ref "$u" "$REF" "shallow"; then + fetch_ok=1 + break + fi + # 2) Optional fallback to full (still filtered blobs) + if (( FULL_IF_NEEDED )); then + log_warn "shallow fetch insufficient. trying without --depth from $u ..." + if git_retry "fetch url: $u $REF" fetch_from_url_for_ref "$u" "$REF" "full"; then + fetch_ok=1 + break + fi + fi +done + +if (( fetch_ok == 0 )); then + log_error "failed fetching reference '$REF' on all URLs." + exit 1 +fi + +# Resolve fetched commit +TARGET_COMMIT="$(git rev-parse --verify --quiet FETCH_HEAD || true)" +if [[ -z "$TARGET_COMMIT" ]]; then + log_error "couldnt resolve FETCH_HEAD." + exit 1 +fi +log_info "fetched: $TARGET_COMMIT" + +# ---- Forced checkout -------------------------------------------------------- +# We checkout detached at the exact commit to be fully deterministic. +# (No implicit branch tracking; avoids surprises if remote refs move.) +if (( CLEAN )); then + log_info "cleaning working dir (reset --hard, clean -fdx)" + git reset --hard + git clean -fdx +fi + +log_info "forced checkout (detached) auf $TARGET_COMMIT" +# Ensure no stray temp ref prevents checkout +git update-ref -d "refs/tmp/$REF" >/dev/null 2>&1 || true +git checkout --force --detach "$TARGET_COMMIT" + +# Small summary +log_info "working dir:" +git --no-pager log -1 --oneline +git --no-pager status --short --branch + +log_info "Done." diff --git a/.github/scripts/install-prereq.sh b/.github/scripts/install-prereq.sh index e04b95dde6..12017ba803 100755 --- a/.github/scripts/install-prereq.sh +++ b/.github/scripts/install-prereq.sh @@ -9,17 +9,16 @@ cd $X11_BUILD_DIR build_meson rendercheck https://gitlab.freedesktop.org/xorg/test/rendercheck rendercheck-1.6 if [ "$X11_OS" = "Linux" ]; then -build_meson drm https://gitlab.freedesktop.org/mesa/drm libdrm-2.4.121 "" \ - -Domap=enabled +build_meson drm https://gitlab.freedesktop.org/mesa/drm libdrm-2.4.121 -Domap=enabled fi build_meson libxcvt https://gitlab.freedesktop.org/xorg/lib/libxcvt libxcvt-0.1.0 build_ac xorgproto https://gitlab.freedesktop.org/xorg/proto/xorgproto xorgproto-2024.1 if [ "$X11_OS" = "Darwin" ]; then build_ac xset https://gitlab.freedesktop.org/xorg/app/xset xset-1.2.5 fi -build_ac_xts xts https://gitlab.freedesktop.org/xorg/test/xts master 12a887c2c72c4258962b56ced7b0aec782f1ffed +build_ac_xts xts https://gitlab.freedesktop.org/xorg/test/xts 12a887c2c72c4258962b56ced7b0aec782f1ffed -clone_source piglit https://gitlab.freedesktop.org/mesa/piglit main 265896c86f90cb72e8f218ba6a3617fca8b9a1e3 +clone_source piglit https://gitlab.freedesktop.org/mesa/piglit 265896c86f90cb72e8f218ba6a3617fca8b9a1e3 echo '[xts]' > piglit/piglit.conf echo "path=$X11_BUILD_DIR/xts" >> piglit/piglit.conf diff --git a/.github/scripts/util.sh b/.github/scripts/util.sh index a85c456e73..7c38e98f51 100644 --- a/.github/scripts/util.sh +++ b/.github/scripts/util.sh @@ -1,41 +1,30 @@ . .github/scripts/conf.sh +SOURCE_DIR=`pwd` + clone_source() { local pkgname="$1" local url="$2" local ref="$3" - local commit="$4" - if [ ! -f $pkgname/.git/config ]; then - echo "need to clone $pkgname" - if [ "$commit" ]; then - git clone $url $pkgname --branch=$ref - else - git clone $url $pkgname --branch=$ref --depth 1 - fi - else - echo "already cloned $pkgname" - fi - - if [ "$commit" ]; then - ( cd $pkgname && git checkout -f "$commit" ) - fi + $SOURCE_DIR/.github/scripts/git-smart-checkout.sh \ + --name "$pkgname" \ + --url "$url" \ + --ref "$ref" } build_meson() { local pkgname="$1" local url="$2" local ref="$3" - local commit="$4" - shift shift shift shift || true if [ -f $X11_PREFIX/$pkgname.DONE ]; then echo "package $pkgname already built" else - clone_source "$pkgname" "$url" "$ref" "$commit" + clone_source "$pkgname" "$url" "$ref" ( cd $pkgname meson "$@" build -Dprefix=$X11_PREFIX @@ -49,15 +38,13 @@ build_ac() { local pkgname="$1" local url="$2" local ref="$3" - local commit="$4" - shift shift shift shift || true if [ -f $X11_PREFIX/$pkgname.DONE ]; then echo "package $pkgname already built" else - clone_source "$pkgname" "$url" "$ref" "$commit" + clone_source "$pkgname" "$url" "$ref" ( cd $pkgname ./autogen.sh --prefix=$X11_PREFIX @@ -71,12 +58,10 @@ build_drv_ac() { local pkgname="$1" local url="$2" local ref="$3" - local commit="$4" - shift shift shift shift || true - clone_source "$pkgname" "$url" "$ref" "$commit" + clone_source "$pkgname" "$url" "$ref" ( cd $pkgname ./autogen.sh # --prefix=$X11_PREFIX @@ -88,15 +73,13 @@ build_ac_xts() { local pkgname="$1" local url="$2" local ref="$3" - local commit="$4" - shift shift shift shift || true if [ -f $X11_PREFIX/$pkgname.DONE ]; then echo "package $pkgname already built" else - clone_source "$pkgname" "$url" "$ref" "$commit" + clone_source "$pkgname" "$url" "$ref" ( cd $pkgname CFLAGS='-fcommon'