configng: refresh clone once, early — fix matrix-prep cache fingerprint

lib/functions/artifacts/artifact-rootfs.sh

@@ -20,10 +20,23 @@ function artifact_rootfs_config_dump() {
 	# invalidates the desktop rootfs cache — the package list, browser
 	# mapping, tier overrides, or branding may have changed.
 	if [[ "${BUILD_DESKTOP}" == "yes" ]]; then
+		# Read CONFIGNG_DESKTOPS_HASH from the cache/sources/armbian-configng
+		# clone. The clone is guaranteed fresh by the fetch_armbian_configng
+		# call in prep_conf_main_{minimal_ni,build_single} — both build
+		# entry points refresh it before this function ever runs. Keeping
+		# the fetch out of here (it used to live here for a couple of
+		# revisions) means the matrix-prep path and the full-build path
+		# converge on the same authoritative state at the same point in
+		# time, instead of each artifact's config_dump fetching mid-flow.
 		declare configng_desktops_hash="undetermined"
 		local configng_dir="${SRC}/cache/sources/armbian-configng"
 		if [[ -d "${configng_dir}/.git" ]]; then
 			configng_desktops_hash="$(git -C "${configng_dir}" log -1 --format=%H -- tools/modules/desktops/ 2>/dev/null || echo "unknown")"
+			# Operator-facing breadcrumb: short hash + commit subject so
+			# build logs make the cache fingerprint trivially traceable.
+			local configng_desktops_subject
+			configng_desktops_subject="$(git -C "${configng_dir}" log -1 --format=%s -- tools/modules/desktops/ 2>/dev/null || true)"
+			display_alert "configng desktops HEAD" "${configng_desktops_hash:0:12} ${configng_desktops_subject}" "info"
 		fi
 		artifact_input_variables[CONFIGNG_DESKTOPS_HASH]="${configng_desktops_hash}"
 	fi

lib/functions/configuration/config-desktop.sh

@@ -25,30 +25,70 @@
 #   DESKTOP_ENVIRONMENT_CONFIG_NAME
 #   DESKTOP_APPGROUPS_SELECTED
 
+# Ensure cache/sources/armbian-configng is a fresh clone of
+# https://github.com/armbian/configng#main. Idempotent and safe to call
+# multiple times (fetch_from_repo handles initial-clone and refresh).
+#
+# Called once per build from both prep_conf_main_minimal_ni (the json-
+# info / matrix-prep path used by armbian/os CI) AND
+# prep_conf_main_build_single (full image build). Having both paths
+# call this guarantees CONFIGNG_DESKTOPS_HASH is computed against the
+# real HEAD of configng, not a stale on-disk snapshot — which was the
+# root cause of the "I pushed to configng but the image still has the
+# old YAML" class of bug.
+#
+# Save/restore PWD because fetch_from_repo cd's into the work tree and
+# doesn't restore. Declare fetched_revision/_ts locals because
+# fetch_from_repo writes to them in caller scope.
+#
+# Non-fatal: failed fetch (offline, network blip, mirror down) logs a
+# warning and falls through with whatever is on disk — downstream
+# consumers handle the "stale or missing clone" case explicitly.
+function fetch_armbian_configng() {
+	# Fetch unconditionally — not gated on BUILD_DESKTOP=yes. The
+	# matrix-prep parent run never has BUILD_DESKTOP=yes (no specific
+	# board selected for the gha-matrix command), so gating here meant
+	# the clone was only refreshed by cli-jsoninfo.sh's secondary
+	# fetch — which is itself gated on the inventory file's absence
+	# and so silently skipped on `CLEAN_INFO=no`. Fetching here always
+	# guarantees the clone is fresh before any artifact's config_dump
+	# reads `git log -1 -- tools/modules/desktops/`, regardless of
+	# whether the caller is a desktop build, a CLI build, or
+	# matrix-prep.
+
+	# Skip the fetch when running inside a config-dump-json subprocess.
+	# info-gatherer-image.py spawns up to 128 parallel workers, each
+	# invoking compile.sh config-dump-json with CONFIG_DEFS_ONLY=yes
+	# (see armbian_run_command_and_parse_json_from_stdout). If every
+	# worker also fetched, they'd race on the same on-disk clone at
+	# cache/sources/armbian-configng — git explodes on .git/index.lock
+	# contention, `git checkout` exits 128, the subprocess produces no
+	# JSON, and the gatherer parse fails with "Expecting value: line 1
+	# column 1 (char 0)" for every desktop target. The parent has
+	# already populated the clone before spawning workers (via
+	# cli-jsoninfo.sh's explicit fetch on the matrix-prep path, or
+	# prep_conf_main_build_single's fetch on the single-image path),
+	# so subprocesses just need to read what's on disk.
+	[[ "${CONFIG_DEFS_ONLY}" == "yes" ]] && return 0
+
+	declare _save_pwd="${PWD}"
+	declare fetched_revision="" fetched_revision_ts=""
+	fetch_from_repo "https://github.com/armbian/configng" "armbian-configng" "branch:main" || \
+		display_alert "armbian-configng fetch_from_repo failed" "falling back to on-disk state if any" "wrn"
+	cd "${_save_pwd}" || true
+}
+
 function interactive_desktop_main_configuration() {
 	[[ $BUILD_DESKTOP != "yes" ]] && return 0
 
 	display_alert "desktop-config" "DESKTOP_ENVIRONMENT entry: ${DESKTOP_ENVIRONMENT}" "debug"
 
-	# Refresh the armbian-configng clone on EVERY desktop build,
-	# regardless of whether DESKTOP_ENVIRONMENT was pre-set. The
-	# clone feeds two downstream consumers:
-	#
-	#   1. The interactive DE-selection dialog below — only fires
-	#      when DESKTOP_ENVIRONMENT is empty.
-	#   2. artifact_rootfs_config_dump, which reads the clone's
-	#      `git log -1 -- tools/modules/desktops/` as the
-	#      CONFIGNG_DESKTOPS_HASH input that (via create-cache.sh's
-	#      cache_type) fingerprints the rootfs tarball filename.
-	#
-	# Keeping the fetch inside the `-z DESKTOP_ENVIRONMENT` branch
-	# meant every non-interactive build (CI, items-from-inventory,
-	# scripted local) skipped it — so the hash was computed against
-	# whatever stale clone happened to be on disk and the build
-	# happily cache-hit a pre-configng-change rootfs. Hoisting the
-	# fetch up here makes the clone authoritative for every
-	# BUILD_DESKTOP=yes invocation.
-	fetch_from_repo "https://github.com/armbian/configng" "armbian-configng" "branch:main"
+	# Refresh the armbian-configng clone before the DE-selection dialog
+	# runs. fetch_armbian_configng is idempotent — typically already
+	# invoked once by prep_conf_main_{minimal_ni,build_single} above
+	# this in the call chain; re-calling here is a cheap no-op that
+	# also handles standalone callers (e.g. plain config-only paths).
+	fetch_armbian_configng
 
 	local configng_dir="${SRC}/cache/sources/armbian-configng"
 	local yaml_dir="${configng_dir}/tools/modules/desktops/yaml"

lib/functions/general/git.sh

@@ -55,10 +55,16 @@ function improved_git_fetch() {
 function git_ensure_safe_directory() {
 	if [[ -n "$(command -v git)" ]]; then
 		local git_dir="$1"
-		if [[ -e "$1/.git" ]]; then
-			display_alert "git: Marking all directories as safe, which should include" "$git_dir" "debug"
-			git config --global --get safe.directory "$1" > /dev/null || regular_git config --global --add safe.directory "$1"
-		fi
+		# Mark the directory safe unconditionally. The previous gate on
+		# `-e "$1/.git"` was a bug: fetch_from_repo calls us BEFORE
+		# running `git init`, so on a freshly-prepared work tree (no
+		# .git yet) the safe.directory add was silently skipped — then
+		# the subsequent `regular_git checkout/clean` after init blew
+		# up with "fatal: detected dubious ownership". Setting
+		# safe.directory on a non-git path is harmless; git only
+		# consults the entry when an actual repo is accessed there.
+		display_alert "git: Marking directory as safe" "$git_dir" "debug"
+		git config --global --get safe.directory "$1" > /dev/null || regular_git config --global --add safe.directory "$1"
 	else
 		display_alert "git not installed" "a true wonder how you got this far without git - it will be installed for you" "warn"
 	fi

lib/functions/main/config-prepare.sh

@@ -28,6 +28,14 @@ function prep_conf_main_build_single() {
 
 	LOG_SECTION="do_main_configuration" do_with_conditional_logging do_main_configuration # This initializes the extension manager among a lot of other things, and call extension_prepare_config() hook
 
+	# Refresh armbian-configng once, early — needs BUILD_DESKTOP to be
+	# authoritative (set by do_main_configuration above). The clone is a
+	# build-wide input: feeds CONFIGNG_DESKTOPS_HASH (cache fingerprint)
+	# AND the DE dialog AND the runtime YAML parser. Doing this here
+	# means artifact_rootfs_config_dump can rely on a fresh on-disk
+	# tree instead of fetching mid-flow per artifact.
+	fetch_armbian_configng
+
 	interactive_desktop_main_configuration
 	interactive_finish # cleans up vars used for interactive
 
@@ -72,6 +80,14 @@ function prep_conf_main_minimal_ni() {
 	allow_no_family="${allow_no_family:-"yes"}" \
 		LOG_SECTION="do_main_configuration" do_with_conditional_logging do_main_configuration # This initializes the extension manager among a lot of other things, and call extension_prepare_config() hook
 
+	# Refresh armbian-configng for the non-interactive path too (matrix-
+	# prep / json-info / cli-build's minimal config). Same rationale as
+	# in prep_conf_main_build_single: every code path that ends up
+	# reading CONFIGNG_DESKTOPS_HASH (from artifact_rootfs_config_dump)
+	# needs the clone fresh, OR the OCI cache fingerprint resolves to
+	# the wrong tag and CI ships the pre-change config.
+	fetch_armbian_configng
+
 	# Required: does a lot of stuff and extension_prepare_config() hook
 	LOG_SECTION="do_extra_configuration" do_with_conditional_logging do_extra_configuration
 

lib/functions/rootfs/create-cache.sh

@@ -167,4 +167,4 @@ function extract_rootfs_artifact() {
 }
 
 # This comment strategically introduced to force a rebuild of all rootfs, as this file's contents are hashed into all rootfs versions.
-# Just a number to force rebuild 005
+# Just a number to force rebuild 007

lib/tools/common/armbian_utils.py

@@ -465,14 +465,34 @@ def armbian_run_command_and_parse_json_from_stdout(exec_cmd: list[str], params:
 			check=True,
 			universal_newlines=False,  # universal_newlines messes up bash encoding, don't use, instead decode utf8 manually;
 			bufsize=-1,  # full buffering
-			# Early (pre-param-parsing) optimizations for those in Armbian bash code, so use an ENV (not PARAM)
-			env={
+			# Inherit the parent's env (HOME, PATH, locale, GIT_*, etc.) and
+			# layer the build-specific overrides on top. Previously this
+			# dict was constructed from scratch, dropping HOME — which made
+			# every git invocation inside the subprocess fail to load
+			# ~/.gitconfig and miss the safe.directory entry the parent
+			# added for cache/sources/armbian-configng. Result was a
+			# "fatal: detected dubious ownership" cascade that broke
+			# config-dump-json for every desktop build target in the
+			# matrix-prep pass.
+			#
+			# CI is explicitly cleared: on GHA the parent runs with
+			# CI=true, which makes start_logging_section emit
+			# `::group::...` workflow markers to stdout. That pollutes
+			# the subprocess's stdout (which must be pure JSON for
+			# json.loads), causing every config-dump-json target to
+			# fail with "Expecting value: line 1 column 1 (char 0)"
+			# in matrix-prep — but only on GHA, not locally. Drop CI
+			# (and the closely-related GITHUB_ACTIONS) so subprocesses
+			# behave like a plain non-CI invocation regardless of where
+			# the matrix-prep parent runs.
+			env=({
+				**{k: v for k, v in os.environ.items() if k not in ("CI", "GITHUB_ACTIONS")},
 				"CONFIG_DEFS_ONLY": "yes",  # Dont do anything. Just output vars.
 				"ANSI_COLOR": "none",  # Do not use ANSI colors in logging output, don't write to log files
 				"WRITE_EXTENSIONS_METADATA": "no",  # Not interested in ext meta here
 				"ALLOW_ROOT": "yes",  # We're gonna be calling it as root, so allow it @TODO not the best option
 				"PRE_PREPARED_HOST": "yes"  # We're gonna be calling it as root, so allow it @TODO not the best option
-			},
+			}),
 			stderr=subprocess.PIPE
 		)
 	except subprocess.CalledProcessError as e:
@@ -500,7 +520,16 @@ def armbian_run_command_and_parse_json_from_stdout(exec_cmd: list[str], params:
 		info["logs"] = logs
 		return info
 	except json.decoder.JSONDecodeError as e:
-		log.error(f"Error parsing Armbian JSON: params: {params}, stderr: {'; '.join(logs[-5:])}")
+		# Dump raw stdout/stderr in full when the JSON parse fails — the
+		# parsed `logs` view drops anything that isn't "type::message" or
+		# bare stderr noise, and the last-5-lines tail hides the actual
+		# failure cause. PR #9866 debug aid; trim once matrix-prep is
+		# stable again.
+		raw_stdout = result.stdout.decode("utf8", errors="replace") if result and result.stdout else "<EMPTY>"
+		raw_stderr = result.stderr.decode("utf8", errors="replace") if result and result.stderr else "<EMPTY>"
+		log.error(f"Error parsing Armbian JSON: params: {params}, stderr_tail: {'; '.join(logs[-5:])}")
+		log.error(f"  RAW STDOUT for failed target {params.get('target_id','?')} ({len(raw_stdout)} bytes):\n{raw_stdout!r}")
+		log.error(f"  RAW STDERR for failed target {params.get('target_id','?')} ({len(raw_stderr)} bytes):\n{raw_stderr!r}")
 		# return {"in": params, "out": {}, "logs": logs, "config_ok": False}
 		raise e