Commit 7fa27d27 authored by Ivan Mazhukin's avatar Ivan Mazhukin

avoid remote sync race in parallel runs

parent 7b8ae14a
......@@ -26,6 +26,8 @@ LOG_ROOT="$DEFAULT_LOG_ROOT"
LOG_FILE=""
REMOTE_ARGS=()
REMOTE_SYNC_DIR=""
PARALLEL_REMOTE_SOURCE_DIR=""
PARALLEL_REMOTE_SYNC_DIR=""
RUN_TOKEN="$(date +%Y%m%d-%H%M%S)-$$"
SYSTEM_INPUTS=()
PRESET_NAMES=()
......@@ -336,24 +338,27 @@ find_remote_builder_source_path() {
remote_sync_dir_for_source() {
local source_dir="$1"
local local_user base_name
local local_user base_name suffix
local_user="$(id -un)"
base_name="$(basename "$source_dir")"
printf '/tmp/epm-docker-test-sync/%s-%s-%s\n' \
suffix="$(sanitize_name "${3:-${SYSTEM_IMAGE:-run}}")"
printf '/tmp/epm-docker-test-sync/%s-%s-%s-%s\n' \
"$(sanitize_name "$local_user")" \
"$(sanitize_name "$base_name")" \
"$(sanitize_name "$RUN_TOKEN")"
"$(sanitize_name "$RUN_TOKEN")" \
"$suffix"
}
sync_local_source_to_remote() {
local source_dir="$1"
local target="$2"
local suffix="${3:-}"
local remote_dir
require_command rsync
remote_dir="$(remote_sync_dir_for_source "$source_dir")"
remote_dir="$(remote_sync_dir_for_source "$source_dir" "$target" "$suffix")"
info "Remote eepm tree not found in /srv; syncing local tree to $target:$remote_dir"
......@@ -647,7 +652,9 @@ build_remote_args() {
target="$(remote_target)"
if [[ "$SOURCE_KIND" == "local" || "$SOURCE_KIND" == "explicit" ]]; then
if remote_source="$(find_remote_builder_source_path "$target")"; then
if [[ -n "$PARALLEL_REMOTE_SOURCE_DIR" ]]; then
remote_source="$PARALLEL_REMOTE_SOURCE_DIR"
elif remote_source="$(find_remote_builder_source_path "$target")"; then
info "Remote eepm tree found at: $remote_source"
else
if [[ "$SOURCE_KIND" == "local" ]]; then
......@@ -876,6 +883,66 @@ run_for_system_parallel() {
return 0
}
prepare_parallel_runner() {
case "$RUN_MODE" in
auto)
if can_use_local_docker; then
EFFECTIVE_RUN_MODE="local"
else
warn "Local Docker is unavailable; falling back to ssh runner"
EFFECTIVE_RUN_MODE="remote"
fi
;;
local|remote)
EFFECTIVE_RUN_MODE="$RUN_MODE"
;;
*)
fatal "Unsupported run mode: $RUN_MODE"
;;
esac
}
prepare_parallel_remote_source() {
local target
local local_source
local remote_source
PARALLEL_REMOTE_SOURCE_DIR=""
PARALLEL_REMOTE_SYNC_DIR=""
[[ "${EFFECTIVE_RUN_MODE:-$RUN_MODE}" == "remote" ]] || return 0
[[ "$SOURCE_KIND" == "local" || "$SOURCE_KIND" == "explicit" ]] || return 0
target="$(remote_target)"
if remote_source="$(find_remote_builder_source_path "$target")"; then
info "Remote eepm tree found at: $remote_source"
PARALLEL_REMOTE_SOURCE_DIR="$remote_source"
return 0
fi
if [[ "$SOURCE_KIND" == "local" ]]; then
local_source="$(resolve_checked_source_path "${SOURCE_PATH:-$(pwd -P)}")" || exit 1
else
local_source="$(resolve_checked_source_path "$SOURCE_PATH")" || exit 1
fi
remote_source="$(sync_local_source_to_remote "$local_source" "$target" parallel)" || exit 1
PARALLEL_REMOTE_SOURCE_DIR="$remote_source"
PARALLEL_REMOTE_SYNC_DIR="$remote_source"
}
cleanup_parallel_remote_source() {
local target
[[ -n "$PARALLEL_REMOTE_SYNC_DIR" ]] || return 0
target="$(remote_target)"
if ! cleanup_remote_sync_dir "$target" "$PARALLEL_REMOTE_SYNC_DIR"; then
warn "Failed to clean up remote sync dir: $PARALLEL_REMOTE_SYNC_DIR"
fi
PARALLEL_REMOTE_SYNC_DIR=""
PARALLEL_REMOTE_SOURCE_DIR=""
}
parse_args() {
local positional=()
......@@ -1027,6 +1094,7 @@ main() {
local result_dir
local i sys res log
result_dir="$(mktemp -d "${TMPDIR:-/tmp}/epm-docker-test-results.XXXXXX")"
prepare_parallel_runner
if ((PARALLEL_JOBS > 0)); then
info "Running $total tests in parallel (max $PARALLEL_JOBS)"
......@@ -1038,6 +1106,7 @@ main() {
else
info "Live output is suppressed in parallel mode; logs will be saved to files"
fi
prepare_parallel_remote_source
for system_image in "${TARGET_SYSTEMS[@]}"; do
((index += 1))
......@@ -1051,6 +1120,7 @@ main() {
done
wait
cleanup_parallel_remote_source
for ((i = 1; i <= total; i++)); do
if [[ -f "$result_dir/$i.result" ]]; then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment