Skip to content

E
etersoft-admin-essentials
Commit e61ab33a authored by Vitaly Lipatov's avatar Vitaly Lipatov
Browse files
Options

route-update: parallel group processing for faster DNS resolution

Process route groups in parallel using background subshells. Each group runs independently with its own pref base computed from the group's alphabetical index. Table allocation in /etc/iproute2/rt_tables is protected by flock to prevent races. On local-test with 3 groups: 163% CPU utilization confirms parallel execution. Co-Authored-By: 's avatarClaude Opus 4.6 (1M context) <noreply@anthropic.com>
parent 5de04da0
Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 78 deletions
router/route-update.sh
View file @ e61ab33a
...@@ -265,22 +265,27 @@ DIGGER ...@@ -265,22 +265,27 @@ DIGGER
} }
# Allocate a free table number in range 200-250 (step 10) # Allocate a free table number in range 200-250 (step 10)
# Uses flock to prevent race conditions during parallel group processing
RT_TABLES_LOCK="/var/lock/rt-tables.lock"
alloc_table() alloc_table()
{ {
local name="$1" local name="$1"
local used=$(awk '/^[0-9]/ {print $1}' /etc/iproute2/rt_tables | sort -n) (
local num=200 flock 8
while [ $num -le 250 ] ; do local used=$(awk '/^[0-9]/ {print $1}' /etc/iproute2/rt_tables | sort -n)
if ! echo "$used" | grep -q "^${num}$" ; then local num=200
[ -z "$SHOW" ] && echo "$num $name" >> /etc/iproute2/rt_tables while [ $num -le 250 ] ; do
vlog "Allocated table $num for $name" >&2 if ! echo "$used" | grep -q "^${num}$" ; then
echo "$num" [ -z "$SHOW" ] && echo "$num $name" >> /etc/iproute2/rt_tables
return vlog "Allocated table $num for $name" >&2
fi echo "$num"
num=$((num + 1)) return
done fi
log "ERROR: Cannot allocate table for $name: range 200-250 exhausted" >&2 num=$((num + 1))
return 1 done
log "ERROR: Cannot allocate table for $name: range 200-250 exhausted" >&2
return 1
) 8>"$RT_TABLES_LOCK"
} }
# Look up table number by name; allocate if not found # Look up table number by name; allocate if not found
...@@ -767,8 +772,85 @@ for line in sys.stdin: ...@@ -767,8 +772,85 @@ for line in sys.stdin:
return 0 return 0
} }
# --- Process a single group (runs in subshell for parallel execution) ---
# Args: gwdir resolve_func ipcmd label pref_base
process_group()
{
_gwdir="$1"
_resolve_func="$2"
_ipcmd="$3"
_label="$4"
_pref_counter="$5"
local name=$(basename "$_gwdir")
read_group_config "$_gwdir" "$_ipcmd" || return 0
_gw="$gw" ; _has_metric="$has_metric"
local group_state="$(dirname "$_gwdir")/$name"
# use routes_dir-relative path for state
group_state="${group_state#./}"
ensure_state_dir "$group_state"
vlog "[$name]$_label gw=$_gw has_metric=${_has_metric:-0} dir=$_gwdir"
# Save gateway in group state (shared by all lists)
[ -z "$SHOW" ] && cp "$_gwdir/gateway" "$STATE_DIR/$group_state/gateway" 2>/dev/null
# Collect .list files (follow symlinks)
local lists=$(find -L "$_gwdir" -maxdepth 1 -name '*.list' -type f 2>/dev/null | sort)
if [ -z "$lists" ] ; then
log "[$name]$_label No .list files, flushing all per-list tables"
if [ -z "$SHOW" ] ; then
for list_state in "$STATE_DIR/$group_state"/*/ ; do
[ -d "$list_state" ] || continue
[ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
$_ipcmd route flush table "$t" 2>/dev/null
$_ipcmd rule del lookup "$t" 2>/dev/null
done
rm -rf "$STATE_DIR/$group_state"/*/
fi
return 0
fi
# --- Per-list loop ---
for _f in $lists ; do
local bname=$(basename "$_f")
local list_name="${bname%.list}"
_table=$(lookup_table "$list_name")
[ -z "$_table" ] && { log "[$name/$list_name]$_label Cannot allocate table" >&2 ; continue ; }
_pref=$_pref_counter
_pref_counter=$((_pref_counter + PREF_STEP))
_state="$group_state/$list_name"
_tag="[$name/$list_name]"
ensure_state_dir "$_state"
vlog "$_tag$_label table=$_table pref=$_pref"
# Detect table number change
if [ -f "$STATE_DIR/$_state/table" ] ; then
local saved_table
read -r saved_table < "$STATE_DIR/$_state/table"
if [ "$saved_table" != "$_table" ] ; then
log "$_tag$_label Table changed $saved_table → $_table, flushing old"
if [ -z "$SHOW" ] ; then
$_ipcmd route flush table "$saved_table" 2>/dev/null
$_ipcmd rule del lookup "$saved_table" 2>/dev/null
rm -f "$STATE_DIR/$_state/hash"
fi
fi
fi
check_list_changed || continue
resolve_list_file
load_list_routes && log "$_tag$_label Done"
done
}
# --- Process a routes directory (works for both IPv4 and IPv6) --- # --- Process a routes directory (works for both IPv4 and IPv6) ---
# Per-list tables: each .list file gets its own routing table # Per-list tables: each .list file gets its own routing table
# Groups are processed in parallel for faster DNS resolution
process_routes() process_routes()
{ {
local routes_dir="$1" local routes_dir="$1"
...@@ -778,76 +860,24 @@ process_routes() ...@@ -778,76 +860,24 @@ process_routes()
[ -d "$routes_dir" ] || return 0 [ -d "$routes_dir" ] || return 0
_pref_counter=$PREF_BASE # Enumerate groups and assign pref bases
local group_index=0
local pids=""
for _gwdir in "$routes_dir"/*/ ; do for _gwdir in "$routes_dir"/*/ ; do
[ -d "$_gwdir" ] || continue [ -d "$_gwdir" ] || continue
local name=$(basename "$_gwdir") local pref_base=$((PREF_BASE + group_index * PREF_GROUP_STEP))
group_index=$((group_index + 1))
read_group_config "$_gwdir" "$_ipcmd" || continue
_gw="$gw" ; _has_metric="$has_metric"
local group_state="$routes_dir/$name"
ensure_state_dir "$group_state"
vlog "[$name]$_label gw=$_gw has_metric=${_has_metric:-0} dir=$_gwdir"
# Save gateway in group state (shared by all lists)
[ -z "$SHOW" ] && cp "$_gwdir/gateway" "$STATE_DIR/$group_state/gateway" 2>/dev/null
# Collect .list files (follow symlinks)
local lists=$(find -L "$_gwdir" -maxdepth 1 -name '*.list' -type f 2>/dev/null | sort)
if [ -z "$lists" ] ; then
log "[$name]$_label No .list files, flushing all per-list tables"
if [ -z "$SHOW" ] ; then
for list_state in "$STATE_DIR/$group_state"/*/ ; do
[ -d "$list_state" ] || continue
[ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
$_ipcmd route flush table "$t" 2>/dev/null
$_ipcmd rule del lookup "$t" 2>/dev/null
done
rm -rf "$STATE_DIR/$group_state"/*/
fi
continue
fi
# --- Per-list loop --- process_group "$_gwdir" "$_resolve_func" "$_ipcmd" "$_label" "$pref_base" &
# Align pref counter to next group boundary pids="$pids $!"
_pref_counter=$(( (_pref_counter + PREF_GROUP_STEP - 1) / PREF_GROUP_STEP * PREF_GROUP_STEP )) done
for _f in $lists ; do
local bname=$(basename "$_f")
local list_name="${bname%.list}"
_table=$(lookup_table "$list_name")
[ -z "$_table" ] && { log "[$name/$list_name]$_label Cannot allocate table" >&2 ; continue ; }
_pref=$_pref_counter
_pref_counter=$((_pref_counter + PREF_STEP))
_state="$group_state/$list_name"
_tag="[$name/$list_name]"
ensure_state_dir "$_state"
vlog "$_tag$_label table=$_table pref=$_pref"
# Detect table number change
if [ -f "$STATE_DIR/$_state/table" ] ; then
local saved_table
read -r saved_table < "$STATE_DIR/$_state/table"
if [ "$saved_table" != "$_table" ] ; then
log "$_tag$_label Table changed $saved_table → $_table, flushing old"
if [ -z "$SHOW" ] ; then
$_ipcmd route flush table "$saved_table" 2>/dev/null
$_ipcmd rule del lookup "$saved_table" 2>/dev/null
rm -f "$STATE_DIR/$_state/hash"
fi
fi
fi
check_list_changed || continue # Wait for all groups to finish
resolve_list_file local failed=0
load_list_routes && log "$_tag$_label Done" for pid in $pids ; do
done wait "$pid" || failed=$((failed + 1))
done done
[ "$failed" -gt 0 ] && log "WARNING:$_label $failed group(s) failed" || true
} }
# --- Cleanup orphaned state dirs --- # --- Cleanup orphaned state dirs ---
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment