Commit 6dd31ef2 authored by Vitaly Lipatov's avatar Vitaly Lipatov

route-update: cache non-volatile domain status for 7 days

detect_volatile_domains() was resolving every domain via dig on each run, which took 30+ minutes for large lists (e.g. 2097 whatsapp domains). Cache domains confirmed as non-volatile (stable TTL, same IPs from different resolvers) and skip them on subsequent runs. Cache expires after 7 days and is ignored with --force. Co-Authored-By: 's avatarClaude Opus 4.6 (1M context) <noreply@anthropic.com>
parent da718350
...@@ -156,10 +156,13 @@ detect_volatile_domains() ...@@ -156,10 +156,13 @@ detect_volatile_domains()
local state="$1" name="$2" label="$3" rtype="$4" local state="$1" name="$2" label="$3" rtype="$4"
shift 4 shift 4
local volatile_file="$STATE_DIR/$state/volatile_domains" local volatile_file="$STATE_DIR/$state/volatile_domains"
local nonvolatile_cache="$STATE_DIR/$state/nonvolatile_cache_${rtype}"
local ttl_threshold=120 local ttl_threshold=120
local cache_max_age=$((7 * 86400)) # 7 days
local domains=$(mktemp) local domains=$(mktemp)
local uncached=$(mktemp)
local checker=$(mktemp) local checker=$(mktemp)
trap "rm -f $domains $checker" RETURN trap "rm -f $domains $uncached $checker" RETURN
# Extract unique domain names from list files # Extract unique domain names from list files
for f in "$@" ; do for f in "$@" ; do
...@@ -168,6 +171,22 @@ detect_volatile_domains() ...@@ -168,6 +171,22 @@ detect_volatile_domains()
[ -s "$domains" ] || { rm -f "$volatile_file" ; return 0 ; } [ -s "$domains" ] || { rm -f "$volatile_file" ; return 0 ; }
# Filter out domains cached as non-volatile (cache valid for cache_max_age)
if [ -f "$nonvolatile_cache" ] && [ -z "$FORCE" ] ; then
local cache_age=$(( $(date +%s) - $(stat -c %Y "$nonvolatile_cache") ))
if [ "$cache_age" -lt "$cache_max_age" ] ; then
# Only check domains not in the non-volatile cache
grep -Fxv -f "$nonvolatile_cache" "$domains" > "$uncached" || true
local total=$(wc -l < "$domains")
local skipped=$(( total - $(wc -l < "$uncached") ))
[ "$skipped" -gt 0 ] && vlog "[$name]$label skipped $skipped cached non-volatile domains" || true
else
cp "$domains" "$uncached"
fi
else
cp "$domains" "$uncached"
fi
# Parallel dig via temp script (avoids quoting issues with xargs) # Parallel dig via temp script (avoids quoting issues with xargs)
# Volatile = low TTL OR different IPs from local vs extra DNS resolver # Volatile = low TTL OR different IPs from local vs extra DNS resolver
cat > "$checker" << 'CHECKER' cat > "$checker" << 'CHECKER'
...@@ -189,7 +208,28 @@ fi ...@@ -189,7 +208,28 @@ fi
[ -n "$reason" ] && printf "%s\t%s records=%s\n" "$domain" "$reason" "$count" [ -n "$reason" ] && printf "%s\t%s records=%s\n" "$domain" "$reason" "$count"
CHECKER CHECKER
chmod +x "$checker" chmod +x "$checker"
xargs -P 10 -I{} "$checker" "$rtype" "$ttl_threshold" {} "$EXTRA_DNS" < "$domains" > "$volatile_file"
if [ -s "$uncached" ] ; then
xargs -P 10 -I{} "$checker" "$rtype" "$ttl_threshold" {} "$EXTRA_DNS" < "$uncached" > "$volatile_file"
else
: > "$volatile_file"
fi
# Rebuild non-volatile cache: all checked domains minus volatile ones
local volatile_names=$(mktemp)
[ -s "$volatile_file" ] && awk -F' ' '{print $1}' "$volatile_file" > "$volatile_names" || : > "$volatile_names"
# Merge previous cache (if valid) with newly confirmed non-volatile domains
local new_nonvolatile=$(mktemp)
if [ -s "$uncached" ] ; then
grep -Fxv -f "$volatile_names" "$uncached" > "$new_nonvolatile" || true
fi
{
[ -f "$nonvolatile_cache" ] && [ -z "$FORCE" ] && cat "$nonvolatile_cache"
cat "$new_nonvolatile"
} | sort -u > "$nonvolatile_cache.tmp"
# Keep only domains still in current list
grep -Fx -f "$domains" "$nonvolatile_cache.tmp" > "$nonvolatile_cache" || true
rm -f "$nonvolatile_cache.tmp" "$volatile_names" "$new_nonvolatile"
if [ -s "$volatile_file" ] ; then if [ -s "$volatile_file" ] ; then
vlog "[$name]$label volatile domains (TTL≤${ttl_threshold}s or diff resolvers): $(wc -l < "$volatile_file")" vlog "[$name]$label volatile domains (TTL≤${ttl_threshold}s or diff resolvers): $(wc -l < "$volatile_file")"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment