Commit 6fb0f9be authored by Vitaly Lipatov's avatar Vitaly Lipatov

route-health: use vpn_status from InfluxDB for faster failure detection

Query vpn_status (connected=0/1) alongside ping data. If VPN tunnel is reported disconnected, gateway is immediately marked dead without waiting for ping loss to accumulate. VPN status is checked with 1m window (vs 3m for ping) since it updates every 10s. Shows vpn=DOWN marker in --show output for affected gateways. Co-Authored-By: 's avatarClaude Opus 4.6 <noreply@anthropic.com>
parent 782133ea
...@@ -21,15 +21,21 @@ DOWN_THRESHOLD=2 ...@@ -21,15 +21,21 @@ DOWN_THRESHOLD=2
SHOW= SHOW=
[ "$1" = "--show" ] && SHOW=1 [ "$1" = "--show" ] && SHOW=1
# --- Query InfluxDB for packet loss --- # --- Query InfluxDB for packet loss and VPN status ---
HEALTH_DATA=$(mktemp) HEALTH_DATA=$(mktemp)
trap 'rm -f "$HEALTH_DATA"' EXIT VPN_DATA=$(mktemp)
trap 'rm -f "$HEALTH_DATA" "$VPN_DATA"' EXIT
response=$(curl -sG "$INFLUXDB_URL" \ response=$(curl -sG "$INFLUXDB_URL" \
--data-urlencode "db=$INFLUXDB_DB" \ --data-urlencode "db=$INFLUXDB_DB" \
--data-urlencode "q=SELECT last(percent_packet_loss) FROM ping WHERE time > now() - 3m GROUP BY gateway" \ --data-urlencode "q=SELECT last(percent_packet_loss) FROM ping WHERE time > now() - 3m GROUP BY gateway" \
--max-time 5 2>/dev/null) --max-time 5 2>/dev/null)
vpn_response=$(curl -sG "$INFLUXDB_URL" \
--data-urlencode "db=$INFLUXDB_DB" \
--data-urlencode "q=SELECT last(connected) FROM vpn_status WHERE time > now() - 1m GROUP BY gateway" \
--max-time 5 2>/dev/null)
if [ -z "$response" ] ; then if [ -z "$response" ] ; then
log "ERROR: Failed to query InfluxDB" log "ERROR: Failed to query InfluxDB"
exit 1 exit 1
...@@ -38,6 +44,7 @@ fi ...@@ -38,6 +44,7 @@ fi
# Parse JSON → "gateway packet_loss" lines # Parse JSON → "gateway packet_loss" lines
epm -y assure jq || fatal "jq is required" epm -y assure jq || fatal "jq is required"
echo "$response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$HEALTH_DATA" echo "$response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$HEALTH_DATA"
echo "$vpn_response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$VPN_DATA" 2>/dev/null
if [ ! -s "$HEALTH_DATA" ] ; then if [ ! -s "$HEALTH_DATA" ] ; then
log "ERROR: No health data from InfluxDB" log "ERROR: No health data from InfluxDB"
...@@ -47,15 +54,28 @@ fi ...@@ -47,15 +54,28 @@ fi
if [ -n "$SHOW" ] ; then if [ -n "$SHOW" ] ; then
log "InfluxDB data (last 3m):" log "InfluxDB data (last 3m):"
sed 's/^/ /' "$HEALTH_DATA" sed 's/^/ /' "$HEALTH_DATA"
if [ -s "$VPN_DATA" ] ; then
log "VPN status (last 1m):"
sed 's/^/ /' "$VPN_DATA"
fi
echo echo
fi fi
# Get health status for a monitor tag # Get health status for a monitor tag
# Usage: get_health TAG # Usage: get_health TAG
# Returns: healthy, degraded, dead # Returns: healthy, degraded, dead
# Checks both packet loss (ping) and VPN tunnel status
get_health() get_health()
{ {
local tag="$1" local tag="$1"
# Check VPN status first — if tunnel is down, immediately dead
local vpn=$(grep "^${tag} " "$VPN_DATA" 2>/dev/null | awk '{print $2}')
if [ "$vpn" = "0" ] ; then
echo "dead"
return
fi
local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}') local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}')
if [ -z "$loss" ] ; then if [ -z "$loss" ] ; then
...@@ -105,7 +125,9 @@ eval_group_health() ...@@ -105,7 +125,9 @@ eval_group_health()
local st=$(get_health "$tag") local st=$(get_health "$tag")
local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}') local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}')
local ld="${loss:-no_data}%" ; [ -z "$loss" ] && ld="no_data" local ld="${loss:-no_data}%" ; [ -z "$loss" ] && ld="no_data"
group_detail="${group_detail:+$group_detail, }$tag=$ld($st)" local vpn=$(grep "^${tag} " "$VPN_DATA" 2>/dev/null | awk '{print $2}')
local vpn_mark="" ; [ "$vpn" = "0" ] && vpn_mark=",vpn=DOWN"
group_detail="${group_detail:+$group_detail, }$tag=$ld($st$vpn_mark)"
[ "$st" = "healthy" ] && any_healthy=1 [ "$st" = "healthy" ] && any_healthy=1
[ "$st" != "dead" ] && all_dead="" [ "$st" != "dead" ] && all_dead=""
done < "$gwdir/gateway" done < "$gwdir/gateway"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment