Watchdog v4 — Intelligent Service Monitor & Auto‑Recovery

#!/bin/bash
# ==================================================================================
# 🐕 WATCHDOG v4.0 - Intelligent Service Monitor & Auto-Recovery System
# ==================================================================================
# Universal Linux service monitoring with smart socket-activation support
# # Features:
# ✅ Socket-activated service support (snapd, packagekit, etc.)
# ✅ Multi-tier service classification
# ✅ Advanced notification system (Email, Webhook, Telegram)
# ✅ Auto-update from GitHub
# ✅ System health monitoring
# ✅ Intelligent restart policies
# ✅ Multi-language support
# ✅ Comprehensive logging and analytics
#
# Author: Slaweally
# GitHub: https://github.com/slaweally/Watchdog/
# License: MIT
# ================================================================================== VERSION="4.0.0"
SCRIPT_NAME="$(basename "$0")"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_PATH="$SCRIPT_DIR/$SCRIPT_NAME" # ==================================================================================
# 🔧 CONFIGURATION PATHS
# ==================================================================================
CONFIG_DIR="$HOME/.config/watchdog"
CONFIG_FILE="$CONFIG_DIR/config.conf"
NOTIFICATION_CONFIG="$CONFIG_DIR/notifications.conf"
SERVICE_TYPES_CONFIG="$CONFIG_DIR/service_types.conf"
LOG_FILE="$CONFIG_DIR/watchdog.log"
PID_FILE="$CONFIG_DIR/watchdog.pid"
LANG_DIR="$CONFIG_DIR/lang"
UPDATE_CHECK_FILE="$CONFIG_DIR/.last_update_check" # ==================================================================================
# 🌐 GITHUB INTEGRATION
# ==================================================================================
GITHUB_REPO="slaweally/Watchdog/"
GITHUB_API="https://api.github.com/repos/$GITHUB_REPO"
GITHUB_RAW="https://raw.githubusercontent.com/$GITHUB_REPO/main"
UPDATE_CHECK_INTERVAL=86400 # 24 hours # ==================================================================================
# 🎨 COLOR SCHEMES
# ==================================================================================
declare -A COLORS=( [RED]='\033[0;31m' [GREEN]='\033[0;32m' [YELLOW]='\033[1;33m' [BLUE]='\033[0;34m' [PURPLE]='\033[0;35m' [CYAN]='\033[0;36m' [WHITE]='\033[1;37m' [GRAY]='\033[0;37m' [BOLD]='\033[1m' [DIM]='\033[2m' [NC]='\033[0m'
) # ==================================================================================
# 📊 SERVICE CLASSIFICATION SYSTEM
# ==================================================================================
declare -A DEFAULT_SERVICE_TYPES=( # Critical Services - Must always be running ["ssh.service"]="critical" ["sshd.service"]="critical" ["systemd-networkd.service"]="critical" ["systemd-resolved.service"]="critical" ["cron.service"]="critical" ["crond.service"]="critical" ["rsyslog.service"]="critical" ["systemd-journald.service"]="critical" # Socket-Activated Services - Normal to be inactive when idle ["snapd.service"]="socket-activated" ["packagekit.service"]="socket-activated" ["accounts-daemon.service"]="socket-activated" ["udisks2.service"]="socket-activated" ["polkit.service"]="socket-activated" ["systemd-hostnamed.service"]="socket-activated" ["systemd-localed.service"]="socket-activated" ["systemd-timedated.service"]="socket-activated" # On-Demand Services - Started when needed ["systemd-tmpfiles-clean.service"]="on-demand" ["apt-daily.service"]="on-demand" ["apt-daily-upgrade.service"]="on-demand" ["man-db.service"]="on-demand" ["dpkg-db-backup.service"]="on-demand" # Application Services - User applications ["apache2.service"]="application" ["nginx.service"]="application" ["mysql.service"]="application" ["mariadb.service"]="application" ["postgresql.service"]="application" ["redis.service"]="application" ["mongodb.service"]="application" ["docker.service"]="application" ["containerd.service"]="application" # Optional Services - Nice to have but not critical ["bluetooth.service"]="optional" ["cups.service"]="optional" ["avahi-daemon.service"]="optional" ["NetworkManager.service"]="optional"
) # ==================================================================================
# 🌍 INTERNATIONALIZATION SYSTEM
# ==================================================================================
declare -A MESSAGES
LANGUAGE="en" init_i18n() { mkdir -p "$LANG_DIR" # English (Default) cat > "$LANG_DIR/en.json" << 'EOF'
{ "welcome": "Welcome to Watchdog v4.0 Setup", "language_select": "Select your language", "service_discovery": "Discovering system services", "service_classification": "Classifying services by type", "found_services": "Found %d services", "critical_services": "Critical Services (always monitored)", "application_services": "Application Services", "socket_services": "Socket-Activated Services", "optional_services": "Optional Services", "select_services": "Select services to monitor", "notification_setup": "Notification Setup", "email_prompt": "Email address for notifications", "webhook_prompt": "Discord/Slack webhook URL", "telegram_setup": "Telegram notification setup", "interval_prompt": "Monitoring interval (minutes)", "config_saved": "Configuration saved successfully", "service_started": "Watchdog service started", "service_stopped": "Watchdog service stopped", "update_available": "Update available: v%s", "update_installing": "Installing update", "update_complete": "Update completed successfully", "monitoring_active": "Monitoring %d services", "service_healthy": "Service %s is healthy", "service_restarted": "Service %s restarted", "service_failed": "Service %s failed to restart", "socket_healthy": "Socket %s is active", "socket_restarted": "Socket %s restarted", "system_health_ok": "System health: OK", "high_load": "High system load detected", "high_memory": "High memory usage detected", "high_disk": "High disk usage detected"
}
EOF # Turkish cat > "$LANG_DIR/tr.json" << 'EOF'
{ "welcome": "Watchdog v4.0 Kurulumuna Hoş Geldiniz", "language_select": "Dilinizi seçin", "service_discovery": "Sistem servisleri keşfediliyor", "service_classification": "Servisler türlerine göre sınıflandırılıyor", "found_services": "%d servis bulundu", "critical_services": "Kritik Servisler (her zaman izlenir)", "application_services": "Uygulama Servisleri", "socket_services": "Socket-Aktif Servisler", "optional_services": "Opsiyonel Servisler", "select_services": "İzlenecek servisleri seçin", "notification_setup": "Bildirim Kurulumu", "email_prompt": "Bildirimler için e-posta adresi", "webhook_prompt": "Discord/Slack webhook URL'si", "telegram_setup": "Telegram bildirim kurulumu", "interval_prompt": "İzleme aralığı (dakika)", "config_saved": "Yapılandırma başarıyla kaydedildi", "service_started": "Watchdog servisi başlatıldı", "service_stopped": "Watchdog servisi durduruldu", "update_available": "Güncelleme mevcut: v%s", "update_installing": "Güncelleme yükleniyor", "update_complete": "Güncelleme başarıyla tamamlandı", "monitoring_active": "%d servis izleniyor", "service_healthy": "Servis %s sağlıklı", "service_restarted": "Servis %s yeniden başlatıldı", "service_failed": "Servis %s yeniden başlatılamadı", "socket_healthy": "Socket %s aktif", "socket_restarted": "Socket %s yeniden başlatıldı", "system_health_ok": "Sistem sağlığı: İYİ", "high_load": "Yüksek sistem yükü tespit edildi", "high_memory": "Yüksek bellek kullanımı tespit edildi", "high_disk": "Yüksek disk kullanımı tespit edildi"
}
EOF load_language "$LANGUAGE"
} load_language() { local lang="${1:-en}" local lang_file="$LANG_DIR/$lang.json" MESSAGES=() if [[ -f "$lang_file" ]] && command -v jq >/dev/null 2>&1; then while IFS='=' read -r key value; do MESSAGES["$key"]="$value" done < <(jq -r 'to_entries[] | "\(.key)=\(.value)"' "$lang_file" 2>/dev/null) fi
} msg() { local key="$1" local text="${MESSAGES[$key]:-$key}" shift printf "$text" "$@"
} # ==================================================================================
# 🖼️ UI COMPONENTS
# ==================================================================================
show_header() { clear echo -e "${COLORS[CYAN]}${COLORS[BOLD]}" cat << 'EOF' ██╗ ██╗ █████╗ ████████╗ ██████╗██╗ ██╗██████╗ ██████╗ ██████╗ ██╗ ██╗ ██████╗ ██║ ██║██╔══██╗╚══██╔══╝██╔════╝██║ ██║██╔══██╗██╔═══██╗██╔════╝ ██║ ██║██╔═████╗ ██║ █╗ ██║███████║ ██║ ██║ ███████║██║ ██║██║ ██║██║ ███╗ ██║ ██║██║██╔██║ ██║███╗██║██╔══██║ ██║ ██║ ██╔══██║██║ ██║██║ ██║██║ ██║ ╚██╗██╔╝████╔╝██║ ╚███╔███╔╝██║ ██║ ██║ ╚██████╗██║ ██║██████╔╝╚██████╔╝╚██████╔╝ ╚███╔╝ ╚██████╔╝ ╚══╝╚══╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚═════╝ ╚═════╝ ╚═════╝ ╚══╝ ╚═════╝ EOF echo -e "${COLORS[NC]}" echo -e "${COLORS[WHITE]}${COLORS[BOLD]}🐕 Intelligent Service Monitor & Auto-Recovery System v$VERSION${COLORS[NC]}" echo -e "${COLORS[BLUE]}═══════════════════════════════════════════════════════════════════════════════════════${COLORS[NC]}" echo
} print_status() { local level="$1" local message="$2" local timestamp="$(date '+%H:%M:%S')" case "$level" in "success") echo -e "${COLORS[GREEN]}✅ [$timestamp] $message${COLORS[NC]}" ;; "error") echo -e "${COLORS[RED]}❌ [$timestamp] $message${COLORS[NC]}" ;; "warning") echo -e "${COLORS[YELLOW]}⚠️ [$timestamp] $message${COLORS[NC]}" ;; "info") echo -e "${COLORS[BLUE]}ℹ️ [$timestamp] $message${COLORS[NC]}" ;; "debug") echo -e "${COLORS[GRAY]}🔍 [$timestamp] $message${COLORS[NC]}" ;; "update") echo -e "${COLORS[PURPLE]}🔄 [$timestamp] $message${COLORS[NC]}" ;; *) echo -e "${COLORS[WHITE]}📝 [$timestamp] $message${COLORS[NC]}" ;; esac
} show_progress_bar() { local current=$1 local total=$2 local width=50 local percentage=$((current * 100 / total)) local completed=$((current * width / total)) printf "\r${COLORS[CYAN]}[" printf "%${completed}s" | tr ' ' '█' printf "%$((width - completed))s" | tr ' ' '░' printf "] %d%% (%d/%d)${COLORS[NC]}" "$percentage" "$current" "$total" [[ $current -eq $total ]] && echo
} # ==================================================================================
# 📝 LOGGING SYSTEM
# ==================================================================================
log_message() { local level="$1" local message="$2" local timestamp="$(date '+%Y-%m-%d %H:%M:%S')" local pid="$$" echo "[$timestamp] [$pid] [$level] $message" >> "$LOG_FILE" # Rotate logs if too large (10MB) if [[ -f "$LOG_FILE" ]] && [[ $(stat -c%s "$LOG_FILE" 2>/dev/null) -gt 10485760 ]]; then tail -n 1000 "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE" log_message "INFO" "Log file rotated" fi
} # ==================================================================================
# 🔄 PID MANAGEMENT
# ==================================================================================
create_pid_file() { mkdir -p "$(dirname "$PID_FILE")" echo $$ > "$PID_FILE" log_message "INFO" "Watchdog started with PID: $$"
} remove_pid_file() { [[ -f "$PID_FILE" ]] && rm -f "$PID_FILE" log_message "INFO" "Watchdog stopped, PID file removed"
} is_watchdog_running() { if [[ -f "$PID_FILE" ]]; then local pid=$(cat "$PID_FILE" 2>/dev/null) if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then return 0 else rm -f "$PID_FILE" return 1 fi fi return 1
} # ==================================================================================
# 🔍 SERVICE DISCOVERY & CLASSIFICATION
# ==================================================================================
discover_services() { # Get all running services local services=() mapfile -t services < <(systemctl list-units --type=service --state=running --no-pager --no-legend 2>/dev/null | \ awk '{print $1}' | grep -E '\.service$' | sort) # Get enabled but not running services local enabled_services=() mapfile -t enabled_services < <(systemctl list-unit-files --type=service --state=enabled --no-pager --no-legend 2>/dev/null | \ awk '{print $1}' | grep -E '\.service$' | sort) # Combine and deduplicate local all_services=() for service in "${services[@]}" "${enabled_services[@]}"; do if [[ ! " ${all_services[*]} " =~ " ${service} " ]]; then all_services+=("$service") fi done echo "${all_services[@]}"
} classify_service() { local service="$1" # Check custom classification first if [[ -f "$SERVICE_TYPES_CONFIG" ]]; then local custom_type=$(grep "^$service=" "$SERVICE_TYPES_CONFIG" 2>/dev/null | cut -d'=' -f2) if [[ -n "$custom_type" ]]; then echo "$custom_type" return 0 fi fi # Check default classification if [[ -n "${DEFAULT_SERVICE_TYPES[$service]}" ]]; then echo "${DEFAULT_SERVICE_TYPES[$service]}" return 0 fi # Auto-detect service type local service_file="/etc/systemd/system/$service" [[ ! -f "$service_file" ]] && service_file="/usr/lib/systemd/system/$service" [[ ! -f "$service_file" ]] && service_file="/lib/systemd/system/$service" if [[ -f "$service_file" ]]; then # Socket activation detection if grep -q "TriggeredBy=.*\.socket" "$service_file" 2>/dev/null || \ systemctl list-dependencies "$service" 2>/dev/null | grep -q "\.socket"; then echo "socket-activated" return 0 fi # Oneshot detection if grep -q "Type=oneshot" "$service_file" 2>/dev/null; then echo "on-demand" return 0 fi # Static services local enabled_status=$(systemctl is-enabled "$service" 2>/dev/null) if [[ "$enabled_status" == "static" ]]; then echo "on-demand" return 0 fi fi # Default classification echo "standard"
} categorize_all_services() { local -n services_ref=$1 declare -A categorized=( ["critical"]="" ["application"]="" ["socket-activated"]="" ["on-demand"]="" ["optional"]="" ["standard"]="" ) print_status "info" "$(msg "service_classification")" local count=0 local total=${#services_ref[@]} for service in "${services_ref[@]}"; do local category=$(classify_service "$service") categorized["$category"]+="$service " ((count++)) show_progress_bar "$count" "$total" done # Return categorized services for category in critical application socket-activated on-demand optional standard; do if [[ -n "${categorized[$category]}" ]]; then echo "$category:${categorized[$category]}" fi done
} # ==================================================================================
# 🔧 SERVICE HEALTH MONITORING
# ==================================================================================
check_critical_service() { local service="$1" if ! systemctl is-active --quiet "$service" 2>/dev/null; then log_message "ERROR" "Critical service $service is down" print_status "error" "Critical service $service is down, restarting..." if systemctl start "$service" 2>/dev/null; then log_message "INFO" "Critical service $service restarted successfully" print_status "success" "$(msg "service_restarted" "$service")" send_notification "$service" "restarted" "critical" return 0 else log_message "ERROR" "Failed to restart critical service $service" print_status "error" "$(msg "service_failed" "$service")" send_notification "$service" "failed" "critical" return 1 fi fi log_message "DEBUG" "Critical service $service is healthy" return 0
} check_socket_activated_service() { local service="$1" local socket_name="${service%.*}.socket" local socket_status=$(systemctl is-active "$socket_name" 2>/dev/null) local service_status=$(systemctl is-active "$service" 2>/dev/null) # Check if socket exists if ! systemctl list-units --all "$socket_name" &>/dev/null; then log_message "DEBUG" "No socket found for $service, treating as standard service" check_standard_service "$service" return $? fi # Socket should be active if [[ "$socket_status" == "active" ]]; then if [[ "$service_status" == "inactive" ]]; then log_message "DEBUG" "Socket-activated service $service is dormant (normal)" return 0 elif [[ "$service_status" == "active" ]]; then log_message "DEBUG" "Socket-activated service $service is currently active" return 0 fi else log_message "WARNING" "Socket $socket_name is not active for $service" print_status "warning" "Socket $socket_name is down, restarting..." if systemctl start "$socket_name" 2>/dev/null; then log_message "INFO" "Socket $socket_name restarted successfully" print_status "success" "$(msg "socket_restarted" "$socket_name")" send_notification "$socket_name" "socket restarted" "socket" return 0 else log_message "ERROR" "Failed to restart socket $socket_name" print_status "error" "Failed to restart socket $socket_name" send_notification "$socket_name" "socket failed" "socket" return 1 fi fi # Check for abnormal exit status local exit_status=$(systemctl show "$service" --property=ExecMainStatus --value 2>/dev/null) if [[ -n "$exit_status" && "$exit_status" != "0" && "$exit_status" != "42" ]]; then log_message "WARNING" "Socket-activated service $service last exit: $exit_status" # Check recent errors local recent_errors=$(journalctl -u "$service" --since "10 minutes ago" -p err --no-pager -q 2>/dev/null) if [[ -n "$recent_errors" ]]; then log_message "ERROR" "Recent errors in $service: $recent_errors" send_notification "$service" "has errors" "socket" return 1 fi fi return 0
} check_application_service() { local service="$1" if ! systemctl is-active --quiet "$service" 2>/dev/null; then log_message "WARNING" "Application service $service is down" print_status "warning" "Application service $service is down, restarting..." if systemctl start "$service" 2>/dev/null; then log_message "INFO" "Application service $service restarted successfully" print_status "success" "$(msg "service_restarted" "$service")" send_notification "$service" "restarted" "application" return 0 else log_message "ERROR" "Failed to restart application service $service" print_status "error" "$(msg "service_failed" "$service")" send_notification "$service" "failed" "application" return 1 fi fi log_message "DEBUG" "Application service $service is healthy" return 0
} check_on_demand_service() { local service="$1" local service_status=$(systemctl is-active "$service" 2>/dev/null) # On-demand services being inactive is normal if [[ "$service_status" == "inactive" ]]; then local last_exit=$(systemctl show "$service" --property=ExecMainStatus --value 2>/dev/null) if [[ "$last_exit" == "0" || -z "$last_exit" ]]; then log_message "DEBUG" "On-demand service $service is dormant (normal)" return 0 else log_message "WARNING" "On-demand service $service last run failed (exit: $last_exit)" return 1 fi elif [[ "$service_status" == "active" ]]; then log_message "DEBUG" "On-demand service $service is currently running" return 0 else log_message "WARNING" "On-demand service $service status: $service_status" return 1 fi
} check_optional_service() { local service="$1" # Optional services - only log if they fail, don't restart if ! systemctl is-active --quiet "$service" 2>/dev/null; then local failed_status=$(systemctl is-failed "$service" 2>/dev/null) if [[ "$failed_status" == "failed" ]]; then log_message "WARNING" "Optional service $service is in failed state" return 1 else log_message "DEBUG" "Optional service $service is inactive (normal)" return 0 fi fi log_message "DEBUG" "Optional service $service is running" return 0
} check_standard_service() { local service="$1" if ! systemctl is-active --quiet "$service" 2>/dev/null; then log_message "ERROR" "Standard service $service is down" print_status "warning" "Standard service $service is down, restarting..." if systemctl start "$service" 2>/dev/null; then log_message "INFO" "Standard service $service restarted successfully" print_status "success" "$(msg "service_restarted" "$service")" send_notification "$service" "restarted" "standard" return 0 else log_message "ERROR" "Failed to restart standard service $service" print_status "error" "$(msg "service_failed" "$service")" send_notification "$service" "failed" "standard" return 1 fi fi log_message "DEBUG" "Standard service $service is healthy" return 0
} check_service_health() { local service="$1" local service_type="$2" case "$service_type" in "critical") check_critical_service "$service" ;; "socket-activated") check_socket_activated_service "$service" ;; "application") check_application_service "$service" ;; "on-demand") check_on_demand_service "$service" ;; "optional") check_optional_service "$service" ;; *) check_standard_service "$service" ;; esac
} # ==================================================================================
# 📱 NOTIFICATION SYSTEM
# ==================================================================================
send_email_notification() { local subject="$1" local message="$2" local email="$3" local priority="$4" [[ -z "$email" ]] && return 1 # Add priority indicator to subject case "$priority" in "critical") subject="🚨 CRITICAL: $subject" ;; "application") subject="⚠️ APPLICATION: $subject" ;; "socket") subject="🔌 SOCKET: $subject" ;; *) subject="ℹ️ INFO: $subject" ;; esac if command -v sendmail >/dev/null; then local priority_num="3" case "$priority" in "critical") priority_num="1" ;; esac { echo "To: $email" echo "Subject: $subject" echo "From: watchdog@$(hostname)" echo "X-Priority: $priority_num" echo "Content-Type: text/html; charset=UTF-8" echo "" echo "<h3>$subject</h3>" echo "<p><strong>Hostname:</strong> $(hostname)</p>" echo "<p><strong>Time:</strong> $(date)</p>" echo "<p><strong>Message:</strong> $message</p>" echo "<hr>" echo "<small>Generated by Watchdog v$VERSION</small>" } | sendmail "$email" 2>/dev/null elif command -v mail >/dev/null; then echo "$message" | mail -s "$subject" "$email" 2>/dev/null else return 1 fi
} send_webhook_notification() { local message="$1" local webhook_url="$2" local priority="$3" [[ -z "$webhook_url" ]] && return 1 local color="" local emoji="" case "$priority" in "critical") color="15158332"; emoji="🚨" ;; # Red "application") color="16776960"; emoji="⚠️" ;; # Yellow "socket") color="3447003"; emoji="🔌" ;; # Blue *) color="8947848"; emoji="ℹ️" ;; # Gray esac local payload payload=$(cat << EOF
{ "embeds": [{ "title": "$emoji Watchdog Alert", "description": "$message", "color": $color, "fields": [ { "name": "Hostname", "value": "$(hostname)", "inline": true }, { "name": "Time", "value": "$(date)", "inline": true }, { "name": "Priority", "value": "$priority", "inline": true } ], "footer": { "text": "Watchdog v$VERSION" } }]
}
EOF
) if command -v curl >/dev/null; then curl -s -X POST "$webhook_url" \ -H "Content-Type: application/json" \ -d "$payload" >/dev/null 2>&1 fi
} send_telegram_notification() { local message="$1" local bot_token="$2" local chat_id="$3" local priority="$4" [[ -z "$bot_token" || -z "$chat_id" ]] && return 1 local emoji="" case "$priority" in "critical") emoji="🚨" ;; "application") emoji="⚠️" ;; "socket") emoji="🔌" ;; *) emoji="ℹ️" ;; esac local formatted_message="$emoji *Watchdog Alert* *Service:* $message
*Hostname:* $(hostname)
*Time:* $(date)
*Priority:* $priority _Generated by Watchdog v$VERSION_" if command -v curl >/dev/null; then curl -s -X POST \ "https://api.telegram.org/bot$bot_token/sendMessage" \ -d "chat_id=$chat_id" \ -d "text=$formatted_message" \ -d "parse_mode=Markdown" \ >/dev/null 2>&1 fi
} send_notification() { local service="$1" local action="$2" local priority="${3:-standard}" local timestamp="$(date '+%Y-%m-%d %H:%M:%S')" local hostname="$(hostname)" local message="Service '$service' $action on $hostname at $timestamp" # Always log log_message "ALERT" "$message" # Load notification config [[ -f "$CONFIG_FILE" ]] && source "$CONFIG_FILE" [[ -f "$NOTIFICATION_CONFIG" ]] && source "$NOTIFICATION_CONFIG" # Send notifications based on priority and configuration if [[ "$priority" == "critical" || "$NOTIFICATION_LEVEL" == "all" ]]; then [[ -n "$EMAIL" ]] && send_email_notification "$service $action" "$message" "$EMAIL" "$priority" [[ -n "$WEBHOOK_URL" ]] && send_webhook_notification "$message" "$WEBHOOK_URL" "$priority" [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$TELEGRAM_CHAT_ID" ]] && \ send_telegram_notification "$message" "$TELEGRAM_BOT_TOKEN" "$TELEGRAM_CHAT_ID" "$priority" fi # System journal command -v systemd-cat >/dev/null && \ echo "WATCHDOG_ALERT: $message" | systemd-cat -t watchdog -p warning
} # ==================================================================================
# 📊 SYSTEM HEALTH MONITORING
# ==================================================================================
check_system_health() { local issues=() # CPU Load Average local load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d',' -f1 | tr -d ' ') if command -v bc >/dev/null && (( $(echo "$load_avg > 10.0" | bc -l 2>/dev/null) )); then issues+=("High CPU load: $load_avg") log_message "WARNING" "High system load detected: $load_avg" fi # Memory Usage local mem_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}') if (( $(echo "$mem_usage > 90.0" | bc -l 2>/dev/null || echo 0) )); then issues+=("High memory usage: ${mem_usage}%") log_message "WARNING" "High memory usage detected: ${mem_usage}%" fi # Disk Usage local disk_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') if [[ "$disk_usage" -gt 90 ]]; then issues+=("High disk usage: ${disk_usage}%") log_message "WARNING" "High disk usage detected: ${disk_usage}%" fi # Failed systemd units local failed_units=$(systemctl list-units --state=failed --no-pager --no-legend 2>/dev/null | wc -l) if [[ $failed_units -gt 0 ]]; then issues+=("$failed_units failed systemd units") log_message "WARNING" "$failed_units systemd units in failed state" fi # Journal size if command -v journalctl >/dev/null; then local journal_size=$(journalctl --disk-usage 2>/dev/null | awk '{print $7}' | sed 's/[^0-9.]//g') if [[ -n "$journal_size" ]] && (( $(echo "$journal_size > 1000" | bc -l 2>/dev/null || echo 0) )); then issues+=("Large journal size: ${journal_size}MB") log_message "INFO" "Large journal size detected: ${journal_size}MB" fi fi # Send alerts for critical issues if [[ ${#issues[@]} -gt 0 ]]; then for issue in "${issues[@]}"; do if [[ "$issue" =~ ^High ]]; then send_notification "system" "$issue" "critical" fi done return 1 fi log_message "DEBUG" "System health check: OK" return 0
} # ==================================================================================
# 🔄 AUTO-UPDATE SYSTEM
# ==================================================================================
check_for_updates() { local force_check="$1" local now=$(date +%s) local last_check=0 # Check if update check is needed if [[ -f "$UPDATE_CHECK_FILE" && "$force_check" != "force" ]]; then last_check=$(cat "$UPDATE_CHECK_FILE" 2>/dev/null || echo 0) if [[ $((now - last_check)) -lt $UPDATE_CHECK_INTERVAL ]]; then return 0 fi fi print_status "update" "Checking for updates..." # Get latest version from GitHub local latest_version if command -v curl >/dev/null; then latest_version=$(curl -s "$GITHUB_API/releases/latest" 2>/dev/null | \ grep '"tag_name"' | cut -d'"' -f4 | sed 's/^v//') elif command -v wget >/dev/null; then latest_version=$(wget -qO- "$GITHUB_API/releases/latest" 2>/dev/null | \ grep '"tag_name"' | cut -d'"' -f4 | sed 's/^v//') else print_status "warning" "curl or wget not found, cannot check for updates" return 1 fi # Update last check time echo "$now" > "$UPDATE_CHECK_FILE" if [[ -z "$latest_version" ]]; then print_status "warning" "Could not retrieve version information from GitHub" return 1 fi # Version comparison if version_gt "$latest_version" "$VERSION"; then print_status "update" "$(msg "update_available" "$latest_version")" log_message "INFO" "New version available: v$latest_version (current: v$VERSION)" if [[ "$force_check" == "force" ]]; then echo -n "Would you like to update now? [y/N]: " read -r update_choice if [[ "$update_choice" =~ ^[Yy] ]]; then upgrade_watchdog "$latest_version" fi else # Auto-update if enabled source "$CONFIG_FILE" 2>/dev/null if [[ "$AUTO_UPDATE" == "true" ]]; then log_message "INFO" "Auto-updating to v$latest_version" upgrade_watchdog "$latest_version" fi fi else print_status "success" "Watchdog is up to date (v$VERSION)" fi
} version_gt() { # Return success (0) if $1 > $2, otherwise non-zero local a="$1" b="$2" if [[ "$a" == "$b" ]]; then return 1 fi local first first=$(printf '%s\n%s\n' "$a" "$b" | sort -V | head -n1) if [[ "$first" == "$b" ]]; then return 0 fi return 1
} upgrade_watchdog() { local target_version="$1" local backup_file="${SCRIPT_PATH}.backup-$(date +%F-%H%M%S)" local temp_file="/tmp/watchdog_new_$$.sh" print_status "update" "$(msg "update_installing")" log_message "INFO" "Starting upgrade to v$target_version" # Stop running service local was_running=false if is_watchdog_running; then print_status "update" "Stopping Watchdog service..." if command -v systemctl >/dev/null && systemctl is-active --quiet watchdog 2>/dev/null; then sudo systemctl stop watchdog 2>/dev/null else kill "$(cat "$PID_FILE")" 2>/dev/null fi was_running=true sleep 3 fi # Create backup cp "$SCRIPT_PATH" "$backup_file" 2>/dev/null || { print_status "error" "Failed to create backup" return 1 } print_status "update" "Backup created: $backup_file" # Download new version print_status "update" "Downloading v$target_version..." if command -v curl >/dev/null; then curl -s "$GITHUB_RAW/watchdog.sh" -o "$temp_file" elif command -v wget >/dev/null; then wget -q "$GITHUB_RAW/watchdog.sh" -O "$temp_file" else print_status "error" "curl or wget not found" return 1 fi # Verify download if [[ ! -f "$temp_file" || ! -s "$temp_file" ]]; then print_status "error" "Failed to download new version" return 1 fi # Verify it's a valid script if ! bash -n "$temp_file" 2>/dev/null; then print_status "error" "Downloaded file is not a valid bash script" rm -f "$temp_file" return 1 fi # Replace script mv "$temp_file" "$SCRIPT_PATH" chmod +x "$SCRIPT_PATH" # Update systemd service if exists if [[ -f "/etc/systemd/system/watchdog.service" ]]; then print_status "update" "Updating systemd service..." create_systemd_service sudo systemctl daemon-reload fi # Download updated language files for lang in en tr; do if command -v curl >/dev/null; then curl -s "$GITHUB_RAW/lang/$lang.json" -o "$LANG_DIR/$lang.json" 2>/dev/null elif command -v wget >/dev/null; then wget -q "$GITHUB_RAW/lang/$lang.json" -O "$LANG_DIR/$lang.json" 2>/dev/null fi done print_status "success" "$(msg "update_complete")" log_message "INFO" "Upgrade completed successfully to v$target_version" # Restart service if it was running if [[ "$was_running" == "true" ]]; then print_status "update" "Restarting Watchdog service..." if command -v systemctl >/dev/null; then sudo systemctl start watchdog else "$SCRIPT_PATH" --daemon & fi fi # Show new version local new_version=$("$SCRIPT_PATH" --version 2>/dev/null | grep -o 'v[0-9.]*' || echo "v$target_version") print_status "success" "Watchdog updated to $new_version"
} # ==================================================================================
# 🛠️ DEPENDENCY CHECKING
# ==================================================================================
check_dependencies() { local missing=() local optional_missing=() # Critical dependencies command -v systemctl >/dev/null || missing+=("systemd") if ! command -v curl >/dev/null && ! command -v wget >/dev/null; then missing+=("curl or wget") fi # Optional dependencies command -v bc >/dev/null || optional_missing+=("bc (for calculations)") command -v jq >/dev/null || optional_missing+=("jq (for JSON parsing)") if ! command -v sendmail >/dev/null && ! command -v mail >/dev/null; then optional_missing+=("mail system (sendmail/mailutils)") fi if [[ ${#missing[@]} -gt 0 ]]; then print_status "error" "Missing critical dependencies: ${missing[*]}" echo "Install with:" echo " Ubuntu/Debian: sudo apt install curl wget systemd" echo " RHEL/CentOS: sudo yum install curl wget systemd" exit 1 fi if [[ ${#optional_missing[@]} -gt 0 ]]; then print_status "warning" "Optional dependencies missing: ${optional_missing[*]}" echo "Install with:" echo " Ubuntu/Debian: sudo apt install bc jq mailutils" echo " RHEL/CentOS: sudo yum install bc jq mailx" echo fi
} # ==================================================================================
# 📋 SETUP WIZARD
# ==================================================================================
setup_wizard() { show_header # Initialize i18n init_i18n # Language selection echo -e "${COLORS[YELLOW]}$(msg "language_select"):${COLORS[NC]}" echo " [1] English" echo " [2] Türkçe" echo -n "Choice [1-2]: " read -r lang_choice case "$lang_choice" in 2) LANGUAGE="tr" ;; *) LANGUAGE="en" ;; esac load_language "$LANGUAGE" show_header # Check for updates first check_for_updates "force" # Service discovery mapfile -t all_services < <(discover_services) print_status "success" "$(msg "found_services" "${#all_services[@]}")" # Categorize services print_status "info" "$(msg "service_classification")" declare -A service_categories while IFS=':' read -r category services; do service_categories["$category"]="$services" done < <(categorize_all_services all_services) echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}📊 Service Categories:${COLORS[NC]}" echo -e "${COLORS[BLUE]}────────────────────────────────────────${COLORS[NC]}" # Show categorized services declare -A selected_services for category in critical application socket-activated on-demand optional standard; do if [[ -n "${service_categories[$category]}" ]]; then local services_array=(${service_categories[$category]}) local count=${#services_array[@]} case "$category" in "critical") echo -e "${COLORS[RED]}🔴 $(msg "critical_services") ($count):${COLORS[NC]}" selected_services["$category"]="${service_categories[$category]}" ;; "application") echo -e "${COLORS[GREEN]}🟢 $(msg "application_services") ($count):${COLORS[NC]}" ;; "socket-activated") echo -e "${COLORS[BLUE]}🔵 $(msg "socket_services") ($count):${COLORS[NC]}" selected_services["$category"]="${service_categories[$category]}" ;; "on-demand") echo -e "${COLORS[YELLOW]}🟡 On-Demand Services ($count):${COLORS[NC]}" ;; "optional") echo -e "${COLORS[PURPLE]}🟣 $(msg "optional_services") ($count):${COLORS[NC]}" ;; "standard") echo -e "${COLORS[GRAY]}⚪ Standard Services ($count):${COLORS[NC]}" ;; esac # Show first few services local display_count=0 for service in ${services_array[@]}; do if [[ $display_count -lt 5 ]]; then echo " • $service" ((display_count++)) elif [[ $display_count -eq 5 ]]; then echo " • ... and $((count - 5)) more" break fi done echo fi done # Service selection for non-automatic categories for category in application standard; do if [[ -n "${service_categories[$category]}" ]]; then local services_array=(${service_categories[$category]}) echo -e "${COLORS[YELLOW]}Select $category services to monitor:${COLORS[NC]}" echo "Available services:" local i=0 for service in ${services_array[@]}; do printf " [%2d] %s\n" "$i" "$service" ((i++)) done echo -n "Enter numbers (comma separated) or 'all' for all, 'none' for none: " read -r selection if [[ "$selection" == "all" ]]; then selected_services["$category"]="${service_categories[$category]}" elif [[ "$selection" != "none" ]]; then IFS=',' read -ra indices <<< "$selection" local selected="" for idx in "${indices[@]}"; do idx=$(echo "$idx" | tr -d ' ') if [[ "$idx" =~ ^[0-9]+$ ]] && [[ $idx -lt ${#services_array[@]} ]]; then selected+="${services_array[$idx]} " fi done selected_services["$category"]="$selected" fi echo fi done # Email configuration echo -e "${COLORS[YELLOW]}$(msg "notification_setup"):${COLORS[NC]}" echo -n "$(msg "email_prompt"): " read -r EMAIL # Additional notifications local notification_method="" echo echo "Additional notification methods:" echo " [1] Discord/Slack Webhook" echo " [2] Telegram Bot" echo " [3] Skip" echo -n "Choice [1-3]: " read -r notification_choice case "$notification_choice" in 1) echo -n "$(msg "webhook_prompt"): " read -r WEBHOOK_URL notification_method="webhook" ;; 2) echo -n "Telegram Bot Token: " read -r TELEGRAM_BOT_TOKEN echo -n "Telegram Chat ID: " read -r TELEGRAM_CHAT_ID notification_method="telegram" ;; esac # Monitoring configuration echo echo -n "$(msg "interval_prompt") [5]: " read -r INTERVAL [[ "$INTERVAL" =~ ^[0-9]+$ ]] || INTERVAL=5 echo -n "Notification level - all/critical only [critical]: " read -r NOTIFICATION_LEVEL [[ "$NOTIFICATION_LEVEL" == "all" ]] || NOTIFICATION_LEVEL="critical" echo -n "Enable auto-updates? [y/N]: " read -r auto_update_choice AUTO_UPDATE="false" [[ "$auto_update_choice" =~ ^[Yy] ]] && AUTO_UPDATE="true" echo -n "Enable auto-start on boot? [y/N]: " read -r autostart_choice AUTOSTART="false" [[ "$autostart_choice" =~ ^[Yy] ]] && AUTOSTART="true" # Save configuration mkdir -p "$CONFIG_DIR" # Main config { echo "# Watchdog v$VERSION Configuration" echo "# Generated on $(date)" echo echo "VERSION=\"$VERSION\"" echo "LANGUAGE=\"$LANGUAGE\"" echo "INTERVAL=\"$INTERVAL\"" echo "NOTIFICATION_LEVEL=\"$NOTIFICATION_LEVEL\"" echo "AUTO_UPDATE=\"$AUTO_UPDATE\"" echo "AUTOSTART=\"$AUTOSTART\"" echo "EMAIL=\"$EMAIL\"" echo echo "# Service Configuration" for category in critical application socket-activated on-demand optional standard; do if [[ -n "${selected_services[$category]}" ]]; then var_suffix=$(echo "$category" | tr '[:lower:]-' '[:upper:]_') echo "SERVICES_${var_suffix}=(${selected_services[$category]})" fi done } > "$CONFIG_FILE" # Notification config { echo "# Notification Configuration" echo "WEBHOOK_URL=\"$WEBHOOK_URL\"" echo "TELEGRAM_BOT_TOKEN=\"$TELEGRAM_BOT_TOKEN\"" echo "TELEGRAM_CHAT_ID=\"$TELEGRAM_CHAT_ID\"" } > "$NOTIFICATION_CONFIG" # Service types config (for custom overrides) { echo "# Custom Service Type Overrides" echo "# Format: service.service=type" echo "# Types: critical, application, socket-activated, on-demand, optional, standard" } > "$SERVICE_TYPES_CONFIG" # Create systemd service create_systemd_service print_status "success" "$(msg "config_saved")" echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}📋 Configuration Summary:${COLORS[NC]}" echo -e "${COLORS[BLUE]}──────────────────────────────────────────${COLORS[NC]}" local total_services=0 for category in critical application socket-activated; do if [[ -n "${selected_services[$category]}" ]]; then local services_array=(${selected_services[$category]}) local count=${#services_array[@]} total_services=$((total_services + count)) echo " 📦 ${category^} services: $count" fi done echo " 📧 Email: ${EMAIL:-"Not configured"}" echo " 🔔 Notifications: $notification_method" echo " ⏱️ Interval: $INTERVAL minutes" echo " 🔄 Auto-update: $AUTO_UPDATE" echo " 🚀 Auto-start: $AUTOSTART" echo " 📁 Config: $CONFIG_DIR/" echo if [[ "$AUTOSTART" == "true" ]]; then sudo systemctl enable watchdog.service 2>/dev/null print_status "success" "Auto-start enabled" fi echo -e "${COLORS[GREEN]}${COLORS[BOLD]}🎉 Setup Complete!${COLORS[NC]}" echo echo -e "${COLORS[WHITE]}Quick Start Commands:${COLORS[NC]}" echo " 🟢 Start monitoring: ${COLORS[GREEN]}sudo systemctl start watchdog${COLORS[NC]}" echo " 📊 Check status: ${COLORS[BLUE]}$SCRIPT_PATH --status${COLORS[NC]}" echo " 📄 View logs: ${COLORS[YELLOW]}tail -f $LOG_FILE${COLORS[NC]}" echo " 🧪 Test notifications: ${COLORS[PURPLE]}$SCRIPT_PATH --test${COLORS[NC]}" echo " 🔄 Check for updates: ${COLORS[CYAN]}$SCRIPT_PATH --update${COLORS[NC]}"
} # ==================================================================================
# 🎛️ MONITORING DAEMON
# ==================================================================================
monitor_services() { # Load configuration if [[ ! -f "$CONFIG_FILE" ]]; then print_status "error" "Configuration not found. Run setup first." exit 1 fi source "$CONFIG_FILE" load_language "$LANGUAGE" # Create PID file create_pid_file # Setup signal handlers trap 'remove_pid_file; exit 0' EXIT INT TERM log_message "INFO" "Watchdog v$VERSION daemon started" # Collect all services to monitor declare -a all_monitored_services declare -A service_type_map for category in critical application socket-activated standard; do local var_suffix var_suffix=$(echo "$category" | tr '[:lower:]-' '[:upper:]_') local var_name="SERVICES_${var_suffix}" local services_var="${!var_name}" if [[ -n "$services_var" ]]; then for service in $services_var; do all_monitored_services+=("$service") service_type_map["$service"]="$category" done fi done log_message "INFO" "Monitoring ${#all_monitored_services[@]} services: ${all_monitored_services[*]}" print_status "success" "$(msg "monitoring_active" ${#all_monitored_services[@]})" local check_counter=0 local update_check_counter=0 local health_check_counter=0 # Main monitoring loop while true; do local start_time=$(date +%s) # Monitor services for service in "${all_monitored_services[@]}"; do local service_type="${service_type_map[$service]}" check_service_health "$service" "$service_type" done # System health check every 5 cycles ((health_check_counter++)) if [[ $((health_check_counter % 5)) -eq 0 ]]; then check_system_health health_check_counter=0 fi # Update check (daily) ((update_check_counter++)) local checks_per_day=$((1440 / INTERVAL)) if [[ $update_check_counter -ge $checks_per_day ]] && [[ "$AUTO_UPDATE" == "true" ]]; then check_for_updates update_check_counter=0 fi # Log periodic status ((check_counter++)) if [[ $((check_counter % 12)) -eq 0 ]]; then # Every hour (if interval=5min) log_message "INFO" "Monitoring cycle #$check_counter completed - ${#all_monitored_services[@]} services checked" print_status "debug" "Monitoring cycle #$check_counter - all services checked" fi # Sleep for interval local end_time=$(date +%s) local elapsed=$((end_time - start_time)) local sleep_time=$((INTERVAL * 60 - elapsed)) if [[ $sleep_time -gt 0 ]]; then sleep "$sleep_time" fi done
} # ==================================================================================
# 🏗️ SYSTEMD SERVICE CREATION
# ==================================================================================
create_systemd_service() { local service_content="[Unit]
Description=Watchdog Service Monitor v$VERSION
Documentation=https://github.com/$GITHUB_REPO
After=network-online.target multi-user.target
Wants=network-online.target [Service]
Type=simple
ExecStart=$SCRIPT_PATH --daemon
ExecReload=/bin/kill -HUP \$MAINPID
KillMode=process
Restart=always
RestartSec=30
User=$USER
Group=$(id -gn)
Environment=HOME=$HOME
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
WorkingDirectory=$SCRIPT_DIR
PIDFile=$PID_FILE
StandardOutput=journal
StandardError=journal [Install]
WantedBy=multi-user.target" echo "$service_content" | sudo tee /etc/systemd/system/watchdog.service >/dev/null sudo systemctl daemon-reload log_message "INFO" "Systemd service created/updated"
} # ==================================================================================
# 🧪 TEST FUNCTIONS
# ==================================================================================
test_notifications() { print_status "info" "Testing notification systems..." [[ -f "$CONFIG_FILE" ]] && source "$CONFIG_FILE" [[ -f "$NOTIFICATION_CONFIG" ]] && source "$NOTIFICATION_CONFIG" load_language "$LANGUAGE" send_notification "test-service" "notification test completed" "application" print_status "success" "Test notifications sent! Check your configured channels."
} test_service_classification() { print_status "info" "Testing service classification..." local services=("ssh.service" "snapd.service" "packagekit.service" "nginx.service" "cron.service") for service in "${services[@]}"; do local classification=$(classify_service "$service") print_status "info" "$service -> $classification" done
} # ==================================================================================
# 📊 STATUS AND ANALYTICS
# ==================================================================================
show_status() { show_header if is_watchdog_running; then local pid=$(cat "$PID_FILE") print_status "success" "Watchdog is running (PID: $pid)" if [[ -f "$CONFIG_FILE" ]]; then source "$CONFIG_FILE" load_language "$LANGUAGE" echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}📊 Current Configuration:${COLORS[NC]}" echo -e "${COLORS[BLUE]}────────────────────────────────${COLORS[NC]}" # Count monitored services local total_services=0 for category in critical application socket-activated standard; do local var_suffix var_suffix=$(echo "$category" | tr '[:lower:]-' '[:upper:]_') local var_name="SERVICES_${var_suffix}" local services_var="${!var_name}" if [[ -n "$services_var" ]]; then local services_array=($services_var) local count=${#services_array[@]} total_services=$((total_services + count)) echo " 📦 ${category^}: $count services" fi done echo " 📧 Email: ${EMAIL:-"Not configured"}" echo " ⏱️ Interval: $INTERVAL minutes" echo " 🌐 Language: $LANGUAGE" echo " 🔄 Auto-update: $AUTO_UPDATE" echo " 📄 Log file: $LOG_FILE" # System health echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}🖥️ System Health:${COLORS[NC]}" echo -e "${COLORS[BLUE]}─────────────────${COLORS[NC]}" local load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d',' -f1 | tr -d ' ') local disk_usage=$(df / | awk 'NR==2 {print $5}') local mem_usage=$(free | awk 'NR==2{printf "%.1f%%", $3*100/$2}') local uptime_info=$(uptime -p 2>/dev/null || uptime | awk -F'up ' '{print $2}' | awk -F',' '{print $1}') echo " 📈 Load Average: $load_avg" echo " 💾 Disk Usage: $disk_usage" echo " 🧠 Memory Usage: $mem_usage" echo " ⏰ System Uptime: $uptime_info" # Recent activity if [[ -f "$LOG_FILE" ]]; then echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}📜 Recent Activity (last 5 entries):${COLORS[NC]}" echo -e "${COLORS[BLUE]}──────────────────────────────────────────${COLORS[NC]}" tail -n 5 "$LOG_FILE" 2>/dev/null | while read -r line; do echo " ${COLORS[GRAY]}$line${COLORS[NC]}" done fi fi else print_status "error" "Watchdog is not running" echo echo -e "${COLORS[YELLOW]}Start with:${COLORS[NC]}" echo " ${COLORS[GREEN]}sudo systemctl start watchdog${COLORS[NC]}" echo " ${COLORS[BLUE]}$SCRIPT_PATH --daemon${COLORS[NC]}" fi
} # ==================================================================================
# 📈 LOG ANALYTICS
# ==================================================================================
analyze_logs() { if [[ ! -f "$LOG_FILE" ]]; then print_status "error" "Log file not found: $LOG_FILE" return 1 fi show_header echo -e "${COLORS[CYAN]}${COLORS[BOLD]}📊 Watchdog Log Analytics${COLORS[NC]}" echo -e "${COLORS[BLUE]}═══════════════════════════════════════════${COLORS[NC]}" # Basic statistics local total_lines=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0) local today=$(date +%Y-%m-%d) local today_lines=$(grep "^$today" "$LOG_FILE" 2>/dev/null | wc -l || echo 0) echo -e "\n${COLORS[WHITE]}📋 General Statistics:${COLORS[NC]}" echo " 📄 Total log entries: $total_lines" echo " 📅 Today's entries: $today_lines" echo " 📁 Log file size: $(du -h "$LOG_FILE" 2>/dev/null | cut -f1 || echo "0B")" # Service restart statistics echo -e "\n${COLORS[WHITE]}🔄 Service Restart Statistics:${COLORS[NC]}" local total_restarts=$(grep -c "restarted successfully\|Service.*restart" "$LOG_FILE" 2>/dev/null || echo 0) local failed_restarts=$(grep -c "Failed to restart\|failed to restart" "$LOG_FILE" 2>/dev/null || echo 0) echo " ✅ Successful restarts: $total_restarts" echo " ❌ Failed restarts: $failed_restarts" if [[ $total_restarts -gt 0 ]]; then echo -e "\n${COLORS[WHITE]}🏆 Most Restarted Services:${COLORS[NC]}" grep "restarted successfully\|Service.*restart" "$LOG_FILE" 2>/dev/null | \ sed -n 's/.*Service \([^ ]*\) restart.*/\1/p' | \ sort | uniq -c | sort -nr | head -5 | \ while read -r count service; do echo " 🔄 $service: $count times" done fi # Error analysis echo -e "\n${COLORS[WHITE]}⚠️ Error Analysis:${COLORS[NC]}" local error_count=$(grep -c "\[ERROR\]\|\[WARNING\]" "$LOG_FILE" 2>/dev/null || echo 0) local critical_count=$(grep -c "\[ALERT\].*critical" "$LOG_FILE" 2>/dev/null || echo 0) echo " ⚠️ Total warnings/errors: $error_count" echo " 🚨 Critical alerts: $critical_count" # System health alerts echo -e "\n${COLORS[WHITE]}🖥️ System Health Alerts:${COLORS[NC]}" local load_alerts=$(grep -c "High.*load\|high load" "$LOG_FILE" 2>/dev/null || echo 0) local memory_alerts=$(grep -c "High.*memory\|high memory" "$LOG_FILE" 2>/dev/null || echo 0) local disk_alerts=$(grep -c "High.*disk\|high disk" "$LOG_FILE" 2>/dev/null || echo 0) echo " 📈 High load alerts: $load_alerts" echo " 🧠 High memory alerts: $memory_alerts" echo " 💾 High disk alerts: $disk_alerts" # Recent activity timeline echo -e "\n${COLORS[WHITE]}⏰ Recent Activity (Last 24 Hours):${COLORS[NC]}" local yesterday=$(date -d "yesterday" +%Y-%m-%d 2>/dev/null || date -v-1d +%Y-%m-%d 2>/dev/null) grep -E "($yesterday|$today)" "$LOG_FILE" 2>/dev/null | tail -10 | \ while read -r line; do if [[ "$line" =~ ERROR|ALERT ]]; then echo " ${COLORS[RED]}$line${COLORS[NC]}" elif [[ "$line" =~ WARNING ]]; then echo " ${COLORS[YELLOW]}$line${COLORS[NC]}" else echo " ${COLORS[GRAY]}$line${COLORS[NC]}" fi done # Performance metrics echo -e "\n${COLORS[WHITE]}⚡ Performance Metrics:${COLORS[NC]}" if command -v awk >/dev/null; then local avg_cycle_time=$(grep "Monitoring cycle" "$LOG_FILE" 2>/dev/null | \ awk '{print $1}' | xargs -I {} date -d {} +%s 2>/dev/null | \ awk 'NR>1{sum+=$1-prev} {prev=$1} END{if(NR>1) print int(sum/(NR-1)); else print 0}') if [[ -n "$avg_cycle_time" && "$avg_cycle_time" -gt 0 ]]; then echo " ⏱️ Average cycle time: ${avg_cycle_time}s" fi fi # Suggestions echo -e "\n${COLORS[WHITE]}💡 Suggestions:${COLORS[NC]}" if [[ $failed_restarts -gt 5 ]]; then echo " ⚠️ Consider investigating services with frequent restart failures" fi if [[ $total_restarts -gt 20 ]]; then echo " 🔍 High restart frequency detected - check system stability" fi if [[ $error_count -gt 50 ]]; then echo " 📝 Consider log rotation or investigating recurring errors" fi if [[ $critical_count -eq 0 && $total_restarts -eq 0 ]]; then echo " ✅ System appears stable - no critical issues detected" fi
} # ==================================================================================
# 🔧 MANAGEMENT FUNCTIONS
# ==================================================================================
backup_configuration() { local backup_dir="$HOME/.watchdog_backup_$(date +%F-%H%M%S)" mkdir -p "$backup_dir" # Copy configuration files [[ -f "$CONFIG_FILE" ]] && cp "$CONFIG_FILE" "$backup_dir/" [[ -f "$NOTIFICATION_CONFIG" ]] && cp "$NOTIFICATION_CONFIG" "$backup_dir/" [[ -f "$SERVICE_TYPES_CONFIG" ]] && cp "$SERVICE_TYPES_CONFIG" "$backup_dir/" [[ -f "$LOG_FILE" ]] && cp "$LOG_FILE" "$backup_dir/" [[ -d "$LANG_DIR" ]] && cp -r "$LANG_DIR" "$backup_dir/" # Copy systemd service [[ -f "/etc/systemd/system/watchdog.service" ]] && \ sudo cp "/etc/systemd/system/watchdog.service" "$backup_dir/" 2>/dev/null print_status "success" "Configuration backed up to: $backup_dir" echo " 📦 Files backed up:" find "$backup_dir" -type f -exec basename {} \; | sed 's/^/ • /'
} restore_configuration() { local backup_dir="$1" if [[ ! -d "$backup_dir" ]]; then print_status "error" "Backup directory not found: $backup_dir" return 1 fi print_status "info" "Restoring configuration from: $backup_dir" # Stop service if running local was_running=false if is_watchdog_running; then print_status "info" "Stopping Watchdog service..." sudo systemctl stop watchdog 2>/dev/null || kill "$(cat "$PID_FILE")" 2>/dev/null was_running=true fi # Restore files mkdir -p "$CONFIG_DIR" [[ -f "$backup_dir/$(basename "$CONFIG_FILE")" ]] && cp "$backup_dir/$(basename "$CONFIG_FILE")" "$CONFIG_FILE" [[ -f "$backup_dir/$(basename "$NOTIFICATION_CONFIG")" ]] && cp "$backup_dir/$(basename "$NOTIFICATION_CONFIG")" "$NOTIFICATION_CONFIG" [[ -f "$backup_dir/$(basename "$SERVICE_TYPES_CONFIG")" ]] && cp "$backup_dir/$(basename "$SERVICE_TYPES_CONFIG")" "$SERVICE_TYPES_CONFIG" [[ -f "$backup_dir/$(basename "$LOG_FILE")" ]] && cp "$backup_dir/$(basename "$LOG_FILE")" "$LOG_FILE" [[ -d "$backup_dir/lang" ]] && cp -r "$backup_dir/lang" "$CONFIG_DIR/" # Restore systemd service [[ -f "$backup_dir/watchdog.service" ]] && \ sudo cp "$backup_dir/watchdog.service" "/etc/systemd/system/" 2>/dev/null && \ sudo systemctl daemon-reload print_status "success" "Configuration restored successfully" # Restart service if it was running if [[ "$was_running" == "true" ]]; then print_status "info" "Restarting Watchdog service..." sudo systemctl start watchdog fi
} uninstall_watchdog() { print_status "warning" "Starting Watchdog uninstallation..." echo -n "Are you sure you want to completely remove Watchdog? [y/N]: " read -r confirm if [[ ! "$confirm" =~ ^[Yy] ]]; then print_status "info" "Uninstallation cancelled" return 0 fi # Stop and disable service if systemctl is-active --quiet watchdog 2>/dev/null; then print_status "info" "Stopping Watchdog service..." sudo systemctl stop watchdog fi if systemctl is-enabled --quiet watchdog 2>/dev/null; then print_status "info" "Disabling Watchdog service..." sudo systemctl disable watchdog fi # Remove systemd service if [[ -f "/etc/systemd/system/watchdog.service" ]]; then sudo rm -f "/etc/systemd/system/watchdog.service" sudo systemctl daemon-reload print_status "success" "Systemd service removed" fi # Create backup before removing backup_configuration # Remove configuration if [[ -d "$CONFIG_DIR" ]]; then rm -rf "$CONFIG_DIR" print_status "success" "Configuration directory removed" fi print_status "success" "Watchdog uninstalled successfully" echo echo "Note: The script file ($SCRIPT_PATH) was not removed." echo "Remove it manually if no longer needed."
} # ==================================================================================
# 📋 HELP SYSTEM
# ==================================================================================
show_help() { show_header echo -e "${COLORS[WHITE]}${COLORS[BOLD]}🐕 WATCHDOG v$VERSION - Usage Guide${COLORS[NC]}" echo -e "${COLORS[BLUE]}════════════════════════════════════════════════${COLORS[NC]}" echo echo -e "${COLORS[YELLOW]}BASIC USAGE:${COLORS[NC]}" echo " $SCRIPT_NAME Run interactive setup wizard" echo " $SCRIPT_NAME --daemon Start monitoring daemon" echo " $SCRIPT_NAME --status Show current status" echo " $SCRIPT_NAME --stop Stop running daemon" echo echo -e "${COLORS[YELLOW]}TESTING & DIAGNOSTICS:${COLORS[NC]}" echo " $SCRIPT_NAME --test Test notification systems" echo " $SCRIPT_NAME --test-classify Test service classification" echo " $SCRIPT_NAME --logs Analyze log files" echo " $SCRIPT_NAME --health Show system health" echo echo -e "${COLORS[YELLOW]}UPDATES & MAINTENANCE:${COLORS[NC]}" echo " $SCRIPT_NAME --update Check for and install updates" echo " $SCRIPT_NAME --backup Backup configuration" echo " $SCRIPT_NAME --restore DIR Restore from backup" echo " $SCRIPT_NAME --uninstall Complete removal" echo echo -e "${COLORS[YELLOW]}SYSTEMD INTEGRATION:${COLORS[NC]}" echo " sudo systemctl start watchdog Start service" echo " sudo systemctl stop watchdog Stop service" echo " sudo systemctl enable watchdog Enable auto-start" echo " sudo systemctl status watchdog Show service status" echo echo -e "${COLORS[YELLOW]}INFORMATION:${COLORS[NC]}" echo " $SCRIPT_NAME --version Show version" echo " $SCRIPT_NAME --help Show this help" echo echo -e "${COLORS[CYAN]}${COLORS[BOLD]}🌟 FEATURES:${COLORS[NC]}" echo " ✅ Intelligent service classification" echo " ✅ Socket-activated service support" echo " ✅ Multi-channel notifications" echo " ✅ Auto-update system" echo " ✅ System health monitoring" echo " ✅ Comprehensive logging" echo " ✅ Multi-language support" echo echo -e "${COLORS[WHITE]}Configuration files:${COLORS[NC]}" echo " 📁 $CONFIG_DIR/" echo " 📄 Log file: $LOG_FILE" echo echo -e "${COLORS[GRAY]}For more information, visit: https://github.com/$GITHUB_REPO${COLORS[NC]}"
} # ==================================================================================
# 🚀 MAIN PROGRAM ENTRY POINT
# ==================================================================================
main() { # Create config directory mkdir -p "$CONFIG_DIR" case "${1:-}" in "--daemon"|"-d") monitor_services ;; "--test"|"-t") test_notifications ;; "--test-classify") test_service_classification ;; "--status"|"-s") show_status ;; "--update"|"-u"|"--upgrade") check_for_updates "force" ;; "--logs"|"-l") analyze_logs ;; "--health") check_system_health if [[ $? -eq 0 ]]; then print_status "success" "$(msg "system_health_ok")" else print_status "warning" "System health issues detected" fi ;; "--backup") backup_configuration ;; "--restore") restore_configuration "$2" ;; "--uninstall") uninstall_watchdog ;; "--stop") if is_watchdog_running; then local pid=$(cat "$PID_FILE") kill "$pid" 2>/dev/null remove_pid_file print_status "success" "$(msg "service_stopped")" # Also stop systemd service if running if systemctl is-active --quiet watchdog 2>/dev/null; then sudo systemctl stop watchdog fi else print_status "info" "Watchdog is not running" fi ;; "--version"|"-v") echo "Watchdog v$VERSION" echo "GitHub: https://github.com/$GITHUB_REPO" ;; "--help"|"-h"|"help") show_help ;; *) # Setup wizard mode check_dependencies if is_watchdog_running; then show_status echo print_status "warning" "Watchdog is already running." echo " 🛑 Stop with: ${COLORS[YELLOW]}$SCRIPT_PATH --stop${COLORS[NC]}" echo " 📊 Status: ${COLORS[BLUE]}$SCRIPT_PATH --status${COLORS[NC]}" echo " ⚙️ Reconfigure: ${COLORS[GREEN]}$SCRIPT_PATH --stop && $SCRIPT_PATH${COLORS[NC]}" exit 1 fi setup_wizard ;; esac
} # ==================================================================================
# 🎬 SCRIPT EXECUTION
# ==================================================================================
# Initialize and run
main "$@"