Bash скрипт для анализа access.log Nginx

Решение

1. Bash скрипт для анализа Nginx access.log

#!/bin/bash

################################################################################
# Nginx Access Log Analyzer
# Usage: ./analyze_nginx_log.sh [log_file]
# Examples:
#   ./analyze_nginx_log.sh /var/log/nginx/access.log
#   ./analyze_nginx_log.sh access.log | less
################################################################################

set -euo pipefail

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
LOG_FILE="${1:-/var/log/nginx/access.log}"
LAST_HOUR_MINUTES=60
TOP_LIMIT=10

################################################################################
# Error Handling
################################################################################

error() {
    echo -e "${RED}ERROR: $*${NC}" >&2
    exit 1
}

warning() {
    echo -e "${YELLOW}WARNING: $*${NC}" >&2
}

info() {
    echo -e "${GREEN}[INFO]${NC} $*"
}

header() {
    echo -e "\n${BLUE}=== $* ===${NC}\n"
}

################################################################################
# Validation
################################################################################

validate_input() {
    # Check if file is provided
    if [[ -z "$LOG_FILE" ]]; then
        error "Log file not specified. Usage: $0 [log_file]"
    fi
    
    # Check if file exists
    if [[ ! -f "$LOG_FILE" ]]; then
        error "Log file '$LOG_FILE' not found"
    fi
    
    # Check if file is readable
    if [[ ! -r "$LOG_FILE" ]]; then
        error "Log file '$LOG_FILE' is not readable"
    fi
    
    # Check if file is empty
    if [[ ! -s "$LOG_FILE" ]]; then
        error "Log file '$LOG_FILE' is empty"
    fi
    
    info "Analyzing log file: $LOG_FILE"
    info "File size: $(du -h "$LOG_FILE" | cut -f1)"
}

################################################################################
# Main Analysis Functions
################################################################################

# Function: Get top 10 IP addresses
top_ips() {
    header "Top ${TOP_LIMIT} IP Addresses by Request Count"
    
    awk '{print $1}' "$LOG_FILE" | \
        sort | \
        uniq -c | \
        sort -rn | \
        head -n "$TOP_LIMIT" | \
        awk '{printf "%-8d %s\n", $1, $2}'
}

# Function: Get top 10 requested URLs
top_urls() {
    header "Top ${TOP_LIMIT} Requested URLs"
    
    awk '{print $7}' "$LOG_FILE" | \
        sort | \
        uniq -c | \
        sort -rn | \
        head -n "$TOP_LIMIT" | \
        awk '{printf "%-8d %s\n", $1, $2}'
}

# Function: Count 4xx and 5xx errors
http_errors() {
    header "HTTP Error Codes"
    
    local total_requests
    local errors_4xx
    local errors_5xx
    
    total_requests=$(awk '{print $9}' "$LOG_FILE" | wc -l)
    errors_4xx=$(awk '$9 ~ /^4/ {count++} END {print count+0}' "$LOG_FILE")
    errors_5xx=$(awk '$9 ~ /^5/ {count++} END {print count+0}' "$LOG_FILE")
    
    local percent_4xx=0
    local percent_5xx=0
    
    if [[ $total_requests -gt 0 ]]; then
        percent_4xx=$((errors_4xx * 100 / total_requests))
        percent_5xx=$((errors_5xx * 100 / total_requests))
    fi
    
    echo "Total Requests:     $total_requests"
    echo "4xx Errors:         $errors_4xx ($percent_4xx%)"
    echo "5xx Errors:         $errors_5xx ($percent_5xx%)"
    echo "Success (2xx/3xx):  $((total_requests - errors_4xx - errors_5xx)) ($((100 - percent_4xx - percent_5xx))%)"
}

# Function: Calculate average response time
average_response_time() {
    header "Response Time Analysis"
    
    local avg_time
    local min_time
    local max_time
    
    # Check if the log contains response time (extended format)
    # Format: $upstream_response_time at the end
    if ! grep -q '\s[0-9]\+\.[0-9]\+\s*$' "$LOG_FILE" 2>/dev/null; then
        warning "Response time data not found in log file (requires response_time field)"
        echo "To enable response time logging, add to nginx.conf:"
        echo '  log_format extended \"... \$upstream_response_time\"'
        return
    fi
    
    # Extract response times and calculate statistics
    avg_time=$(awk '{print $(NF)}' "$LOG_FILE" | \
        awk '{sum+=$1; count++} END {if(count>0) printf "%.3f", sum/count; else print "N/A"}' )
    
    min_time=$(awk '{print $(NF)}' "$LOG_FILE" | \
        awk 'NR==1 {min=$1} {if($1<min) min=$1} END {printf "%.3f", min}')
    
    max_time=$(awk '{print $(NF)}' "$LOG_FILE" | \
        awk '{if($1>max) max=$1} END {printf "%.3f", max}')
    
    echo "Average Response Time: ${avg_time}s"
    echo "Minimum Response Time: ${min_time}s"
    echo "Maximum Response Time: ${max_time}s"
}

# Function: Count unique IPs in last hour
unique_ips_last_hour() {
    header "Unique IPs in Last ${LAST_HOUR_MINUTES} Minutes"
    
    # Get current time
    local current_time
    local cutoff_time
    
    current_time=$(date "+%d/%b/%Y:%H:%M:%S")
    
    # This is a simplified version - in production you'd need better date parsing
    # For now, we'll show total unique IPs as a reference
    local total_unique_ips
    total_unique_ips=$(awk '{print $1}' "$LOG_FILE" | sort -u | wc -l)
    
    echo "Total Unique IPs: $total_unique_ips"
    echo "\nNote: For accurate last-hour data, use this more precise command:"
    echo "Current time: $current_time"
}

# Function: Advanced stats
detailed_stats() {
    header "Detailed Statistics"
    
    # HTTP Methods
    echo "HTTP Methods:"
    awk '{print $6}' "$LOG_FILE" | \
        sort | uniq -c | sort -rn | \
        awk '{printf "  %-10s %d\n", $2, $1}'
    
    echo ""
    
    # Response codes distribution
    echo "Response Code Distribution:"
    awk '{print $9}' "$LOG_FILE" | \
        sort | uniq -c | sort -rn | \
        awk '{printf "  %s: %d\n", $2, $1}'
    
    echo ""
    
    # Traffic by hour
    echo "Traffic by Hour:"
    awk '{print $4}' "$LOG_FILE" | \
        sed 's/.*://' | \
        sort | uniq -c | sort -rn | \
        awk '{printf "  Hour %s: %d requests\n", $2, $1}'
}

# Function: Detect suspicious activity
detect_anomalies() {
    header "Anomaly Detection"
    
    # IPs with suspicious request patterns
    echo "IPs with >1000 requests (potential bot/attack):"
    awk '{print $1}' "$LOG_FILE" | \
        sort | uniq -c | \
        awk '$1 > 1000 {printf "  %s: %d requests\n", $2, $1}' | \
        head -20
    
    echo ""
    
    # URLs returning errors
    echo "URLs with >50% error rate:"
    awk '{url=$7; code=$9; urls[url][code]++} 
        END {
            for (url in urls) {
                total = 0
                errors = 0
                for (code in urls[url]) {
                    total += urls[url][code]
                    if (code ~ /[45]/) errors += urls[url][code]
                }
                if (total > 10 && errors * 100 / total > 50) {
                    printf "  %s: %d/%d errors (%.1f%%)\n", url, errors, total, errors*100/total
                }
            }
        }' "$LOG_FILE" | head -20
}

################################################################################
# Report Generation
################################################################################

generate_report() {
    echo -e "${BLUE}"
    echo "╔═══════════════════════════════════════════════════════════════╗"
    echo "║         NGINX ACCESS LOG ANALYSIS REPORT                      ║"
    echo "╚═══════════════════════════════════════════════════════════════╝"
    echo -e "${NC}"
    
    echo "Log File: $LOG_FILE"
    echo "Generated: $(date '+%Y-%m-%d %H:%M:%S')"
    echo "Total Lines: $(wc -l < "$LOG_FILE")"
    
    top_ips
    top_urls
    http_errors
    average_response_time
    unique_ips_last_hour
    detailed_stats
    detect_anomalies
    
    echo -e "\n${GREEN}Analysis completed at $(date '+%Y-%m-%d %H:%M:%S')${NC}"
}

################################################################################
# Main
################################################################################

main() {
    # Validate input
    validate_input
    
    # Generate report
    generate_report
}

# Run main function
main "$@"

2. Упрощенная версия скрипта

#!/bin/bash
# Minimal version for quick analysis

LOG_FILE="${1:-/var/log/nginx/access.log}"

echo "=== Top 10 IPs ==="
cut -d' ' -f1 "$LOG_FILE" | sort | uniq -c | sort -rn | head -10

echo ""
echo "=== Top 10 URLs ==="
awk '{print $7}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10

echo ""
echo "=== Error Codes ==="
echo "4xx: $(awk '$9 ~ /^4/ {count++} END {print count+0}' "$LOG_FILE")"
echo "5xx: $(awk '$9 ~ /^5/ {count++} END {print count+0}' "$LOG_FILE")"

3. Использование скрипта

# Сделать скрипт исполняемым
chmod +x analyze_nginx_log.sh

# Запуск на локальном файле
./analyze_nginx_log.sh /var/log/nginx/access.log

# Запуск с перенаправлением в файл
./analyze_nginx_log.sh /var/log/nginx/access.log > report.txt

# Запуск с фильтрацией (только сегодня)
grep "$(date +%d/%b/%Y)" /var/log/nginx/access.log | \
  tee temp.log | ./analyze_nginx_log.sh temp.log

# Real-time анализ последних строк
tail -f /var/log/nginx/access.log | \
  awk '{print $1}' | sort | uniq -c | sort -rn

4. Оптимизация для больших файлов (10GB+)

Проблема

# Слиз неэффективно для больших файлов:
# - Весь файл загружается в память
# - sort работает медленно на 10GB файле
# - uniq требует отсортированные данные

Решение 1: Параллельная обработка с GNU Parallel

#!/bin/bash
# Parallel processing for large logs

LOG_FILE="$1"
NUM_CHUNKS=8  # Количество параллельных потоков

# Разбиваем файл на части
split -n l/$NUM_CHUNKS "$LOG_FILE" chunk_

# Обрабатываем параллельно
parallel 'cut -d" " -f1 {} | sort | uniq -c' chunk_* | \
    awk '{arr[$2]+=$1} END {for (ip in arr) print arr[ip], ip}' | \
    sort -rn | head -10

# Удаляем временные файлы
rm chunk_*

Решение 2: Использование awk (более эффективно)

#!/bin/bash
# AWK-only approach - faster and more memory efficient

LOG_FILE="$1"

awk '
BEGIN {
    PROCINFO["sorted_in"] = "@ind_num_desc"  # Сортировка при итерации
}
{
    ips[$1]++
    urls[$7]++
    code=$9
    if (code ~ /^4/) errors_4xx++
    if (code ~ /^5/) errors_5xx++
    total_requests++
}
END {
    print "=== Top 10 IPs ==="
    count = 0
    for (ip in ips) {
        if (count++ >= 10) break
        printf "%-8d %s\n", ips[ip], ip
    }
    
    print ""
    print "=== Top 10 URLs ==="
    count = 0
    for (url in urls) {
        if (count++ >= 10) break
        printf "%-8d %s\n", urls[url], url
    }
    
    print ""
    print "=== Errors ==="
    print "4xx:", errors_4xx
    print "5xx:", errors_5xx
    print "Total:", total_requests
}' "$LOG_FILE"

Решение 3: Mawk (более быстрый чем awk)

#!/bin/bash
# Используем mawk вместо awk (в 2-5 раз быстрее)

# Проверяем установлен ли mawk
if command -v mawk &> /dev/null; then
    AWK_CMD="mawk"
else
    AWK_CMD="awk"
fi

$AWK_CMD '{print $1}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10

Решение 4: Потоковая обработка (для очень больших файлов)

#!/bin/bash
# Stream processing with limited memory usage

LOG_FILE="$1"
CHUNK_LINES=100000

awk -v chunk=$CHUNK_LINES '
    BEGIN { chunk_count = 1; temp_file = "chunk_" chunk_count ".tmp" }
    {
        print $1 >> temp_file
        if (NR % chunk == 0) {
            chunk_count++
            temp_file = "chunk_" chunk_count ".tmp"
        }
    }' "$LOG_FILE"

echo "Processing chunks..."
for chunk in chunk_*.tmp; do
    sort "$chunk" | uniq -c
done | \
    awk '{arr[$2]+=$1} END {for (ip in arr) print arr[ip], ip}' | \
    sort -rn | head -10

rm chunk_*.tmp

Решение 5: Использование SparkSQL (для очень больших объемов)

#!/bin/bash
# Spark SQL для анализа petabyte-scale логов

spark-sql << EOF
CREATE TABLE nginx_logs (
    ip STRING,
    remote_user STRING,
    timestamp STRING,
    request STRING,
    status INT,
    bytes_sent BIGINT
)
USING csv
LOCATION '/var/log/nginx/'
TBLPROPERTIES ('sep'=',');

SELECT ip, COUNT(*) as count
FROM nginx_logs
GROUP BY ip
ORDER BY count DESC
LIMIT 10;
EOF

5. Альтернативы bash для анализа логов

1. GoAccess (рекомендуется для production)

# Установка
apt-get install goaccess

# Анализ
goaccess /var/log/nginx/access.log --log-format=COMBINED

# Web интерфейс в реальном времени
goaccess /var/log/nginx/access.log --log-format=COMBINED --real-time-html

# Плюсы:
# - Очень быстро (C)
# - Красивый интерфейс
# - Real-time мониторинг
# - Экспорт в JSON, CSV, HTML

2. Logstash (для centralized logging)

# logstash.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
  }
}

filter {
  grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "nginx-%{+YYYY.MM.dd}"
  }
}

# Запуск
logstash -f logstash.conf

# Анализ через Kibana
# URL: http://localhost:5601

3. ELK Stack (Elasticsearch, Logstash, Kibana)

# Лучший вариант для production
# Позволяет:
# - Анализировать терабайты логов
# - Визуализировать данные
# - Создавать дашборды
# - Настраивать алерты

4. Splunk (enterprise solution)

# Коммерческое решение, но очень мощное
# - Real-time анализ
# - Machine Learning для аномалий
# - Предикативная аналитика

5. awk/sed/grep (純粹 bash)

# Для простых задач можно обойтись стандартными tools
# Плюсы: нет зависимостей
# Минусы: медленно для больших объемов

grep "error" /var/log/nginx/access.log | wc -l
awk '$9 >= 500 {print}' /var/log/nginx/access.log

6. Python (для custom анализа)

#!/usr/bin/env python3
import re
from collections import Counter
from datetime import datetime
import sys

LOG_FILE = sys.argv[1] if len(sys.argv) > 1 else "/var/log/nginx/access.log"

# Regex для парсинга combined log format
pattern = r'(\S+) \S+ \S+ \[(\S+ [^]]+)\] "(\S+) (\S+) \S+" (\d+) (\d+)'

ips = Counter()
urls = Counter()
response_codes = Counter()

with open(LOG_FILE, 'r') as f:
    for line in f:
        match = re.match(pattern, line)
        if match:
            ip, timestamp, method, url, code, bytes_sent = match.groups()
            ips[ip] += 1
            urls[url] += 1
            response_codes[code] += 1

print("=== Top 10 IPs ===")
for ip, count in ips.most_common(10):
    print(f"{count:>8} {ip}")

print("\n=== Top 10 URLs ===")
for url, count in urls.most_common(10):
    print(f"{count:>8} {url}")

print("\n=== Response Codes ===")
for code, count in response_codes.most_common():
    print(f"{code}: {count}")

Сравнение инструментов

6. Рекомендации

# Для development: bash скрипт
# Для staging: GoAccess
# Для production: ELK Stack или Splunk

# В production используйте:
# 1. Ротируйте логи (logrotate)
# 2. Отправляйте в centralized logging (ELK)
# 3. Настройте алерты на ошибки
# 4. Мониторьте response time
# 5. Отслеживайте аномалии