分布式处理脚本

#!/bin/bash
# 分布式日志处理脚本

SERVERS=("server1" "server2" "server3")
LOG_FILE="/var/log/nginx/access.log"

distribute_process() {
    local total_lines=$(wc -l < "$LOG_FILE")
    local lines_per_server=$((total_lines / ${#SERVERS[@]}))
    
    echo "总行数: $total_lines, 每台服务器处理: $lines_per_server 行"
    
    for i in "${!SERVERS[@]}"; do
        local server="${SERVERS[$i]}"
        local start_line=$((i * lines_per_server + 1))
        local end_line=$(((i + 1) * lines_per_server))
        
        echo "分发给 $server: 行 $start_line - $end_line"
        
        # 提取对应行数并发送到远程服务器处理
        sed -n "${start_line},${end_line}p" "$LOG_FILE" | \
        ssh "$server" "
            awk '{ip_count[\$1]++} END {
                for (ip in ip_count) print ip, ip_count[ip]
            }' > /tmp/result_$i.txt
        " &
    done
    
    wait
    
    # 收集结果
    echo "收集结果..."
    for i in "${!SERVERS[@]}"; do
        scp "${SERVERS[$i]}:/tmp/result_$i.txt" "result_$i.txt"
    done
    
    # 合并最终结果
    awk '{sum[$1] += $2} END {
        for (ip in sum) print ip, sum[ip]
    }' result_*.txt | sort -k2 -nr > distributed_result.txt
    
    rm result_*.txt
}