Malin revised this gist . Go to revision
1 file changed, 111 insertions
ips-dataset.sh(file created)
| @@ -0,0 +1,111 @@ | |||
| 1 | + | #!/bin/bash | |
| 2 | + | ||
| 3 | + | # Configuration | |
| 4 | + | LOG_DIR="/www/wwwlogs" | |
| 5 | + | DATASET_URL="https://ipinfo.io/data/free/country_asn.csv.gz?token=b807873daea34f" | |
| 6 | + | DATASET_FILE="country_asn.csv.gz" | |
| 7 | + | UNZIPPED_DATASET="country_asn.csv" | |
| 8 | + | ||
| 9 | + | # Function to convert IP to integer | |
| 10 | + | ip2int() { | |
| 11 | + | local ip=$1 | |
| 12 | + | IFS=. read -r i1 i2 i3 i4 <<< "$ip" | |
| 13 | + | echo $(( (i1 << 24) + (i2 << 16) + (i3 << 8) + i4 )) | |
| 14 | + | } | |
| 15 | + | ||
| 16 | + | # Function to get IP info from local dataset | |
| 17 | + | get_ip_info() { | |
| 18 | + | local ip=$1 | |
| 19 | + | local ip_int=$(ip2int "$ip") | |
| 20 | + | local info=$(awk -F',' -v ip_int="$ip_int" ' | |
| 21 | + | function ip2int(ip) { | |
| 22 | + | split(ip, octets, ".") | |
| 23 | + | return (octets[1] * 2^24) + (octets[2] * 2^16) + (octets[3] * 2^8) + octets[4] | |
| 24 | + | } | |
| 25 | + | NR > 1 { | |
| 26 | + | start = ip2int($1) | |
| 27 | + | end = ip2int($2) | |
| 28 | + | if (ip_int >= start && ip_int <= end) { | |
| 29 | + | print $8 "|" $4 | |
| 30 | + | exit | |
| 31 | + | } | |
| 32 | + | } | |
| 33 | + | ' "$UNZIPPED_DATASET") | |
| 34 | + | if [ -z "$info" ]; then | |
| 35 | + | echo "Unknown|Unknown" | |
| 36 | + | else | |
| 37 | + | echo "$info" | |
| 38 | + | fi | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | # Function to display spinning animation | |
| 42 | + | spinner() { | |
| 43 | + | local pid=$1 | |
| 44 | + | local delay=0.1 | |
| 45 | + | local spinstr='|/-\' | |
| 46 | + | while [ "$(ps a | awk '{print $1}' | grep $pid)" ]; do | |
| 47 | + | local temp=${spinstr#?} | |
| 48 | + | printf " [%c] " "$spinstr" | |
| 49 | + | local spinstr=$temp${spinstr%"$temp"} | |
| 50 | + | sleep $delay | |
| 51 | + | printf "\b\b\b\b\b\b" | |
| 52 | + | done | |
| 53 | + | printf " \b\b\b\b" | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | # Download and prepare dataset | |
| 57 | + | echo "Checking for dataset updates..." | |
| 58 | + | if [ -f "$UNZIPPED_DATASET" ]; then | |
| 59 | + | wget -q -O "${DATASET_FILE}.new" "$DATASET_URL" | |
| 60 | + | if ! cmp -s "$DATASET_FILE" "${DATASET_FILE}.new"; then | |
| 61 | + | echo "New dataset available. Updating..." | |
| 62 | + | mv "${DATASET_FILE}.new" "$DATASET_FILE" | |
| 63 | + | gunzip -f "$DATASET_FILE" | |
| 64 | + | else | |
| 65 | + | echo "Dataset is up to date." | |
| 66 | + | rm "${DATASET_FILE}.new" | |
| 67 | + | fi | |
| 68 | + | else | |
| 69 | + | echo "Downloading initial dataset..." | |
| 70 | + | wget -q "$DATASET_URL" -O "$DATASET_FILE" | |
| 71 | + | gunzip -f "$DATASET_FILE" | |
| 72 | + | fi | |
| 73 | + | ||
| 74 | + | echo "Analyzing Nginx logs for 444 status codes..." | |
| 75 | + | ||
| 76 | + | # Process all log files and sort results | |
| 77 | + | find "$LOG_DIR" -name "*.log" -type f -print0 | xargs -0 awk '$9 == 444 {print $1}' | sort | uniq -c | sort -rn > temp_ip_list.txt | |
| 78 | + | ||
| 79 | + | # Display header | |
| 80 | + | printf "\n%-6s %-15s %-40s %s\n" "Count" "IP Address" "AS Name" "Country" | |
| 81 | + | printf "%s\n" "$(printf '=%.0s' {1..80})" | |
| 82 | + | ||
| 83 | + | # Process each IP | |
| 84 | + | total_ips=$(wc -l < temp_ip_list.txt) | |
| 85 | + | current_ip=0 | |
| 86 | + | ||
| 87 | + | while read -r count ip; do | |
| 88 | + | current_ip=$((current_ip + 1)) | |
| 89 | + | printf "Processing IP %d of %d " "$current_ip" "$total_ips" | |
| 90 | + | ||
| 91 | + | # Start spinner in background | |
| 92 | + | spinner $$ & | |
| 93 | + | SPIN_PID=$! | |
| 94 | + | ||
| 95 | + | # Fetch IP info | |
| 96 | + | ip_info=$(get_ip_info "$ip") | |
| 97 | + | ||
| 98 | + | # Stop spinner | |
| 99 | + | kill $SPIN_PID &>/dev/null | |
| 100 | + | ||
| 101 | + | as_name=$(echo "$ip_info" | cut -d'|' -f1) | |
| 102 | + | country=$(echo "$ip_info" | cut -d'|' -f2) | |
| 103 | + | ||
| 104 | + | # Clear the processing line and print the result | |
| 105 | + | printf "\r%-6s %-15s %-40s %s\n" "$count" "$ip" "${as_name:0:40}" "$country" | |
| 106 | + | done < temp_ip_list.txt | |
| 107 | + | ||
| 108 | + | # Clean up | |
| 109 | + | rm temp_ip_list.txt | |
| 110 | + | ||
| 111 | + | echo "Analysis complete." | |
Newer
Older