ips-dataset.sh
· 3.0 KiB · Bash
Raw
#!/bin/bash
# Configuration
LOG_DIR="/www/wwwlogs"
DATASET_URL="https://ipinfo.io/data/free/country_asn.csv.gz?token=b807873daea34f"
DATASET_FILE="country_asn.csv.gz"
UNZIPPED_DATASET="country_asn.csv"
# Function to convert IP to integer
ip2int() {
local ip=$1
IFS=. read -r i1 i2 i3 i4 <<< "$ip"
echo $(( (i1 << 24) + (i2 << 16) + (i3 << 8) + i4 ))
}
# Function to get IP info from local dataset
get_ip_info() {
local ip=$1
local ip_int=$(ip2int "$ip")
local info=$(awk -F',' -v ip_int="$ip_int" '
function ip2int(ip) {
split(ip, octets, ".")
return (octets[1] * 2^24) + (octets[2] * 2^16) + (octets[3] * 2^8) + octets[4]
}
NR > 1 {
start = ip2int($1)
end = ip2int($2)
if (ip_int >= start && ip_int <= end) {
print $8 "|" $4
exit
}
}
' "$UNZIPPED_DATASET")
if [ -z "$info" ]; then
echo "Unknown|Unknown"
else
echo "$info"
fi
}
# Function to display spinning animation
spinner() {
local pid=$1
local delay=0.1
local spinstr='|/-\'
while [ "$(ps a | awk '{print $1}' | grep $pid)" ]; do
local temp=${spinstr#?}
printf " [%c] " "$spinstr"
local spinstr=$temp${spinstr%"$temp"}
sleep $delay
printf "\b\b\b\b\b\b"
done
printf " \b\b\b\b"
}
# Download and prepare dataset
echo "Checking for dataset updates..."
if [ -f "$UNZIPPED_DATASET" ]; then
wget -q -O "${DATASET_FILE}.new" "$DATASET_URL"
if ! cmp -s "$DATASET_FILE" "${DATASET_FILE}.new"; then
echo "New dataset available. Updating..."
mv "${DATASET_FILE}.new" "$DATASET_FILE"
gunzip -f "$DATASET_FILE"
else
echo "Dataset is up to date."
rm "${DATASET_FILE}.new"
fi
else
echo "Downloading initial dataset..."
wget -q "$DATASET_URL" -O "$DATASET_FILE"
gunzip -f "$DATASET_FILE"
fi
echo "Analyzing Nginx logs for 444 status codes..."
# Process all log files and sort results
find "$LOG_DIR" -name "*.log" -type f -print0 | xargs -0 awk '$9 == 444 {print $1}' | sort | uniq -c | sort -rn > temp_ip_list.txt
# Display header
printf "\n%-6s %-15s %-40s %s\n" "Count" "IP Address" "AS Name" "Country"
printf "%s\n" "$(printf '=%.0s' {1..80})"
# Process each IP
total_ips=$(wc -l < temp_ip_list.txt)
current_ip=0
while read -r count ip; do
current_ip=$((current_ip + 1))
printf "Processing IP %d of %d " "$current_ip" "$total_ips"
# Start spinner in background
spinner $$ &
SPIN_PID=$!
# Fetch IP info
ip_info=$(get_ip_info "$ip")
# Stop spinner
kill $SPIN_PID &>/dev/null
as_name=$(echo "$ip_info" | cut -d'|' -f1)
country=$(echo "$ip_info" | cut -d'|' -f2)
# Clear the processing line and print the result
printf "\r%-6s %-15s %-40s %s\n" "$count" "$ip" "${as_name:0:40}" "$country"
done < temp_ip_list.txt
# Clean up
rm temp_ip_list.txt
echo "Analysis complete."
| 1 | #!/bin/bash |
| 2 | |
| 3 | # Configuration |
| 4 | LOG_DIR="/www/wwwlogs" |
| 5 | DATASET_URL="https://ipinfo.io/data/free/country_asn.csv.gz?token=b807873daea34f" |
| 6 | DATASET_FILE="country_asn.csv.gz" |
| 7 | UNZIPPED_DATASET="country_asn.csv" |
| 8 | |
| 9 | # Function to convert IP to integer |
| 10 | ip2int() { |
| 11 | local ip=$1 |
| 12 | IFS=. read -r i1 i2 i3 i4 <<< "$ip" |
| 13 | echo $(( (i1 << 24) + (i2 << 16) + (i3 << 8) + i4 )) |
| 14 | } |
| 15 | |
| 16 | # Function to get IP info from local dataset |
| 17 | get_ip_info() { |
| 18 | local ip=$1 |
| 19 | local ip_int=$(ip2int "$ip") |
| 20 | local info=$(awk -F',' -v ip_int="$ip_int" ' |
| 21 | function ip2int(ip) { |
| 22 | split(ip, octets, ".") |
| 23 | return (octets[1] * 2^24) + (octets[2] * 2^16) + (octets[3] * 2^8) + octets[4] |
| 24 | } |
| 25 | NR > 1 { |
| 26 | start = ip2int($1) |
| 27 | end = ip2int($2) |
| 28 | if (ip_int >= start && ip_int <= end) { |
| 29 | print $8 "|" $4 |
| 30 | exit |
| 31 | } |
| 32 | } |
| 33 | ' "$UNZIPPED_DATASET") |
| 34 | if [ -z "$info" ]; then |
| 35 | echo "Unknown|Unknown" |
| 36 | else |
| 37 | echo "$info" |
| 38 | fi |
| 39 | } |
| 40 | |
| 41 | # Function to display spinning animation |
| 42 | spinner() { |
| 43 | local pid=$1 |
| 44 | local delay=0.1 |
| 45 | local spinstr='|/-\' |
| 46 | while [ "$(ps a | awk '{print $1}' | grep $pid)" ]; do |
| 47 | local temp=${spinstr#?} |
| 48 | printf " [%c] " "$spinstr" |
| 49 | local spinstr=$temp${spinstr%"$temp"} |
| 50 | sleep $delay |
| 51 | printf "\b\b\b\b\b\b" |
| 52 | done |
| 53 | printf " \b\b\b\b" |
| 54 | } |
| 55 | |
| 56 | # Download and prepare dataset |
| 57 | echo "Checking for dataset updates..." |
| 58 | if [ -f "$UNZIPPED_DATASET" ]; then |
| 59 | wget -q -O "${DATASET_FILE}.new" "$DATASET_URL" |
| 60 | if ! cmp -s "$DATASET_FILE" "${DATASET_FILE}.new"; then |
| 61 | echo "New dataset available. Updating..." |
| 62 | mv "${DATASET_FILE}.new" "$DATASET_FILE" |
| 63 | gunzip -f "$DATASET_FILE" |
| 64 | else |
| 65 | echo "Dataset is up to date." |
| 66 | rm "${DATASET_FILE}.new" |
| 67 | fi |
| 68 | else |
| 69 | echo "Downloading initial dataset..." |
| 70 | wget -q "$DATASET_URL" -O "$DATASET_FILE" |
| 71 | gunzip -f "$DATASET_FILE" |
| 72 | fi |
| 73 | |
| 74 | echo "Analyzing Nginx logs for 444 status codes..." |
| 75 | |
| 76 | # Process all log files and sort results |
| 77 | find "$LOG_DIR" -name "*.log" -type f -print0 | xargs -0 awk '$9 == 444 {print $1}' | sort | uniq -c | sort -rn > temp_ip_list.txt |
| 78 | |
| 79 | # Display header |
| 80 | printf "\n%-6s %-15s %-40s %s\n" "Count" "IP Address" "AS Name" "Country" |
| 81 | printf "%s\n" "$(printf '=%.0s' {1..80})" |
| 82 | |
| 83 | # Process each IP |
| 84 | total_ips=$(wc -l < temp_ip_list.txt) |
| 85 | current_ip=0 |
| 86 | |
| 87 | while read -r count ip; do |
| 88 | current_ip=$((current_ip + 1)) |
| 89 | printf "Processing IP %d of %d " "$current_ip" "$total_ips" |
| 90 | |
| 91 | # Start spinner in background |
| 92 | spinner $$ & |
| 93 | SPIN_PID=$! |
| 94 | |
| 95 | # Fetch IP info |
| 96 | ip_info=$(get_ip_info "$ip") |
| 97 | |
| 98 | # Stop spinner |
| 99 | kill $SPIN_PID &>/dev/null |
| 100 | |
| 101 | as_name=$(echo "$ip_info" | cut -d'|' -f1) |
| 102 | country=$(echo "$ip_info" | cut -d'|' -f2) |
| 103 | |
| 104 | # Clear the processing line and print the result |
| 105 | printf "\r%-6s %-15s %-40s %s\n" "$count" "$ip" "${as_name:0:40}" "$country" |
| 106 | done < temp_ip_list.txt |
| 107 | |
| 108 | # Clean up |
| 109 | rm temp_ip_list.txt |
| 110 | |
| 111 | echo "Analysis complete." |