This script will return a list of IP addresses to web servers that are linked from a given domain's home page.

This script was taken from the BackTrack Wiki. For updates please check the original site.

#!/bin/bash

clear
echo
echo Crawl
echo
echo
echo By Lee Baird
echo March 23, 2007
echo "v 0.2"
echo
echo "This script will return a list of IP addresses to web servers that are linked from a given domain's home page."
echo
echo Usage:  domain.com
echo Enter the domain you wish to crawl.
echo
read domain
echo
echo "####################"
echo
wget www.$domain

grep "href=" index.html | cut -d "/" -f3 | grep $domain | cut -d '"' -f1 | sort -u > $domain-servers.txt

for x in $(cat $domain-servers.txt);do
host $x | grep "has address" | cut -d " " -f4 >> $domain-ips.txt
done
cat $domain-ips.txt | sort -u > $domain.txt

rm index.html
rm $domain-servers.txt
rm $domain-ips.txt

echo "####################"
echo
cat $domain.txt
echo
echo The following file has been created:  $domain.txt
echo

Submitted by Lee Baird