On Jun 10, 8:21 pm, Miki <miki.teb...@gm ail.comwrote:
Hello,
Hi. I'm stretching my boundaries in programming with a little python
shell-script which is going to loop through a list of domain names,
grab the whois record, parse it, and put the results into a csv.
I've got the results coming back fine, but since I have *no*
experience with python I'm wondering what would be the preferred
"pythonic" way of parsing the whois string into a csv record.
Tips/thoughts/examples more than welcome!
from os import popen
import re
find_record = re.compile("\s+ ([^:]+): (.*)\s*").match
for line in popen("whois google.com"):
match = find_record(lin e)
if not match:
continue
print "%s --%s" % (match.groups()[0], match.groups()[1])
HTH,
--
Miki <miki.teb...@gm ail.com>http://pythonwise.blogspot.com
OK, here's what I've got so far. I'm treating this as a learning
exercise, so the resulting file isn't so important as understanding
and thinking in python (although I believe the results are adequate
for my needs). I'd appreciate the community's comments as this is my
*first* attempt at python and has taken me a couple of hours
(including googling).
#!/usr/bin/env python
import subprocess
import re
src = open('./domains.txt')
dest = open('./whois.csv', 'w');
def trim( txt ):
x = []
for line in txt.split("\n") :
if line.strip() == "":
continue
if line.strip().st artswith('WHOIS '):
continue
if line.strip().st artswith('>>>') :
continue
if line.strip().st artswith('%'):
continue
if line.startswith ("--"):
return ''.join(x)
x.append(" "+line)
return "\n".join(x )
def clean( txt ):
x = []
isok = re.compile("^\s ?([^:]+): ").match
for line in txt.split("\n") :
match = isok(line)
if not match:
continue
x.append(line)
return "\n".join(x );
def clean_co_uk( rec ):
rec = rec.replace('Co mpany number:', 'Company number -')
rec = rec.replace("\n \n", "\n")
rec = rec.replace("\n ", "")
rec = rec.replace(": ", ":\n")
rec = re.sub("([^(][a-zA-Z']+\s?[a-zA-Z]*:\n)", "\n\g<0>", rec)
rec = rec.replace(":\ n", ": ")
rec = re.sub("^[ ]+\n", "", rec)
return rec
def clean_net( rec ):
rec = rec.replace("\n \n", "\n")
rec = rec.replace("\n ", "")
rec = rec.replace(": ", ":\n")
rec = re.sub("([a-zA-Z']+\s?[a-zA-Z]*:\n)", "\n\g<0>", rec)
rec = rec.replace(":\ n", ": ")
return rec
def clean_info( rec ):
x = []
for line in rec.split("\n") :
x.append(re.sub ("^([^:]+):", "\g<0", line))
return "\n".join(x )
def record(domain, record):
## Records are as follows: [ domain, registrant, registrant's address
registrar, type, registered, renewal, updated name servers ]
details = ['','','','','', '','','','']
for k, v in record.items():
try:
details[0] = domain.lower()
result = {
"registrant ": lambda: 1,
"registrant name": lambda: 1,
"registrant type": lambda: 4,
"registrant 's address": lambda: 2,
"registrant address1": lambda: 2,
"registrar" : lambda: 3,
"sponsoring registrar": lambda: 3,
"registered on": lambda: 5,
"registered ": lambda: 5,
"domain registeration date": lambda: 5,
"renewal date": lambda: 6,
"last updated": lambda: 7,
"domain last updated date": lambda: 7,
"name servers": lambda: 8,
"name server": lambda: 8,
"nameserver s": lambda: 8,
"updated date": lambda: 7,
"creation date": lambda: 5,
"expiration date": lambda: 6,
"domain expiration date": lambda: 6,
"administra tive contact": lambda: 2
}[k.lower()]()
if v != '':
details[result] = v
except:
continue
dest.write('|'. join(details)+" \n")
## Loop through domains
for domain in src:
domain = domain.strip()
if domain == '':
continue
rec = subprocess.Pope n(["whois",dom ain],
stdout=subproce ss.PIPE).commun icate()[0]
if rec.startswith( "No whois server") == True:
continue
if rec.startswith( "This TLD has no whois server") == True:
continue
rec = trim(rec)
if domain.endswith (".net"):
rec = clean_net(rec)
if domain.endswith (".com"):
rec = clean_net(rec)
if domain.endswith (".tv"):
rec = clean_net(rec)
if domain.endswith (".co.uk"):
rec = clean_co_uk(rec )
if domain.endswith (".info"):
rec = clean_info(rec)
rec = clean(rec)
details = {}
try:
for line in rec.split("\n") :
bits = line.split(': ')
a = bits.pop(0)
b = bits.pop(0)
details[a.strip()] = b.strip().repla ce("\t", ", ")
except:
continue
record(domain, details)
src.close()
dest.close()
