PeterHarding: Created page with "= Searching WhitePages = The following Python script uses HTTPLib to search against whitepages.com.au (via a proxy).

  #!/usr/bin/env python  #  #  #-------------------..."

2021-10-16T10:45:21Z

Created page with "= Searching WhitePages = The following Python script uses HTTPLib to search against whitepages.com.au (via a proxy). <pre> #!/usr/bin/env python # # #-------------------..."

New page

= Searching WhitePages =

The following Python script uses HTTPLib to search against whitepages.com.au (via a proxy).

<pre>
#!/usr/bin/env python
#
#
#-------------------------------------------------------------------------------

import re
import sys
import base64
import pprint
import urllib
import httplib

from copy import copy

#-------------------------------------------------------------------------------

PROXY = 'PROXY:8080'
SITE = 'www.whitepages.com.au'

connection = None

#===== Headers =================================================================

# 'Accept' : 'text/plain, text/html',

get_headers = {
'Accept-Encoding' : 'gzip, deflate',
'Accept' : '*/*',
'Accept-Language' : 'en-au',
'Host' : SITE,
'Connection' : 'Keep-Alive',
'User-Agent' : 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727'
}

post_headers = {
'Content-type' : 'application/x-www-form-urlencoded',
'Accept' : 'text/plain'
}

idx = 0

#===== Logging =================================================================

def log_req_header(idx, hdr):
of = open('log/%04d.req' % idx, 'w')
of.write("%s\n" % pprint.pformat(hdr))
of.close()

#-------------------------------------------------------------------------------

def log_resp_header(idx, resp):
of = open('log/%04d.hdr' % idx, 'w')

of.write("resp.__dict__ ->\n%s\n\n" % pprint.pformat(resp.__dict__))
of.write("Status %s Reason [%s]\n" % (resp.status, resp.reason))
of.write("Msg ->\n%s\n\n" % resp.msg)
of.write("Msg.__dict__ ->\n%s\n\n" % pprint.pformat(resp.msg.__dict__))

of.close()

#-------------------------------------------------------------------------------

def log_resp_body(idx, resp_body):
of = open('log/%04d.bdy' % idx, 'w')
of.write(resp_body);
of.close()

#===== Encapsulate the request code ============================================

def request(method, url, params, headers):
global idx

print '>>>> %s %s <<<<' % (method, url)

connection.request(method, url, params, headers)

resp = connection.getresponse()

log_req_header(idx, headers)
log_resp_header(idx, resp)

resp_body = resp.read()

log_resp_body(idx, resp_body)

print resp_body

idx += 1

return resp

#===============================================================================

def do():
global connection
connection = httplib.HTTPConnection(PROXY)

BASE_URL = 'http://%s' % SITE

#------------------------------------------------------------------------

DO = 'GET'
URL = BASE_URL + '/'

headers = copy(get_headers)

request(DO, URL, None, headers)

#------------------------------------------------------------------------

DO = 'GET'
URL = BASE_URL + '/wp/index.jsp'

headers = copy(get_headers)

resp = request(DO, URL, None, headers)

m = re.search('JSESSIONID=(.*);', resp.msg.__dict__['dict']['set-cookie'])

if m:
print m.group(1)
JSESSIONID = m.group(1)

print JSESSIONID

#---------------------------------------------------------------------

DO = 'POST'
URL = BASE_URL + '/wp/busSearch.do;jsessionid=%s' % JSESSIONID

headers = copy(post_headers)

form_data = {
'subscriberName' : 'Hard',
'state' : 'VIC',
'suburb' : '',
'street' : '',
'Search' : 'Search'
}

params = urllib.urlencode(form_data)

headers['Content-Length'] = len(params)
headers['Cookie'] = 'JSESSIONID=%s' % JSESSIONID

request(DO, URL, params, headers)

#---------------------------------------------------------------------

URL = BASE_URL + '/wp/busSearch.do'

form_data = {
'subscriberName' : 'Hard',
'state' : 'VIC',
'page' : '2'
}

params = urllib.urlencode(form_data)

headers['Content-Length'] = len(params)
headers['Cookie'] = 'JSESSIONID=%s' % JSESSIONID

request(DO, URL, params, headers)

#===============================================================================

do()

#-------------------------------------------------------------------------------
</pre>

This script writes the search results into files (page 1 => 'log/0002.bdy' and page 2 => 'log/0003.bdy'). Amend the above code handle more pages of search results being produced.

= Parsing the Search Results =

The search results HTML looks as follows:

<pre>
<div class="encap_result" id="result-10"><ul><li id='res10-ln0'><h4><span class='blackboldcaps'>Hard ...</ul></li></ul><div class="clearMe"> </div></div><div class="encap_result" id="result-11"><ul><li>
</pre>

=Results of Search=

Are wriiten to a file in a block of XML.

=A more Complex Script=

This script loops through all the available pages in the search results and parses out the search results using the BeautifulSoup HTTP parsing module.

==wp_get.py==

<pre>
#!/usr/bin/env python
#
#
#-------------------------------------------------------------------------------

import re
import sys
import base64
import pprint
import urllib
import httplib

from copy import copy

#-------------------------------------------------------------------------------

from wp_parser import parse

#-------------------------------------------------------------------------------

PROXY = 'PROXY:8080'
SITE = 'www.whitepages.com.au'

connection = None
next_page_pat = None

results = ''

#-------------------------------------------------------------------------------

# 'Accept' : 'text/plain, text/html',

get_headers = {
'Accept-Encoding' : 'gzip, deflate',
'Accept' : '*/*',
'Accept-Language' : 'en-au',
'Host' : SITE,
'Connection' : 'Keep-Alive',
'User-Agent' : 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727'
}

post_headers = {
'Content-type' : 'application/x-www-form-urlencoded',
'Accept' : 'text/plain'
}

idx = 0

#-------------------------------------------------------------------------------

def log_req_header(idx, hdr):
of = open('log/%04d.req' % idx, 'w')
of.write("%s\n" % pprint.pformat(hdr))
of.close()

#-------------------------------------------------------------------------------

def log_resp_header(idx, resp):
of = open('log/%04d.hdr' % idx, 'w')

of.write("resp.__dict__ ->\n%s\n\n" % pprint.pformat(resp.__dict__))
of.write("Status %s Reason [%s]\n" % (resp.status, resp.reason))
of.write("Msg ->\n%s\n\n" % resp.msg)
of.write("Msg.__dict__ ->\n%s\n\n" % pprint.pformat(resp.msg.__dict__))

of.close()

#-------------------------------------------------------------------------------

def log_resp_body(idx, resp_body):
of = open('log/%04d.bdy' % idx, 'w')
of.write(resp_body);
of.close()

#-------------------------------------------------------------------------------

def do(pattern, state):
global connection
global results

print "Pattern ==> '%s'" % pattern

connection = httplib.HTTPConnection(PROXY)

BASE_URL = 'http://%s' % SITE

#------------------------------------------------------------------------

DO = 'GET'
URL = BASE_URL + '/'

headers = copy(get_headers)

request(DO, URL, None, headers)

#------------------------------------------------------------------------

DO = 'GET'
URL = BASE_URL + '/wp/index.jsp'

headers = copy(get_headers)

r = request(DO, URL, None, headers)

m = re.search('JSESSIONID=(.*);', r['response_header'].msg.__dict__['dict']['set-cookie'])

if m:
print m.group(1)
JSESSIONID = m.group(1)

print JSESSIONID

#---------------------------------------------------------------------

DO = 'POST'
URL = BASE_URL + '/wp/busSearch.do;jsessionid=%s' % JSESSIONID

headers = copy(post_headers)

form_data = {
'subscriberName' : pattern,
'state' : state,
'suburb' : '',
'street' : '',
'Search' : 'Search'
}

params = urllib.urlencode(form_data)

headers['Content-Length'] = len(params)
headers['Cookie'] = 'JSESSIONID=%s' % JSESSIONID

r = request(DO, URL, params, headers)

xml = parse('log/%04d.bdy' % r['idx'])

results += xml

body = r['response_body']

m = next_page_pat.findall(body)

cnt = 0

if m:
no_pages = len(m)
if no_pages > 0:
for i in range(no_pages):
print m[i]
#print m[i].group(1)

cnt = no_pages - 1

#---------------------------------------------------------------------

for i in range(cnt):

URL = BASE_URL + '/wp/busSearch.do'

form_data = {
'subscriberName' : pattern,
'state' : state,
'page' : i + 1
}

params = urllib.urlencode(form_data)

headers['Content-Length'] = len(params)
headers['Cookie'] = 'JSESSIONID=%s' % JSESSIONID

r = request(DO, URL, params, headers)

xml = parse('log/%04d.bdy' % r['idx'])

results += xml

ofh = open('results.xml', 'a+')

ofh.write(results)

ofh.close()

#-------------------------------------------------------------------------------

def request(method, url, params, headers):
global idx

print '>>>> %s %s <<<<' % (method, url)

connection.request(method, url, params, headers)

resp = connection.getresponse()

log_req_header(idx, headers)
log_resp_header(idx, resp)

resp_body = resp.read()

log_resp_body(idx, resp_body)

# print resp_body

r = {'idx' : idx, 'request_header' : headers, 'response_header' : resp, 'response_body' : resp_body}

idx += 1

return r

#-------------------------------------------------------------------------------

def process(search_patterns):
global next_page_pat

next_page_pat = re.compile(r';(page=[0-9]*">[0-9]*<\/a>)')

for search_pattern in search_patterns:
do(search_pattern, 'VIC')

#-------------------------------------------------------------------------------

def searches():

searches = []

for i in range(4):
ch = chr(ord('W') + i)
for x in ('a', 'e', 'i', 'o', 'u'):
pat = ch + x
searches.append(pat)

return searches

#-------------------------------------------------------------------------------

def used():
searches = []
searches.append('Zeus')

#-------------------------------------------------------------------------------

def main():
searches = []

searches.append('Zany')
searches.append('Zan')
searches.append('Zen')
searches.append('Zend')

process(searches)

#-------------------------------------------------------------------------------

main()

#-------------------------------------------------------------------------------
</pre>

==wp_parse.py==

<pre>
#!/usr/bin/env python

import pprint

from BeautifulSoup import BeautifulSoup

#-------------------------------------------------------------------------------------

def parse(fname):
doc = open(fname, 'r')

soup = BeautifulSoup(doc)

# print len(soup('table', { "class" : "table_style"}))

# tables = soup.findAll('table', { "class" : "table_style"})
objs = soup.findAll('div', { "class" : "encap_result"})

pp = pprint.PrettyPrinter(3)

xml = ''

for obj in objs:
t = obj.find(text=True)

if t:
xml += '<entry>\n'

#print '[[%s]]\n\n' % obj.__dict__
# print '[[%s]]\n\n' % obj

f = obj.findAll('span', { 'class' : 'black'})

for s in f:
xml += ' <tag>%s</tag>\n' % s.find(text=True)

f = obj.findAll('input', { "name" : 'placeName'})

for s in f:
# pp.pprint(s.__dict__)
# print 'attrMap -> "%s"' % s.attrMap
xml += ' <placeName>%s</placeName>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'subscriberName'}):
xml += ' <subscriberName>%s</subscriberName>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'address'}):
xml += ' <address>%s</address>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'streetNumber'}):
xml += ' <streetNumber>%s</streetNumber>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'streetName'}):
xml += ' <streetName>%s</streetName>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'streetType'}):
xml += ' <streetType>%s</streetType>\n' % s.attrMap['value']

for s in obj.findAll('input', { "name" : 'locality'}):
xml += ' <locality>%s</locality>\n' % s.attrMap['value']

http://www.magusco.com/kb/index.php/Parsing_WhitePages_Search_Results_HTML
lis = obj.findAll('li', { "class" : None})

for li in lis:
xml += ' <li>%s</li>\n' % li.find(text=True)

for address in obj.findAll('li', { "class" : "entryData address"}):
xml += ' <addr>%s</addr>\n' % address.find(text=True)

for phone in obj.findAll('li', { "class" : "entryData phoneNumber"}):
xml += ' <phoneNumber>%s</phoneNumber>\n' % phone.find(text=True)

xml += '</entry>\n\n'

return xml

#-------------------------------------------------------------------------------------

def test():
# xml = parse('html/0002.html')

# print xml

#-------------------------------------------------------------------------------------
</pre>

=Results of Search=

Are written to a file as XML.

[[Category:Python]]
[[Category:Python httplib]]
[[Category:Internet]]

Editing Parsing WhitePages Search Results HTML - Revision history

PeterHarding: Created page with "= Searching WhitePages = The following Python script uses HTTPLib to search against whitepages.com.au (via a proxy).
#!/usr/bin/env python # # #-------------------..."