Question

我正在使用

中的以下代码抓取网站

URL：99acres.com/property-in-velachery-chennai-south-ffid?

from string import punctuation, whitespace
import urllib2
import datetime
import re
import MySQLdb
from bs4 import BeautifulSoup as Soup

today = datetime.date.today()
html = urllib2.urlopen("http://www.99acres.com/property-in-velachery-chennai-south-ffid").read()

soup = Soup(html)
print "INSERT INTO Property (URL,Rooms, Place, Phonenumber1,Phonenumber2,Phonenumber3,Typeofperson, Name)"
print "VALUES ("
re_digit = re.compile('(\d+)')
pdate = soup.findAll('i', {'class':'pdate'})
properties = soup.findAll('a', title=re.compile('Bedroom'))
for eachproperty in properties:
# title = today,","+"http:/"+ eachproperty['href']+",", eachproperty.string+"," +",".join(re.findall("'([a-zA-Z0-9,\s]*)'", eachproperty['onclick'])) 
    for eachdate in pdate:
        pdates = re.sub('(\s{2,})', ' ', eachdate.text)
    for div in soup.find_all('div', {'class': 'sT_disc grey'}):
        try:
            project = div.find('span').find('b').text.strip()
        except:
            project = 'No project'  
        area = re.findall(re_digit, div.find('i', {'class': 'blk'}).text.strip())

        print today,","+"http:/"+ eachproperty['href']+",", eachproperty.string+"," +",".join(re.findall("'([a-zA-Z0-9,\s]*)'", eachproperty['onclick']))+","+ ", ".join([project] + area),","+pdates
print ")"

如果您运行此操作，您将看到数据重复出现。除此之外，所有必需的数据都被刮掉了。但我无法想象我哪里出错了。

Answer 1

你正在不必要地循环。您需要做的是：

from string import punctuation, whitespace
import urllib2
import datetime
import re
from bs4 import BeautifulSoup as Soup

today = datetime.date.today()
html = urllib2.urlopen("http://www.99acres.com/property-in-velachery-chennai-south-ffid").read()

soup = Soup(html)
print "INSERT INTO Property (URL,Rooms, Place, Phonenumber1,Phonenumber2,Phonenumber3,Typeofperson, Name)"
print "VALUES ("
re_digit = re.compile('(\d+)')
properties = soup.findAll('a', title=re.compile('Bedroom'))

for eachproperty in soup.findAll('div', {'class':'sT'}):
  a      = eachproperty.find('a', title=re.compile('Bedroom'))
  pdate  = eachproperty.find('i', {'class':'pdate'})
  pdates = re.sub('(\s{2,})', ' ', pdate.text)
  div    = eachproperty.find('div', {'class': 'sT_disc grey'})
  try:
    project = div.find('span').find('b').text.strip()
  except:
    project = 'No project'        
  area = re.findall(re_digit, div.find('i', {'class': 'blk'}).text.strip())
  print today,","+"http:/"+ (a['href'] if a else '')+",", (a.string if a else '')+ "," +",".join(re.findall("'([a-zA-Z0-9,\s]*)'", (a['onclick'] if a else '')))+","+ ", ".join([project] + area),","+pdates

获取重复数据 - 使用bs4进行屏幕抓取

1 个答案: