无法将抓取的数据转换为CSV文件

时间：2018-11-06 03:41:42

标签： python beautifulsoup 

import csv
import requests
from bs4 import BeautifulSoup

html = requests.get("https://www.medplusmedicalsupply.com/exam-and-
diagnostic").text
soup = BeautifulSoup(html, "lxml")
products = soup.findAll('div', {"class": "product details product-item-details"})
for product in products:
    product_details = dict()
    product_details['name'] = product.find('a').text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['brand'] = product.find('div', {'class': 'value'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['packaging'] = product.find('div', {'class': 'pack'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['availability'] = product.find('div', {'class': 'avail pack'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['price'] = product.find('span', {'class': 'price'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    print(product_details)

prod = product_details

with open('../../www/products.csv', 'w+', newline='') as csvFile:
    writer = csv.writer(csvFile)
    writer.writerow(['Name', 'Brand', 'Packaging', 'Availability', 'Price'])
    for product in products:
        writer.writerow([product['Name'], product['Brand'],product['Packaging'], product['Availability'], product['Price']])

结果：

{'name': 'Kimberly Clark Blue Nitrile Exam Gloves -100/BX', 'brand': 'Kimberly Clark', 'packaging': 'Brand: \n\nKimberly Clark', 'availability': 'Availability: \n\n1-3 Days', 'price': '$26.94'}
{'name': 'Pro Advantage Gloves - 200/BX', 'brand': 'Pro Advantage', 'packaging': 'Brand: \n\nPro Advantage', 'availability': 'Availability: \n\n1-3 Days', 'price': '$28.94'}
{'name': 'Kimberly Clark Powder Free Exam Gloves - 500/CS', 'brand': 'Kimberly Clark', 'packaging': 'Brand: \n\nKimberly Clark', 'availability': 'Availability: \n\n1-3 Days', 'price': '$136.99'}
{'name': 'Kimberly Clark Sterile Pairs Exam Gloves - 400/CS', 'brand': 'Kimberly Clark', 'packaging': 'Brand: \n\nKimberly Clark', 'availability': 'Availability: \n\n1-3 Days', 'price': '$205.99'}
{'name': 'Dynarex Multi Care Vinyl Exam Gloves Non-Latex - 1000/CS', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$61.99'}
{'name': 'Dynarex Tillotson Powder Free Plus Latex Exam Gloves - 1000/CS', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$109.99'}
{'name': 'Dynarex 6512 True Advantage High Risk Nitrile Exam Gloves, 8 mil - 500/CS', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$105.99'}
{'name': 'Dynarex 6512 True Advantage High Risk Nitrile Exam Gloves, 8 mil - 50/BX', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$20.99'}
{'name': 'Dynarex Tillotson Tru Advantage PF Nitrile Exam Gloves - 1000/CS', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$101.99'}
{'name': 'Dynarex Tillotson Tru Advantage PF Nitrile Exam Gloves - 100/BX', 'brand': 'Dynarex', 'packaging': 'Brand: \n\nDynarex', 'availability': 'Availability: \n\n1-3 Days', 'price': '$18.99'}

KeyError

Traceback (most recent call last)
<ipython-input-10-668855af45c8> in <module>()
     24     writer.writerow(['Name', 'Brand', 'Packaging', 'Availability', 'Price'])
     25 for product in products:
---> 26     writer.writerow([product['Name'], product['Brand'],product['Packaging'], product['Availability'], product['Price']])
     27
     28

~\Anaconda3\lib\site-packages\bs4\element.py in __getitem__(self, key)
   1009         """tag[key] returns the value of the 'key' attribute for the tag,
   1010         and throws an exception if it's not there."""
-> 1011         return self.attrs[key]
   1012
   1013     def __iter__(self):

KeyError: 'Name'

2 个答案:

答案 0 :(得分：0)

实际上Martineau是正确的-Name应该换成name，并且product_details字典中的每个键都应进行类似的更改。

但这不是这里的唯一问题-您的抓取将无法按您的要求进行，因为product_details['packaging']和product_details['availability']的字符串中间包含换行符。因此，它们也需要清洗。

在当前流程中，您也不会获得结果，因为您正在将每行数据收集到product_details字典中，但是您正在从csv对象写入products（与所有标签的东西）。最初，您只需在每次迭代中重写product_details，甚至不用此字典就可以写入输出文件。

因此，在编写循环时，您需要在products上更改product_details。

这是一个可行的方案：

import csv
import requests
from bs4 import BeautifulSoup

html = requests.get("https://www.medplusmedicalsupply.com/exam-and-diagnostic").text
soup = BeautifulSoup(html, "lxml")
products = soup.findAll('div', {"class": "product details product-item-details"})


with open('.../products.csv', 'w+', newline='') as csvFile:
    writer = csv.writer(csvFile)
    writer.writerow(['Name' + '\t' + 'Brand' + '\t' + 'Packaging' + '\t' + 'Availability' + '\t' + 'Price'])
    for product in products:
        product_details = dict()
        product_details['name'] = product.find('a').text.strip('\n\r\t": ').strip('\n\r\t": ')
        product_details['brand'] = product.find('div', {'class': 'value'}).text.strip('\n\r\t": ')
        product_details['packaging'] = product.find('div', {'class': 'pack'}).text.strip('\n\r\t": ')
        product_details['availability'] = product.find('div', {'class': 'avail pack'}).text.strip('\n\r\t": ')
        product_details['price'] = product.find('span', {'class': 'price'}).text.strip('\n\r\t": ')

        product_details['packaging'] = product_details['packaging'][9:] # here we're cutting redundant part of string "Brand: \n\n"
        product_details['availability'] = product_details['availability'][16:] # here we're cutting redundant part of string "Availability: \n\n"
        writer.writerow([product_details['name'] + '\t' + product_details['brand'] + '\t' + product_details['packaging'] + '\t' + product_details['availability'] + '\t' + product_details['price']])

我将csv文件的分隔符更改为tab，因为在某些情况下name的产品包含逗号，例如对于excel来说，这是错误的字符，列分隔符，因此在某些行中，您会有多余的列。

答案 1 :(得分：0)

您在keyName（应为lowercase和以下代码中出现错误

for product in products:
    writer.writerow......

products不是BeautifulSoup object的上一个循环中生成的产品列表，您不能将其用于csv值。

最后，您可以尝试此解决方案

import csv
import requests
from bs4 import BeautifulSoup

html = requests.get("https://www.medplusmedicalsupply.com/exam-and-diagnostic").text
soup = BeautifulSoup(html, "html.parser")
products = soup.findAll('div', {"class": "product details product-item-details"})
all_product = []

for product in products:
    product_details = dict()
    product_details['name'] = product.find('a').text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['brand'] = product.find('div', {'class': 'value'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['packaging'] = product.find('div', {'class': 'pack'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['availability'] = product.find('div', {'class': 'avail pack'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    product_details['price'] = product.find('span', {'class': 'price'}).text.strip('\n\r\t": ').strip('\n\r\t": ').strip('\n\r\t": ')
    all_product.append(product_details)

print(all_product)

with open('products.csv', 'w+') as csvFile:
    writer = csv.writer(csvFile)
    writer.writerow(['Name', 'Brand', 'Packaging', 'Availability', 'Price'])
    for product in all_product:
        writer.writerow([product['name'], product['brand'],product['packaging'], product['availability'], product['price']])
相关问题
最新问题