尝试在使用Python的JavaScript之后抓取HTML ...

时间：2019-06-18 03:26:51

标签： javascript python web-scraping beautifulsoup selenium-chromedriver

我已经尝试了几乎所有内容... PyQt5，Selenium，BS4，requests_html等...仍然无法得到我想要的东西。

我正在尝试从https://www.tokenanalyst.io/exchange网上抓取数据，特别是这些美元值。似乎此数据是在某些Javascript代码之后生成的。我无法成功抓取上述数据。有没有人能够成功返回HTML代码或整数？

2 个答案:

答案 0 :(得分：0)

由于这个问题有点不清楚。它没有显示您尝试了什么方法。 对于类似的网站，您不能使用漂亮的汤。但是，Selenium和chromedriver将是一个不错的选择，因为您始终可以抓取html来获取数据。 您必须locate what you want to scrape 但是对我来说，这种方法有点慢。

答案 1 :(得分：0)

您可以从其API解析JSON值：

import requests
import json

exchanges = ['Bitfinex', 'Binance', 'Bitstamp', 'Bittrex', 'Poloniex', 'BitMEX']

data = {}
for e in exchanges:
    url_inflow = 'https://api.tokenanalyst.io/analytics/last?job=btc_{}_{}_24h_rolling_v5&format=json'.format(e, 'inflow')
    url_outflow = 'https://api.tokenanalyst.io/analytics/last?job=btc_{}_{}_24h_rolling_v5&format=json'.format(e, 'outflow')

    j_inflow = json.loads(requests.get(url_inflow).text)
    j_outflow = json.loads(requests.get(url_outflow).text)

    print(e)
    print(j_inflow)
    print(j_outflow)

    data[e] = (j_inflow[0]['inflow_usd'], j_outflow[0]['outflow_usd'])

print('*' * 80)

for k, v in data.items():
    print('Exchange: {} Inflow_USD: {} Outflow_USD: {}'.format(k, v[0], v[1]))

打印：

Bitfinex
[{'avg_txn_value': 3.4021, 'entity': 'Bitfinex', 'inflow': 945.79000615, 'number_of_entity_receiving_addresses': 246, 'number_of_nonentity_sending_addresses': 1427, 'number_of_txns': 278, 'period_ending_at': '2019-06-17T22:12:27.500Z', 'inflow_usd': 8707062.91194768, 'avg_txn_value_usd': 31320.164666700002, 'inflow_usd_pct_change': '-54.67230315707747', 'avg_txn_value_usd_pct_change': '-47.98800964713564', 'inflow_pct_change': '-55.32126412430956', 'avg_txn_value_pct_change': '-48.7326702833032'}]
[{'avg_txn_value': 3.199, 'entity': 'Bitfinex', 'number_of_entity_sending_addresses': 245, 'number_of_nonentity_receiving_addresses': 644, 'number_of_txns': 258, 'outflow': 2060.15622225, 'period_ending_at': '2019-06-17T22:12:21.346Z', 'outflow_usd': 18966059.82187373, 'avg_txn_value_usd': 29450.400273, 'outflow_usd_pct_change': '-16.706252918178574', 'avg_txn_value_usd_pct_change': '-33.13281601840719', 'outflow_pct_change': '-17.898777454731185', 'avg_txn_value_pct_change': '-34.09015988132521'}]
Binance
[{'avg_txn_value': 3.6763, 'entity': 'Binance', 'inflow': 13205.11846359, 'number_of_entity_receiving_addresses': 3282, 'number_of_nonentity_sending_addresses': 24453, 'number_of_txns': 3592, 'period_ending_at': '2019-06-17T22:12:35.270Z', 'inflow_usd': 121569489.80424081, 'avg_txn_value_usd': 33844.900111999996, 'inflow_usd_pct_change': 78.7755357787967, 'avg_txn_value_usd_pct_change': 75.14686308991297, 'inflow_pct_change': 76.21343804513097, 'avg_txn_value_pct_change': 72.6367691946466}]
[{'avg_txn_value': 1.2547, 'entity': 'Binance', 'number_of_entity_sending_addresses': 1, 'number_of_nonentity_receiving_addresses': 9317, 'number_of_txns': 151, 'outflow': 11689.74225155, 'period_ending_at': '2019-06-17T22:12:25.426Z', 'outflow_usd': 107617251.76503526, 'avg_txn_value_usd': 11550.9275469, 'outflow_usd_pct_change': 18.73881008764625, 'avg_txn_value_usd_pct_change': 27.803672981371243, 'outflow_pct_change': 17.03881519689553, 'avg_txn_value_pct_change': 25.97389558232931}]
Bitstamp
[{'avg_txn_value': 4.9848, 'entity': 'Bitstamp', 'inflow': 4845.21760603, 'number_of_entity_receiving_addresses': 363, 'number_of_nonentity_sending_addresses': 5109, 'number_of_txns': 972, 'period_ending_at': '2019-06-18T03:13:29.366Z', 'inflow_usd': 44827992.052730404, 'avg_txn_value_usd': 46119.4094784, 'inflow_usd_pct_change': 20.9229998843688, 'avg_txn_value_usd_pct_change': 12.338050679950097, 'inflow_pct_change': 19.18043979551204, 'avg_txn_value_pct_change': 10.71920394473812}]
[{'avg_txn_value': 2.1021, 'entity': 'Bitstamp', 'number_of_entity_sending_addresses': 211, 'number_of_nonentity_receiving_addresses': 2818, 'number_of_txns': 86, 'outflow': 5923.80448966, 'period_ending_at': '2019-06-17T22:05:56.784Z', 'outflow_usd': 54531167.56325086, 'avg_txn_value_usd': 19350.734403000002, 'outflow_usd_pct_change': 23.4058365064299, 'avg_txn_value_usd_pct_change': '-13.467075488325623', 'outflow_pct_change': 21.635752626972593, 'avg_txn_value_pct_change': '-14.70826909031891'}]
Bittrex
[{'avg_txn_value': 0.455, 'entity': 'Bittrex', 'inflow': 613.35675971, 'number_of_entity_receiving_addresses': 1362, 'number_of_nonentity_sending_addresses': 4700, 'number_of_txns': 1348, 'period_ending_at': '2019-06-18T03:13:29.480Z', 'inflow_usd': 5674781.647690997, 'avg_txn_value_usd': 4209.66364, 'inflow_usd_pct_change': '-3.5874967447576775', 'avg_txn_value_usd_pct_change': '-15.028043006702655', 'inflow_pct_change': '-4.976848484294497', 'avg_txn_value_pct_change': '-16.252530830112274'}]
[{'avg_txn_value': 0.326, 'entity': 'Bittrex', 'number_of_entity_sending_addresses': 1396, 'number_of_nonentity_receiving_addresses': 2029, 'number_of_txns': 154, 'outflow': 661.49258585, 'period_ending_at': '2019-06-17T22:05:52.827Z', 'outflow_usd': 6089323.694561165, 'avg_txn_value_usd': 3000.9701800000003, 'outflow_usd_pct_change': '-15.905765601357832', 'avg_txn_value_usd_pct_change': '-24.950292648797074', 'outflow_pct_change': '-17.11197960936857', 'avg_txn_value_pct_change': '-26.026775584297702'}]
Poloniex
[{'avg_txn_value': 0.9987, 'entity': 'Poloniex', 'inflow': 551.29290849, 'number_of_entity_receiving_addresses': 688, 'number_of_nonentity_sending_addresses': 2832, 'number_of_txns': 552, 'period_ending_at': '2019-06-18T03:13:29.395Z', 'inflow_usd': 5100566.399692748, 'avg_txn_value_usd': 9239.9803896, 'inflow_usd_pct_change': 40.079661028653504, 'avg_txn_value_usd_pct_change': 8.863578042174874, 'inflow_pct_change': 38.06104400126806, 'avg_txn_value_pct_change': 7.294800171895152}]
[{'avg_txn_value': 0.5784, 'entity': 'Poloniex', 'number_of_entity_sending_addresses': 2, 'number_of_nonentity_receiving_addresses': 1048, 'number_of_txns': 302, 'outflow': 606.13545851, 'period_ending_at': '2019-06-17T22:05:48.914Z', 'outflow_usd': 5579737.53383171, 'avg_txn_value_usd': 5324.420712, 'outflow_usd_pct_change': '-8.109879467421727', 'avg_txn_value_usd_pct_change': '-21.789008537427183', 'outflow_pct_change': '-9.427914542915346', 'avg_txn_value_pct_change': '-22.9108356657337'}]
BitMEX
[{'avg_txn_value': 0.6535, 'entity': 'Bitmex', 'inflow': 2120.51020525, 'number_of_entity_receiving_addresses': 3392, 'number_of_nonentity_sending_addresses': 10107, 'number_of_txns': 3245, 'period_ending_at': '2019-06-18T03:13:29.505Z', 'inflow_usd': 19618977.383054644, 'avg_txn_value_usd': 6046.187228, 'inflow_usd_pct_change': '-41.10742264036966', 'avg_txn_value_usd_pct_change': '-33.7342645462863', 'inflow_pct_change': '-41.95609373631447', 'avg_txn_value_pct_change': '-34.68918648810713'}]
[{'avg_txn_value': 1.9551, 'entity': 'Bitmex', 'number_of_entity_sending_addresses': 2101, 'number_of_nonentity_receiving_addresses': 2063, 'number_of_txns': 2231, 'outflow': 4033.36989488, 'period_ending_at': '2019-06-17T22:06:16.970Z', 'outflow_usd': 37128904.2314252, 'avg_txn_value_usd': 17997.536193, 'outflow_usd_pct_change': 247.60891113166488, 'avg_txn_value_usd_pct_change': 97.97896643193221, 'outflow_pct_change': 242.62294817100863, 'avg_txn_value_pct_change': 95.13923545264}]
********************************************************************************
Exchange: Bitfinex Inflow_USD: 8707062.91194768 Outflow_USD: 18966059.82187373
Exchange: Binance Inflow_USD: 121569489.80424081 Outflow_USD: 107617251.76503526
Exchange: Bitstamp Inflow_USD: 44827992.052730404 Outflow_USD: 54531167.56325086
Exchange: Bittrex Inflow_USD: 5674781.647690997 Outflow_USD: 6089323.694561165
Exchange: Poloniex Inflow_USD: 5100566.399692748 Outflow_USD: 5579737.53383171
Exchange: BitMEX Inflow_USD: 19618977.383054644 Outflow_USD: 37128904.2314252
相关问题
最新问题