Question

我正在构建一个小型网络剪贴板，我偶然发现了以下问题：我的应用程序需要抓取网站的不同部分并将信息放入数据库。有时它会产生疯狂的结果，例如重复的条目，或者从函数getPhoto（）返回undefined。但是，如果我只调用该函数（并且不运行脚本的其余部分），则返回正确的结果！

我有一个for循环，循环遍历不同的URL。它转到每个URL并抓取以下信息：1。标题，2.description，3。内部链接，4。调用根据标题生成图像的函数（getPhoto（...）），5。保存结果给了DB。一切都发生在服务器上（我正在使用Cron作业，没有客户端交互）

 for (i = 0; i < AllLinks.length; i++) {  

  if (AllLinks[i] != undefined && AllLinks[i] != null && sepLink[2] == "www.fly4free.pl") {

    var t2 = {
      travelTitle: null,
      travelTitle2: null,
      travelTitle3: null,
      travelDescription: null,
      travelDescription2: null,
      travelDescription3: null,
      travelBuy: null,
      travelBuy2: null,
      travelImage: null
    };

    var TravelLink1 = AllLinks[i];

    result = HTTP.get(AllLinks[i], {});
    $ = cheerio.load(result.content);

    t2.travelTitle = $('.article__title').text();
    t2.travelDescription = $('.article__content').find('p').first().text();

    if ($("img[src$='//www.fly4free.pl/wp-content/uploads/2016/09/lotJm.png']").parent().attr('href') != null) {
      t2.travelBuy = $("img[src$='//www.fly4free.pl/wp-content/uploads/2016/09/lotJm.png']").parent().attr('href'); // Link to buy
    }

    if (t2.travelBuy) {
      if (t2.travelBuy.split('https://').pop().split('http://').pop() != null) {
        t2.travelBuy2 = t2.travelBuy.split('https://').pop().split('http://').pop(); // link ready for DB
      } else {
        t2.travelBuy2 = t2.travelBuy;
      }
    }        

    t2.travelTitle3 = convertCurrencyInText(t2.travelTitle, 'PLN');
    t2.travelDescription3 = convertCurrencyInText(t2.travelDescription, 'PLN');

    translate(t2.travelTitle3, {from: 'pl', to: 'en'}).then(res => {
      t2.travelTitle2 = res.text; // title for DB
      if (t2.travelTitle2) { t2.travelImage = getPhoto(t2.travelTitle2); }

      translate(t2.travelDescription3, {from: 'pl', to: 'en'}).then(response => {
        t2.travelDescription2 = response.text; // description for DB

        if (t2.travelDescription2 != null && t2.travelTitle2 != null && t2.travelBuy2 != null && TravelLink1 != null && t2.travelImage != null) {
          Links.insert({ title: t2.travelTitle2, description:t2.travelDescription2, image: t2.travelImage, buyLink:t2.travelBuy2, link: TravelLink1, datetime: new Date() });
        } 

      }).catch(err => {
        console.error(err);
      });

    }).catch(err => {
      console.error(err);
    });

  }

}

“AllLinks”包含不同的网址。我在抓取此网址时遇到问题：http://www.fly4free.pl/na-wakacje-do-toskanii-tanie-loty-do-pizy-z-gdanska-za-170-pln/

getPhoto（）函数

    function getPhoto(title) {

  var travelPlace = nlp(title).match('to *').out('text').replace('to','').trim();
  if (travelPlace) {var travelPlace2 = travelPlace.split(' '); } 
  if (travelPlace2) {var travelPlace3 = travelPlace2[0] + "+" + travelPlace2[1]; } 
  if (travelPlace3) {
    var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+travelPlace3+"&category=travel&orientation=horizontal";
    var images = (HTTP.get(URL, {}));

    if (images.data.totalHits > 0) {
      var imageLink = images.data.hits[0].webformatURL;
      return imageLink;

    } else if (images.data.totalHits == 0) {
      var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+travelPlace2[0]+"&category=travel&orientation=horizontal";
      var images = (HTTP.get(URL, {}));

      if (images.data.totalHits > 0) {
        var imageLink = images.data.hits[0].webformatURL;
        return imageLink;
      }

    }
  } else if (nlp(title).places().data().length > 0) {
    var result = nlp(title).places().data()[0].text.replace(/[^a-zA-Z ]/g, "").trim();
    var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+result+"&category=travel&orientation=horizontal";
    var images = (HTTP.get(URL, {}));

    if (images.data.totalHits > 0) {
      var imageLink = images.data.hits[0].webformatURL;
      return imageLink;
    }

  } else {
    var title2 = title.replace(/[^a-zA-Z ]/g, "").split(" ");
    if (title2) {
      for(i = 0; i < title2.length; i++) {

        if (cities[title2[i]] == 1) {

          var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+title2[i]+"&category=travel&orientation=horizontal";
          var images = (HTTP.get(URL, {}));

          if (images.data.totalHits > 0) {
            var imageLink = images.data.hits[0].webformatURL;
            return imageLink;
          }

        } else {

          var URL = "https://pixabay.com/api/?key="+API_KEY+"&q=travel&category=travel&orientation=horizontal";
          var images = (HTTP.get(URL, {}));

          if (images.data.totalHits > 0) {
            var imageLink = images.data.hits[0].webformatURL;
            return imageLink;
          }


        }
      }
    }  
  }

}

我尝试控制记录结果 - 有时我从getPhoto（）获得正确的图像，但是来自t2.travelBuy的未定义链接，有时反之亦然。你能告诉我我做错了什么吗？我看到有些人在这类问题上使用Promises或async / await函数。你觉得那会对我有帮助吗？我应该如何更改我的代码以便在没有“未定义”的情况下抓取网站？

“translate”来自“google-translate-api”软件包

Answer 1

你可以尝试var new_func = Meteor.wrapAsync（你的CALLBACK功能）和你使用new_func（）的时候它将返回正常函数所期望的结果而不是等待回调

Meteor不会等待函数的结果，返回undefined

1 个答案: