Question

我们正在尝试使用node.js google_cloud sdk从Google云端存储中将数据加载到BigQuery中。以下是加载代码的片段：

//init Google Cloud storage
var gcs = require("@google-cloud/storage")({
  projectId: bq_projectId,
  keyFilename: "./" + keyfile
});

//init Google BigQuery
var bq = require("@google-cloud/bigquery")({
  projectId: bq_projectId,
  keyFilename: "./" + keyfile
});

const datasetId = "my-dataset";
const tableId = "data_load";
const fileName = "data-20170518-082238.csv";

bq
  .dataset(datasetId)
  .table(tableId)
  .import(gcs.bucket(bucketName).file(fileName), (err, job, apiResponse) => {
    if (err){
      throw err
      return;
    }

    console.log(apiResponse);
    console.log(`Job ${job.id} started.`);
  });

作业启动并返回jobId，不会抛出任何错误，apiResponse为：

{ kind: 'bigquery#job',
  etag: '"TcVKUQ2ft7DS9Q8U3noJdmpEDQ4/tFIJTWM2yuacXB5EvzWR1ffuKig"',
  id: 'my-project:job_FTO4_Jb5ctr2oEy2IsDSAUCWrgw',
  selfLink: 'https://www.googleapis.com/bigquery/v2/projects/my-project/jobs/job_FTO4_Jb5ctr2oEy2IsDSAUCWrgw',
  jobReference:
   { projectId: 'my-project',
     jobId: 'job_FTO4_Jb5ctr2oEy2IsDSAUCWrgw' },
  configuration:
   { load:
      { sourceUris: [Object],
        schema: [Object],
        destinationTable: [Object],
        sourceFormat: 'CSV' } },
  status: { state: 'RUNNING' },
  statistics: { creationTime: '1495151832686', startTime: '1495151833523' },
  user_email: 'service-account' }

完全相同的文件从BigQuery接口加载到指定的表中。

Answer 1

事实证明，BigQuery加载过程中的错误不会通过回调或promise.catch抛出。

在这种情况下，您需要读取job.status.errors对象以确定加载中是否存在错误，以下是一些示例代码：

bq
  .dataset(datasetId)
  .table(tableId)
  .import(gcs.bucket(bucketName).file(fileName),{
    skipLeadingRows: 1
  })
  .then(results => {
    console.log("initial results:")
    console.log(results);
    const job = results[0];
    console.log(`Job ${job.id} started.`);
    return job.promise();
  })
  .then(results => {
    console.log("completed results:")
    console.log(results);

    const job = results[0];

    // determine if there are any errors in the completed job
    if(job.status.errorResult || (job.status.errors && job.status.errors.count > 0)) {
      console.error(JSON.stringify(job.status.errors));
      return;
    }

    console.log(`Job ${job.id} completed successfully`);
  })
  .catch(err => {
    console.error(err);
  });

Google BigQuery Node SDK加载作业失败，没有错误

1 个答案: