Question

我想逐行读取大文件并将数据插入DB存储区。我的功能在其创建的流中返回一个Promise，并在调用事件stream.on('end')时解决它，但这不是我真正想要的，因为在stream.on('data')中它在每一行上产生Promise.map()，并且我想确保在调用resolve()之前完成所有插入操作。在这种情况下我如何能够生产出正确的链？

    var loadFromFile = (options) => new Promise(function (resolve, reject) {
        let stream = fs.createReadStream(options.filePath, {
          flags: 'r',
        });

        stream.on('data', (chunk) => {
            /* process data chunk to string Array */  
            Promise.map(lines, (line) => {
            /* process and insert each line here */
            })
            .catch((err)=>{
                reject(err);
            });
          });

          stream.on('end', () => {
              if (someBuisnessLogicCheckHere) {
                reject('Invalid data was found in file');
              }
              resolve(); // Here I am not sure, that all inserts for each chunk in Promise.map was completed
          });
    });

Answer 1

如果您想确定在映射操作中的所有承诺都已解决之前无法解决，请等待Promise.map返回的承诺;见评论：

var loadFromFile = (options) => new Promise(function(resolve, reject) {
    let stream = fs.createReadStream(options.filePath, {
        flags: 'r',
    });
    let promises = [];                                 // Array of promises we'll wait for

    stream.on('data', (chunk) => {
        promises.push(                                 // Remember this promise
            Promise.map(lines, (line) => {
                /* process and insert each line here */
            })
            .catch((err) => {
                reject(err);
            })
        );
    });

    stream.on('end', () => {
        if (someBuisnessLogicCheckHere) {
            reject('Invalid data was found in file');
        }
        Promise.all(promises).then(() => { resolve()}); // Wait for it before resolving
    });
});

请注意，我没有Promise.all(promises).then(resolve);，因为这会将一系列分辨率传递给resolve，原始代码与调用者无法分享。

如果阵列在填充大多数已经解决的承诺时会变得非常大，你可以在他们解决时主动删除它们，只等待最后留下的那些。

Answer 2

如果您需要在管道方法中使用自动暂停/恢复机制处理文件，请查看scramjet。

你的例子看起来会更清楚，见：

var loadFromFile = (options) => 
    fs.createReadStream(options.filePath, {
        flags: 'r',
    })
    .split(/\r?\n/) 
         // split by line by regex
    .parse((line) => aPromiseReturningFunction(line))
         // will wait until promises are resolved
    .map((data) => doSomeAsyncAndReturnPromise(data))      
         // will wait as above
    .accumulate((acc, resolved) => acc.push(resolved), [])
    .then((arr) => {
         if (someBuisnessLogicCheckHere) {
              return Promise.reject('Invalid data was found in file');
         }
         // Here all your items are saved (i.e. all Promises resolved)
     });
});

承诺结合流

2 个答案: