我不会通过两个具有单个相互依赖关系的任务来运行两个SQL语句,但是在此过程中,我遇到了“未填充依赖关系错误”(请参见下文):
最初,我认为这是由于写入输出时出现问题,但看起来两个任务都已成功写入数据库中的目标表,所以我不确定记录器为什么会抛出此错误?
这是我的简单管道的设置:
class RunSessionsTable(luigi.Task):
job_name = 'Creating_Sessions_Table' # unique task name or table impacted
date_param = luigi.DateParameter(default=datetime.date.today()) # today's date
def generate_job_number(self):
pacific = pytz.timezone("US/Mountain")
return pacific.localize(datetime.datetime.now()).strftime('%H')
def get_target(self):
update_id = '{}_{}_{}'.format(self.job_name,self.date_param, self.generate_job_number())
return ToSnowflake (connection_string = ToSnowflake.connection_string, target_table= self.job_name, update_id = update_id)
def output(self):
return self.get_target().touch()
def run (self):
init = ToSnowflake.engine
conn = init.connect()
results = conn.execute(sessions_query)
#conn.close()
return self.output()
class RunEventsTable(luigi.Task):
job_name = 'Creating_Events_Table' # unique task name or table impacted
date_param = luigi.DateParameter(default=datetime.date.today()) # today's date
def generate_job_number(self):
pacific = pytz.timezone("US/Mountain")
return pacific.localize(datetime.datetime.now()).strftime('%H')
def requires(self):
return [RunSessionsTable()]
def get_target(self):
update_id = '{}_{}_{}'.format(self.job_name,self.date_param, self.generate_job_number())
return ToSnowflake (connection_string = ToSnowflake.connection_string, target_table= self.job_name, update_id = update_id)
def output(self):
return self.get_target().touch()
def run (self):
init = ToSnowflake.engine
conn = init.connect()
results = conn.execute(events_query)
#conn.close()
return self.output()
if __name__ == '__main__':
luigi.build([RunEventsTable()])