Question

我想运行一个包含2个任务的dag，循环条件为1..10。一种方法是我可以在dag中动态生成具有不同taskId的任务。因此，最后我将不需要40个任务。因此，我创建了一个TriggerDagRunOperator，它将称为self dag。问题是我不知道如何设置计数更新，以便在下一次运行中可以获取更新的计数值。有什么我可以共享信息给下一个实例的东西吗？下面是代码：

import airflow
from airflow import DAG
from airflow.operators import BashOperator,PythonOperator
from datetime import datetime, timedelta
from airflow.operators.dagrun_operator import TriggerDagRunOperator

today = datetime.today()

dag = DAG(dag_id='main_dag', default_args={
        "owner": "airflow",
        "start_date": airflow.utils.dates.days_ago(2),
    },
    schedule_interval=None,
   )

# t1, t2 are examples of tasks created using operators




bash_task_1 = BashOperator(
    task_id="bash_task_1",
    bash_command='echo "Here is the message1: '
                 '{{ dag_run.conf["message"] if dag_run else "" }}" ',
    dag=dag)

bash_task_2 = BashOperator(
    task_id="bash_task_2",
    bash_command='echo "Here is the message2: '
                 '{{ dag_run.conf["message"] if dag_run else "" }}" ',
    dag=dag)


def run_this_func(**kwargs):
    """
    Print the payload "message" passed to the DagRun conf attribute.
    :param dict kwargs: Context
    """
    print("Remotely received value of {} for key=message".
          format(kwargs['dag_run'].conf['counter']))
    return kwargs['dag_run'].conf['counter']    


def conditionally_trigger(context, dag_run_obj):
    """
    This function decides whether or not to Trigger the remote DAG
    """             
    counter = context['params']['counter']
    print("Controller DAG : conditionally_trigger = {}".format(c_p))
    while counter < 3:
       counter += 1 # Where should i set this back to be available for next instance      
       return dag_run_obj
    return None   


trigger = TriggerDagRunOperator(
     task_id='trigger_job1',
     trigger_dag_id="main_dag",
     python_callable=conditionally_trigger,
     params={'counter': 1, 'message': 'Hello'},
     dag=dag)

bash_task_1 >> bash_task_2 >> trigger

循环条件下的气流自我DAG触发器

0 个答案: