Source code for tests.system.google.cloud.dataflow.example_dataflow_template
## Licensed to the Apache Software Foundation (ASF) under one# or more contributor license agreements. See the NOTICE file# distributed with this work for additional information# regarding copyright ownership. The ASF licenses this file# to you under the Apache License, Version 2.0 (the# "License"); you may not use this file except in compliance# with the License. You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing,# software distributed under the License is distributed on an# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY# KIND, either express or implied. See the License for the# specific language governing permissions and limitations# under the License."""Example Airflow DAG for testing Google Dataflow.:class:`~airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator` operator."""from__future__importannotationsimportosfromdatetimeimportdatetimefrompathlibimportPathfromairflow.models.baseoperatorimportchainfromairflow.models.dagimportDAGfromairflow.providers.google.cloud.operators.dataflowimport(DataflowStartFlexTemplateOperator,DataflowTemplatedJobStartOperator,)fromairflow.providers.google.cloud.operators.gcsimportGCSCreateBucketOperator,GCSDeleteBucketOperatorfromairflow.providers.google.cloud.transfers.local_to_gcsimportLocalFilesystemToGCSOperatorfromairflow.utils.trigger_ruleimportTriggerRulefromsystem.googleimportDEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
upload_file=LocalFilesystemToGCSOperator(task_id="upload_file_to_bucket",src=PYTHON_FILE_LOCAL_PATH,dst=CSV_FILE_NAME,bucket=BUCKET_NAME,)upload_schema=LocalFilesystemToGCSOperator(task_id="upload_schema_to_bucket",src=SCHEMA_LOCAL_PATH,dst=AVRO_SCHEMA,bucket=BUCKET_NAME,)# [START howto_operator_start_template_job]start_template_job=DataflowTemplatedJobStartOperator(task_id="start_template_job",project_id=PROJECT_ID,template="gs://dataflow-templates/latest/Word_Count",parameters={"inputFile":f"gs://{BUCKET_NAME}/{CSV_FILE_NAME}","output":GCS_OUTPUT},location=LOCATION,wait_until_finished=True,)# [END howto_operator_start_template_job]# [START howto_operator_start_flex_template_job]start_flex_template_job=DataflowStartFlexTemplateOperator(task_id="start_flex_template_job",project_id=PROJECT_ID,body=BODY,location=LOCATION,append_job_name=False,wait_until_finished=True,)# [END howto_operator_start_flex_template_job]# [START howto_operator_start_template_job_deferrable]start_template_job_deferrable=DataflowTemplatedJobStartOperator(task_id="start_template_job_deferrable",project_id=PROJECT_ID,template="gs://dataflow-templates/latest/Word_Count",parameters={"inputFile":f"gs://{BUCKET_NAME}/{CSV_FILE_NAME}","output":GCS_OUTPUT},location=LOCATION,deferrable=True,)# [END howto_operator_start_template_job_deferrable]# [START howto_operator_start_flex_template_job_deferrable]start_flex_template_job_deferrable=DataflowStartFlexTemplateOperator(task_id="start_flex_template_job_deferrable",project_id=PROJECT_ID,body=BODY,location=LOCATION,append_job_name=False,deferrable=True,)# [END howto_operator_start_flex_template_job_deferrable]delete_bucket=GCSDeleteBucketOperator(task_id="delete_bucket",bucket_name=BUCKET_NAME,trigger_rule=TriggerRule.ALL_DONE)chain(create_bucket,upload_file,upload_schema,[start_template_job,start_flex_template_job],[start_template_job_deferrable,start_flex_template_job_deferrable],delete_bucket,)fromtests_common.test_utils.watcherimportwatcher# This test needs watcher in order to properly mark success/failure# when "teardown" task with trigger rule is part of the DAGlist(dag.tasks)>>watcher()fromtests_common.test_utils.system_testsimportget_test_run# noqa: E402# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)