# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
#
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
[docs]
def get_provider_info():
return {
"package-name": "apache-airflow-providers-amazon",
"name": "Amazon",
"description": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
"state": "ready",
"source-date-epoch": 1739958314,
"versions": [
"9.4.0",
"9.2.0",
"9.1.0",
"9.0.0",
"8.29.0",
"8.28.0",
"8.27.0",
"8.26.0",
"8.25.0",
"8.24.0",
"8.23.0",
"8.22.0",
"8.21.0",
"8.20.0",
"8.19.0",
"8.18.0",
"8.17.0",
"8.16.0",
"8.15.0",
"8.14.0",
"8.13.0",
"8.12.0",
"8.11.0",
"8.10.0",
"8.9.0",
"8.8.0",
"8.7.1",
"8.7.0",
"8.6.0",
"8.5.1",
"8.5.0",
"8.4.0",
"8.3.1",
"8.3.0",
"8.2.0",
"8.1.0",
"8.0.0",
"7.4.1",
"7.4.0",
"7.3.0",
"7.2.1",
"7.2.0",
"7.1.0",
"7.0.0",
"6.2.0",
"6.1.0",
"6.0.0",
"5.1.0",
"5.0.0",
"4.1.0",
"4.0.0",
"3.4.0",
"3.3.0",
"3.2.0",
"3.1.1",
"3.0.0",
"2.6.0",
"2.5.0",
"2.4.0",
"2.3.0",
"2.2.0",
"2.1.0",
"2.0.0",
"1.4.0",
"1.3.0",
"1.2.0",
"1.1.0",
"1.0.0",
],
"integrations": [
{
"integration-name": "Amazon Athena",
"external-doc-url": "https://aws.amazon.com/athena/",
"logo": "/docs/integration-logos/Amazon-Athena_light-bg@4x.png",
"how-to-guide": [
"/docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst",
"/docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst",
],
"tags": ["aws"],
},
{
"integration-name": "Amazon Bedrock",
"external-doc-url": "https://aws.amazon.com/bedrock/",
"logo": "/docs/integration-logos/Amazon-Bedrock_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/bedrock.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Chime",
"external-doc-url": "https://aws.amazon.com/chime/",
"logo": "/docs/integration-logos/Amazon-Chime-light-bg.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon CloudFormation",
"external-doc-url": "https://aws.amazon.com/cloudformation/",
"logo": "/docs/integration-logos/AWS-CloudFormation_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/cloudformation.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon CloudWatch Logs",
"external-doc-url": "https://aws.amazon.com/cloudwatch/",
"logo": "/docs/integration-logos/Amazon-CloudWatch_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Comprehend",
"external-doc-url": "https://aws.amazon.com/comprehend/",
"logo": "/docs/integration-logos/Amazon-Comprehend_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/comprehend.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon DataSync",
"external-doc-url": "https://aws.amazon.com/datasync/",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/datasync.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon DynamoDB",
"external-doc-url": "https://aws.amazon.com/dynamodb/",
"logo": "/docs/integration-logos/Amazon-DynamoDB_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/dynamodb.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon EC2",
"external-doc-url": "https://aws.amazon.com/ec2/",
"logo": "/docs/integration-logos/Amazon-EC2_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/ec2.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Elastic Container Registry (ECR)",
"external-doc-url": "https://aws.amazon.com/ecr/",
"logo": "/docs/integration-logos/Amazon-Elastic-Container-Registry_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon ECS",
"external-doc-url": "https://aws.amazon.com/ecs/",
"logo": "/docs/integration-logos/Amazon-Elastic-Container-Service_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Elastic Kubernetes Service (EKS)",
"external-doc-url": "https://aws.amazon.com/eks/",
"logo": "/docs/integration-logos/Amazon-Elastic-Kubernetes-Service_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/eks.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon ElastiCache",
"external-doc-url": "https://aws.amazon.com/elasticache/redis//",
"logo": "/docs/integration-logos/Amazon-ElastiCache_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon EMR",
"external-doc-url": "https://aws.amazon.com/emr/",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/emr/emr.rst"],
"logo": "/docs/integration-logos/Amazon-EMR_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon EMR on EKS",
"external-doc-url": "https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/emr-eks.html",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst"],
"logo": "/docs/integration-logos/Amazon-EMR_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon EMR Serverless",
"external-doc-url": "https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/emr-serverless.html",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst"],
"logo": "/docs/integration-logos/Amazon-EMR_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon EventBridge",
"external-doc-url": "https://docs.aws.amazon.com/eventbridge/latest/APIReference/Welcome.html",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/eventbridge.rst"],
"logo": "/docs/integration-logos/Amazon-EventBridge_64.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Glacier",
"external-doc-url": "https://aws.amazon.com/glacier/",
"logo": "/docs/integration-logos/Amazon-S3-Glacier_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Kinesis Data Firehose",
"external-doc-url": "https://aws.amazon.com/kinesis/data-firehose/",
"logo": "/docs/integration-logos/Amazon-Kinesis-Data-Firehose_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Managed Service for Apache Flink",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst"],
"external-doc-url": "https://aws.amazon.com/managed-service-apache-flink/",
"logo": "/docs/integration-logos/Amazon-Kinesis-Analytics_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon OpenSearch Serverless",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst"],
"external-doc-url": "https://aws.amazon.com/opensearchserverless/",
"logo": "/docs/integration-logos/Amazon-OpenSearch_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon RDS",
"external-doc-url": "https://aws.amazon.com/rds/",
"logo": "/docs/integration-logos/Amazon-RDS_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/rds.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Redshift",
"external-doc-url": "https://aws.amazon.com/redshift/",
"logo": "/docs/integration-logos/Amazon-Redshift_light-bg@4x.png",
"how-to-guide": [
"/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst",
"/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst",
],
"tags": ["aws"],
},
{
"integration-name": "Amazon Redshift Data",
"external-doc-url": "https://aws.amazon.com/redshift/",
"logo": "/docs/integration-logos/Amazon-Redshift_light-bg@4x.png",
"how-to-guide": [
"/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst"
],
"tags": ["aws"],
},
{
"integration-name": "Amazon SageMaker",
"external-doc-url": "https://aws.amazon.com/sagemaker/",
"logo": "/docs/integration-logos/Amazon-SageMaker_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/sagemaker.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon SecretsManager",
"external-doc-url": "https://aws.amazon.com/secrets-manager/",
"logo": "/docs/integration-logos/AWS-Secrets-Manager_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Simple Email Service (SES)",
"external-doc-url": "https://aws.amazon.com/ses/",
"logo": "/docs/integration-logos/Amazon-Simple-Email-Service-SES_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/ecs.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Simple Notification Service (SNS)",
"external-doc-url": "https://aws.amazon.com/sns/",
"logo": "/docs/integration-logos/Amazon-Simple-Notification-Service-SNS_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/sns.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Simple Queue Service (SQS)",
"external-doc-url": "https://aws.amazon.com/sqs/",
"logo": "/docs/integration-logos/Amazon-Simple-Queue-Service-SQS_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/sqs.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Simple Storage Service (S3)",
"external-doc-url": "https://aws.amazon.com/s3/",
"logo": "/docs/integration-logos/Amazon-Simple-Storage-Service-S3_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/s3/s3.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon Systems Manager (SSM)",
"external-doc-url": "https://aws.amazon.com/systems-manager/",
"logo": "/docs/integration-logos/AWS-Systems-Manager_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Web Services",
"external-doc-url": "https://aws.amazon.com/",
"logo": "/docs/integration-logos/AWS-Cloud-alt_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "AWS Batch",
"external-doc-url": "https://aws.amazon.com/batch/",
"logo": "/docs/integration-logos/AWS-Batch_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/batch.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS DataSync",
"external-doc-url": "https://aws.amazon.com/datasync/",
"logo": "/docs/integration-logos/AWS-DataSync_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "AWS Glue",
"external-doc-url": "https://aws.amazon.com/glue/",
"logo": "/docs/integration-logos/AWS-Glue_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/glue.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS Lambda",
"external-doc-url": "https://aws.amazon.com/lambda/",
"logo": "/docs/integration-logos/AWS-Lambda_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/lambda.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS Step Functions",
"external-doc-url": "https://aws.amazon.com/step-functions/",
"logo": "/docs/integration-logos/AWS-Step-Functions_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/step_functions.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS Database Migration Service",
"external-doc-url": "https://aws.amazon.com/dms/",
"logo": "/docs/integration-logos/AWS-Database-Migration-Service_64@5x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/dms.rst"],
"tags": ["aws"],
},
{
"integration-name": "Amazon QuickSight",
"external-doc-url": "https://aws.amazon.com/quicksight/",
"logo": "/docs/integration-logos/Amazon-Quicksight_light-bg@4x.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/quicksight.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS Security Token Service (STS)",
"external-doc-url": "https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html",
"logo": "/docs/integration-logos/AWS-STS_light-bg@4x.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Appflow",
"external-doc-url": "https://docs.aws.amazon.com/appflow/1.0/APIReference/Welcome.html",
"logo": "/docs/integration-logos/Amazon_AppFlow_light.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/appflow.rst"],
"tags": ["aws"],
},
{
"integration-name": "AWS Glue DataBrew",
"external-doc-url": "https://docs.aws.amazon.com/databrew/latest/dg/what-is.html",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/glue_databrew.rst"],
"logo": "/docs/integration-logos/AWS-Glue-DataBrew_64.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Verified Permissions",
"external-doc-url": "https://aws.amazon.com/verified-permissions/",
"logo": "/docs/integration-logos/Amazon-Verified-Permissions.png",
"tags": ["aws"],
},
{
"integration-name": "Amazon Neptune",
"external-doc-url": "https://aws.amazon.com/neptune/",
"logo": "/docs/integration-logos/Amazon-Neptune_64.png",
"how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/neptune.rst"],
"tags": ["aws"],
},
],
"operators": [
{
"integration-name": "Amazon Athena",
"python-modules": ["airflow.providers.amazon.aws.operators.athena"],
},
{
"integration-name": "Amazon Web Services",
"python-modules": ["airflow.providers.amazon.aws.operators.base_aws"],
},
{
"integration-name": "AWS Batch",
"python-modules": ["airflow.providers.amazon.aws.operators.batch"],
},
{
"integration-name": "Amazon Bedrock",
"python-modules": ["airflow.providers.amazon.aws.operators.bedrock"],
},
{
"integration-name": "Amazon CloudFormation",
"python-modules": ["airflow.providers.amazon.aws.operators.cloud_formation"],
},
{
"integration-name": "Amazon Comprehend",
"python-modules": ["airflow.providers.amazon.aws.operators.comprehend"],
},
{
"integration-name": "Amazon DataSync",
"python-modules": ["airflow.providers.amazon.aws.operators.datasync"],
},
{
"integration-name": "AWS Database Migration Service",
"python-modules": ["airflow.providers.amazon.aws.operators.dms"],
},
{
"integration-name": "Amazon EC2",
"python-modules": ["airflow.providers.amazon.aws.operators.ec2"],
},
{
"integration-name": "Amazon ECS",
"python-modules": ["airflow.providers.amazon.aws.operators.ecs"],
},
{
"integration-name": "Amazon Elastic Kubernetes Service (EKS)",
"python-modules": ["airflow.providers.amazon.aws.operators.eks"],
},
{
"integration-name": "Amazon EMR",
"python-modules": ["airflow.providers.amazon.aws.operators.emr"],
},
{
"integration-name": "Amazon EMR on EKS",
"python-modules": ["airflow.providers.amazon.aws.operators.emr"],
},
{
"integration-name": "Amazon EventBridge",
"python-modules": ["airflow.providers.amazon.aws.operators.eventbridge"],
},
{
"integration-name": "Amazon Glacier",
"python-modules": ["airflow.providers.amazon.aws.operators.glacier"],
},
{
"integration-name": "AWS Glue",
"python-modules": [
"airflow.providers.amazon.aws.operators.glue",
"airflow.providers.amazon.aws.operators.glue_crawler",
],
},
{
"integration-name": "AWS Lambda",
"python-modules": ["airflow.providers.amazon.aws.operators.lambda_function"],
},
{
"integration-name": "Amazon Managed Service for Apache Flink",
"python-modules": ["airflow.providers.amazon.aws.operators.kinesis_analytics"],
},
{
"integration-name": "Amazon Managed Workflows for Apache Airflow (MWAA)",
"python-modules": ["airflow.providers.amazon.aws.operators.mwaa"],
},
{
"integration-name": "Amazon Simple Storage Service (S3)",
"python-modules": ["airflow.providers.amazon.aws.operators.s3"],
},
{
"integration-name": "Amazon SageMaker",
"python-modules": ["airflow.providers.amazon.aws.operators.sagemaker"],
},
{
"integration-name": "Amazon Simple Notification Service (SNS)",
"python-modules": ["airflow.providers.amazon.aws.operators.sns"],
},
{
"integration-name": "Amazon Simple Queue Service (SQS)",
"python-modules": ["airflow.providers.amazon.aws.operators.sqs"],
},
{
"integration-name": "AWS Step Functions",
"python-modules": ["airflow.providers.amazon.aws.operators.step_function"],
},
{
"integration-name": "Amazon RDS",
"python-modules": ["airflow.providers.amazon.aws.operators.rds"],
},
{
"integration-name": "Amazon Redshift",
"python-modules": [
"airflow.providers.amazon.aws.operators.redshift_cluster",
"airflow.providers.amazon.aws.operators.redshift_data",
],
},
{
"integration-name": "Amazon QuickSight",
"python-modules": ["airflow.providers.amazon.aws.operators.quicksight"],
},
{
"integration-name": "Amazon Appflow",
"python-modules": ["airflow.providers.amazon.aws.operators.appflow"],
},
{
"integration-name": "AWS Glue DataBrew",
"python-modules": ["airflow.providers.amazon.aws.operators.glue_databrew"],
},
{
"integration-name": "Amazon Neptune",
"python-modules": ["airflow.providers.amazon.aws.operators.neptune"],
},
],
"sensors": [
{
"integration-name": "Amazon Athena",
"python-modules": ["airflow.providers.amazon.aws.sensors.athena"],
},
{
"integration-name": "Amazon Web Services",
"python-modules": ["airflow.providers.amazon.aws.sensors.base_aws"],
},
{
"integration-name": "AWS Batch",
"python-modules": ["airflow.providers.amazon.aws.sensors.batch"],
},
{
"integration-name": "Amazon Bedrock",
"python-modules": ["airflow.providers.amazon.aws.sensors.bedrock"],
},
{
"integration-name": "Amazon CloudFormation",
"python-modules": ["airflow.providers.amazon.aws.sensors.cloud_formation"],
},
{
"integration-name": "Amazon Comprehend",
"python-modules": ["airflow.providers.amazon.aws.sensors.comprehend"],
},
{
"integration-name": "AWS Database Migration Service",
"python-modules": ["airflow.providers.amazon.aws.sensors.dms"],
},
{
"integration-name": "Amazon DynamoDB",
"python-modules": ["airflow.providers.amazon.aws.sensors.dynamodb"],
},
{
"integration-name": "Amazon EC2",
"python-modules": ["airflow.providers.amazon.aws.sensors.ec2"],
},
{
"integration-name": "Amazon ECS",
"python-modules": ["airflow.providers.amazon.aws.sensors.ecs"],
},
{
"integration-name": "Amazon Elastic Kubernetes Service (EKS)",
"python-modules": ["airflow.providers.amazon.aws.sensors.eks"],
},
{
"integration-name": "Amazon EMR",
"python-modules": ["airflow.providers.amazon.aws.sensors.emr"],
},
{
"integration-name": "Amazon EMR on EKS",
"python-modules": ["airflow.providers.amazon.aws.sensors.emr"],
},
{
"integration-name": "Amazon Glacier",
"python-modules": ["airflow.providers.amazon.aws.sensors.glacier"],
},
{
"integration-name": "AWS Glue",
"python-modules": [
"airflow.providers.amazon.aws.sensors.glue",
"airflow.providers.amazon.aws.sensors.glue_crawler",
"airflow.providers.amazon.aws.sensors.glue_catalog_partition",
],
},
{
"integration-name": "AWS Lambda",
"python-modules": ["airflow.providers.amazon.aws.sensors.lambda_function"],
},
{
"integration-name": "Amazon Managed Service for Apache Flink",
"python-modules": ["airflow.providers.amazon.aws.sensors.kinesis_analytics"],
},
{
"integration-name": "Amazon OpenSearch Serverless",
"python-modules": ["airflow.providers.amazon.aws.sensors.opensearch_serverless"],
},
{
"integration-name": "Amazon RDS",
"python-modules": ["airflow.providers.amazon.aws.sensors.rds"],
},
{
"integration-name": "Amazon Redshift",
"python-modules": ["airflow.providers.amazon.aws.sensors.redshift_cluster"],
},
{
"integration-name": "Amazon Simple Storage Service (S3)",
"python-modules": ["airflow.providers.amazon.aws.sensors.s3"],
},
{
"integration-name": "Amazon SageMaker",
"python-modules": ["airflow.providers.amazon.aws.sensors.sagemaker"],
},
{
"integration-name": "Amazon Simple Queue Service (SQS)",
"python-modules": ["airflow.providers.amazon.aws.sensors.sqs"],
},
{
"integration-name": "AWS Step Functions",
"python-modules": ["airflow.providers.amazon.aws.sensors.step_function"],
},
{
"integration-name": "Amazon QuickSight",
"python-modules": ["airflow.providers.amazon.aws.sensors.quicksight"],
},
],
"asset-uris": [
{
"schemes": ["s3"],
"handler": "airflow.providers.amazon.aws.assets.s3.sanitize_uri",
"to_openlineage_converter": "airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage",
"factory": "airflow.providers.amazon.aws.assets.s3.create_asset",
}
],
"dataset-uris": [
{
"schemes": ["s3"],
"handler": "airflow.providers.amazon.aws.assets.s3.sanitize_uri",
"to_openlineage_converter": "airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage",
"factory": "airflow.providers.amazon.aws.assets.s3.create_asset",
}
],
"filesystems": ["airflow.providers.amazon.aws.fs.s3"],
"hooks": [
{
"integration-name": "Amazon Athena",
"python-modules": [
"airflow.providers.amazon.aws.hooks.athena",
"airflow.providers.amazon.aws.hooks.athena_sql",
],
},
{
"integration-name": "Amazon Bedrock",
"python-modules": ["airflow.providers.amazon.aws.hooks.bedrock"],
},
{
"integration-name": "Amazon Chime",
"python-modules": ["airflow.providers.amazon.aws.hooks.chime"],
},
{
"integration-name": "Amazon Comprehend",
"python-modules": ["airflow.providers.amazon.aws.hooks.comprehend"],
},
{
"integration-name": "Amazon DynamoDB",
"python-modules": ["airflow.providers.amazon.aws.hooks.dynamodb"],
},
{
"integration-name": "Amazon Web Services",
"python-modules": ["airflow.providers.amazon.aws.hooks.base_aws"],
},
{
"integration-name": "AWS Batch",
"python-modules": [
"airflow.providers.amazon.aws.hooks.batch_client",
"airflow.providers.amazon.aws.hooks.batch_waiters",
],
},
{
"integration-name": "Amazon CloudFormation",
"python-modules": ["airflow.providers.amazon.aws.hooks.cloud_formation"],
},
{
"integration-name": "Amazon DataSync",
"python-modules": ["airflow.providers.amazon.aws.hooks.datasync"],
},
{
"integration-name": "AWS Database Migration Service",
"python-modules": ["airflow.providers.amazon.aws.hooks.dms"],
},
{"integration-name": "Amazon EC2", "python-modules": ["airflow.providers.amazon.aws.hooks.ec2"]},
{
"integration-name": "Amazon Elastic Container Registry (ECR)",
"python-modules": ["airflow.providers.amazon.aws.hooks.ecr"],
},
{"integration-name": "Amazon ECS", "python-modules": ["airflow.providers.amazon.aws.hooks.ecs"]},
{
"integration-name": "Amazon ElastiCache",
"python-modules": ["airflow.providers.amazon.aws.hooks.elasticache_replication_group"],
},
{
"integration-name": "Amazon Elastic Kubernetes Service (EKS)",
"python-modules": ["airflow.providers.amazon.aws.hooks.eks"],
},
{"integration-name": "Amazon EMR", "python-modules": ["airflow.providers.amazon.aws.hooks.emr"]},
{
"integration-name": "Amazon EMR on EKS",
"python-modules": ["airflow.providers.amazon.aws.hooks.emr"],
},
{
"integration-name": "Amazon EventBridge",
"python-modules": ["airflow.providers.amazon.aws.hooks.eventbridge"],
},
{
"integration-name": "Amazon Glacier",
"python-modules": ["airflow.providers.amazon.aws.hooks.glacier"],
},
{
"integration-name": "AWS Glue",
"python-modules": [
"airflow.providers.amazon.aws.hooks.glue",
"airflow.providers.amazon.aws.hooks.glue_crawler",
"airflow.providers.amazon.aws.hooks.glue_catalog",
],
},
{
"integration-name": "Amazon Kinesis Data Firehose",
"python-modules": ["airflow.providers.amazon.aws.hooks.kinesis"],
},
{
"integration-name": "AWS Lambda",
"python-modules": ["airflow.providers.amazon.aws.hooks.lambda_function"],
},
{
"integration-name": "Amazon Managed Service for Apache Flink",
"python-modules": ["airflow.providers.amazon.aws.hooks.kinesis_analytics"],
},
{
"integration-name": "Amazon CloudWatch Logs",
"python-modules": ["airflow.providers.amazon.aws.hooks.logs"],
},
{
"integration-name": "Amazon Managed Workflows for Apache Airflow (MWAA)",
"python-modules": ["airflow.providers.amazon.aws.hooks.mwaa"],
},
{
"integration-name": "Amazon OpenSearch Serverless",
"python-modules": ["airflow.providers.amazon.aws.hooks.opensearch_serverless"],
},
{"integration-name": "Amazon RDS", "python-modules": ["airflow.providers.amazon.aws.hooks.rds"]},
{
"integration-name": "Amazon Redshift",
"python-modules": [
"airflow.providers.amazon.aws.hooks.redshift_sql",
"airflow.providers.amazon.aws.hooks.redshift_cluster",
"airflow.providers.amazon.aws.hooks.redshift_data",
],
},
{
"integration-name": "Amazon Simple Storage Service (S3)",
"python-modules": ["airflow.providers.amazon.aws.hooks.s3"],
},
{
"integration-name": "Amazon SageMaker",
"python-modules": ["airflow.providers.amazon.aws.hooks.sagemaker"],
},
{
"integration-name": "Amazon Simple Email Service (SES)",
"python-modules": ["airflow.providers.amazon.aws.hooks.ses"],
},
{
"integration-name": "Amazon Systems Manager (SSM)",
"python-modules": ["airflow.providers.amazon.aws.hooks.ssm"],
},
{
"integration-name": "Amazon SecretsManager",
"python-modules": ["airflow.providers.amazon.aws.hooks.secrets_manager"],
},
{
"integration-name": "Amazon Simple Notification Service (SNS)",
"python-modules": ["airflow.providers.amazon.aws.hooks.sns"],
},
{
"integration-name": "Amazon Simple Queue Service (SQS)",
"python-modules": ["airflow.providers.amazon.aws.hooks.sqs"],
},
{
"integration-name": "AWS Step Functions",
"python-modules": ["airflow.providers.amazon.aws.hooks.step_function"],
},
{
"integration-name": "Amazon QuickSight",
"python-modules": ["airflow.providers.amazon.aws.hooks.quicksight"],
},
{
"integration-name": "AWS Security Token Service (STS)",
"python-modules": ["airflow.providers.amazon.aws.hooks.sts"],
},
{
"integration-name": "Amazon Appflow",
"python-modules": ["airflow.providers.amazon.aws.hooks.appflow"],
},
{
"integration-name": "AWS Glue DataBrew",
"python-modules": ["airflow.providers.amazon.aws.hooks.glue_databrew"],
},
{
"integration-name": "Amazon Verified Permissions",
"python-modules": ["airflow.providers.amazon.aws.hooks.verified_permissions"],
},
{
"integration-name": "Amazon Neptune",
"python-modules": ["airflow.providers.amazon.aws.hooks.neptune"],
},
],
"triggers": [
{
"integration-name": "Amazon Web Services",
"python-modules": ["airflow.providers.amazon.aws.triggers.base"],
},
{
"integration-name": "Amazon Athena",
"python-modules": ["airflow.providers.amazon.aws.triggers.athena"],
},
{
"integration-name": "AWS Batch",
"python-modules": ["airflow.providers.amazon.aws.triggers.batch"],
},
{
"integration-name": "Amazon Bedrock",
"python-modules": ["airflow.providers.amazon.aws.triggers.bedrock"],
},
{
"integration-name": "Amazon Comprehend",
"python-modules": ["airflow.providers.amazon.aws.triggers.comprehend"],
},
{
"integration-name": "Amazon EC2",
"python-modules": ["airflow.providers.amazon.aws.triggers.ec2"],
},
{
"integration-name": "AWS Lambda",
"python-modules": ["airflow.providers.amazon.aws.triggers.lambda_function"],
},
{
"integration-name": "Amazon Managed Service for Apache Flink",
"python-modules": ["airflow.providers.amazon.aws.triggers.kinesis_analytics"],
},
{
"integration-name": "Amazon OpenSearch Serverless",
"python-modules": ["airflow.providers.amazon.aws.triggers.opensearch_serverless"],
},
{
"integration-name": "Amazon Redshift",
"python-modules": [
"airflow.providers.amazon.aws.triggers.redshift_cluster",
"airflow.providers.amazon.aws.triggers.redshift_data",
],
},
{
"integration-name": "Amazon SageMaker",
"python-modules": ["airflow.providers.amazon.aws.triggers.sagemaker"],
},
{
"integration-name": "AWS Glue",
"python-modules": [
"airflow.providers.amazon.aws.triggers.glue",
"airflow.providers.amazon.aws.triggers.glue_crawler",
],
},
{
"integration-name": "Amazon Simple Storage Service (S3)",
"python-modules": ["airflow.providers.amazon.aws.triggers.s3"],
},
{
"integration-name": "Amazon EMR",
"python-modules": ["airflow.providers.amazon.aws.triggers.emr"],
},
{
"integration-name": "Amazon Elastic Kubernetes Service (EKS)",
"python-modules": ["airflow.providers.amazon.aws.triggers.eks"],
},
{
"integration-name": "Amazon ECS",
"python-modules": ["airflow.providers.amazon.aws.triggers.ecs"],
},
{
"integration-name": "Amazon RDS",
"python-modules": ["airflow.providers.amazon.aws.triggers.rds"],
},
{
"integration-name": "AWS Step Functions",
"python-modules": ["airflow.providers.amazon.aws.triggers.step_function"],
},
{
"integration-name": "Amazon Simple Queue Service (SQS)",
"python-modules": ["airflow.providers.amazon.aws.triggers.sqs"],
},
{
"integration-name": "AWS Glue DataBrew",
"python-modules": ["airflow.providers.amazon.aws.triggers.glue_databrew"],
},
{
"integration-name": "Amazon Neptune",
"python-modules": ["airflow.providers.amazon.aws.triggers.neptune"],
},
{
"integration-name": "AWS Database Migration Service",
"python-modules": ["airflow.providers.amazon.aws.triggers.dms"],
},
],
"transfers": [
{
"source-integration-name": "Amazon DynamoDB",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.dynamodb_to_s3",
},
{
"source-integration-name": "Google Cloud Storage (GCS)",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.gcs_to_s3",
},
{
"source-integration-name": "Amazon Glacier",
"target-integration-name": "Google Cloud Storage (GCS)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst",
"python-module": "airflow.providers.amazon.aws.transfers.glacier_to_gcs",
},
{
"source-integration-name": "Google",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.google_api_to_s3",
},
{
"source-integration-name": "Apache Hive",
"target-integration-name": "Amazon DynamoDB",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst",
"python-module": "airflow.providers.amazon.aws.transfers.hive_to_dynamodb",
},
{
"source-integration-name": "Hypertext Transfer Protocol (HTTP)",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.http_to_s3",
},
{
"source-integration-name": "Internet Message Access Protocol (IMAP)",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.imap_attachment_to_s3",
},
{
"source-integration-name": "MongoDB",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.mongo_to_s3",
},
{
"source-integration-name": "Amazon Redshift",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.redshift_to_s3",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "Amazon Redshift",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst",
"python-module": "airflow.providers.amazon.aws.transfers.s3_to_redshift",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "SSH File Transfer Protocol (SFTP)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst",
"python-module": "airflow.providers.amazon.aws.transfers.s3_to_sftp",
},
{
"source-integration-name": "SSH File Transfer Protocol (SFTP)",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.sftp_to_s3",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "File Transfer Protocol (FTP)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst",
"python-module": "airflow.providers.amazon.aws.transfers.s3_to_ftp",
},
{
"source-integration-name": "Exasol",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"python-module": "airflow.providers.amazon.aws.transfers.exasol_to_s3",
},
{
"source-integration-name": "File Transfer Protocol (FTP)",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.ftp_to_s3",
},
{
"source-integration-name": "Salesforce",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.salesforce_to_s3",
},
{
"source-integration-name": "Local",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.local_to_s3",
},
{
"source-integration-name": "Common SQL",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.sql_to_s3",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "Common SQL",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst",
"python-module": "airflow.providers.amazon.aws.transfers.s3_to_sql",
},
{
"source-integration-name": "Amazon Web Services",
"target-integration-name": "Amazon Web Services",
"python-module": "airflow.providers.amazon.aws.transfers.base",
},
{
"source-integration-name": "Microsoft Azure Blob Storage",
"target-integration-name": "Amazon Simple Storage Service (S3)",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst",
"python-module": "airflow.providers.amazon.aws.transfers.azure_blob_to_s3",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "Amazon DynamoDB",
"how-to-guide": "/docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst",
"python-module": "airflow.providers.amazon.aws.transfers.s3_to_dynamodb",
},
],
"extra-links": [
"airflow.providers.amazon.aws.links.athena.AthenaQueryResultsLink",
"airflow.providers.amazon.aws.links.batch.BatchJobDefinitionLink",
"airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink",
"airflow.providers.amazon.aws.links.batch.BatchJobQueueLink",
"airflow.providers.amazon.aws.links.emr.EmrClusterLink",
"airflow.providers.amazon.aws.links.emr.EmrLogsLink",
"airflow.providers.amazon.aws.links.emr.EmrServerlessCloudWatchLogsLink",
"airflow.providers.amazon.aws.links.emr.EmrServerlessDashboardLink",
"airflow.providers.amazon.aws.links.emr.EmrServerlessLogsLink",
"airflow.providers.amazon.aws.links.emr.EmrServerlessS3LogsLink",
"airflow.providers.amazon.aws.links.glue.GlueJobRunDetailsLink",
"airflow.providers.amazon.aws.links.logs.CloudWatchEventsLink",
"airflow.providers.amazon.aws.links.sagemaker.SageMakerTransformJobLink",
"airflow.providers.amazon.aws.links.step_function.StateMachineDetailsLink",
"airflow.providers.amazon.aws.links.step_function.StateMachineExecutionsDetailsLink",
"airflow.providers.amazon.aws.links.comprehend.ComprehendPiiEntitiesDetectionLink",
"airflow.providers.amazon.aws.links.comprehend.ComprehendDocumentClassifierLink",
"airflow.providers.amazon.aws.links.datasync.DataSyncTaskLink",
"airflow.providers.amazon.aws.links.datasync.DataSyncTaskExecutionLink",
"airflow.providers.amazon.aws.links.ec2.EC2InstanceLink",
"airflow.providers.amazon.aws.links.ec2.EC2InstanceDashboardLink",
],
"connection-types": [
{
"hook-class-name": "airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook",
"connection-type": "aws",
},
{
"hook-class-name": "airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook",
"connection-type": "chime",
},
{"hook-class-name": "airflow.providers.amazon.aws.hooks.emr.EmrHook", "connection-type": "emr"},
{
"hook-class-name": "airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook",
"connection-type": "redshift",
},
{
"hook-class-name": "airflow.providers.amazon.aws.hooks.athena_sql.AthenaSQLHook",
"connection-type": "athena",
},
],
"notifications": [
"airflow.providers.amazon.aws.notifications.chime.ChimeNotifier",
"airflow.providers.amazon.aws.notifications.sns.SnsNotifier",
"airflow.providers.amazon.aws.notifications.sqs.SqsNotifier",
],
"secrets-backends": [
"airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend",
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
],
"logging": [
"airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler",
"airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler",
],
"config": {
"aws": {
"description": "This section contains settings for Amazon Web Services (AWS) integration.",
"options": {
"session_factory": {
"description": "Full import path to the class which implements a custom session factory for\n``boto3.session.Session``. For more details please have a look at\n:ref:`howto/connection:aws:session-factory`.\n",
"default": None,
"example": "my_company.aws.MyCustomSessionFactory",
"type": "string",
"version_added": "3.1.1",
},
"cloudwatch_task_handler_json_serializer": {
"description": "By default, when logging non-string messages, all non-json objects are logged as `null`.\nExcept `datetime` objects which are ISO formatted. Users can optionally use a `repr` serializer or\nprovide their own JSON serializer for any non-JSON-serializable objects in the logged message.\n\n* `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize` uses `repr` (be aware\n there is the potential of logging sensitive data depending on the `repr` method of logged objects)\n* `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy` uses `null`.\n\nIf a custom serializer is provided, it must adhere to `Callable[[Any], str | None]`, where `None`\nserializes to `null` (e.g. `def my_serializer(o: Any) -> str | None`). Since this is on the logging\npath and it's possible there's an exception being handled, special care should be taken to fail\ngracefully without raising a new exception inside of your serializer.\n",
"type": "string",
"version_added": "8.7.2",
"example": "airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize",
"default": "airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy",
},
},
},
"aws_batch_executor": {
"description": "This section only applies if you are using the AwsBatchExecutor in\nAirflow's ``[core]`` configuration.\nFor more information on any of these execution parameters, see the link below:\nhttps://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.submit_job\nFor boto3 credential management, see\nhttps://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html\n",
"options": {
"conn_id": {
"description": "The Airflow connection (i.e. credentials) used by the Batch executor to make API calls to AWS Batch.\n",
"version_added": "8.11",
"type": "string",
"example": "aws_default",
"default": "aws_default",
},
"region_name": {
"description": "The name of the AWS Region where Amazon Batch is configured. Required.\n",
"version_added": "8.11",
"type": "string",
"example": "us-east-1",
"default": None,
},
"max_submit_job_attempts": {
"description": "The maximum number of times the Batch Executor should attempt to run a Batch Job.\n",
"version_added": "8.11",
"type": "integer",
"example": "3",
"default": "3",
},
"check_health_on_startup": {
"description": "Whether or not to check the Batch Executor health on startup.\n",
"version_added": "8.11",
"type": "boolean",
"example": "True",
"default": "True",
},
"job_name": {
"description": "The name of the job submitted to AWS Batch. It can be up to 128 letters long.\nThe first character must be alphanumeric, can contain uppercase and lowercase\nletters, numbers, hyphens (-), and underscores (_).\n",
"version_added": "8.11",
"type": "string",
"example": "airflow-batch-executor-job",
"default": None,
},
"job_queue": {
"description": "The job queue where the job is submitted. You can specify either the name\nor the Amazon Resource Name (ARN) of the queue.\n",
"version_added": "8.11",
"type": "string",
"example": "airflow-batch-executor-job-queue",
"default": None,
},
"job_definition": {
"description": "The job definition used by the job. You can specify either the name\nor the Amazon Resource Name (ARN) of the job definition with or\nwithout the revision. If the revision is not specified, then the\nlatest active revision is used.\n",
"version_added": "8.11",
"type": "string",
"example": "airflow-batch-executor-job-definition",
"default": None,
},
"submit_job_kwargs": {
"description": "Additional parameters to pass to the submit_job method of the\nAWS Batch client.\n",
"version_added": "8.11",
"type": "string",
"example": '{"Tags": [{"Key": "key", "Value": "value"}]}',
"default": None,
},
},
},
"aws_ecs_executor": {
"description": "This section only applies if you are using the AwsEcsExecutor in\nAirflow's ``[core]`` configuration.\nFor more information on any of these execution parameters, see the link below:\nhttps://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs/client/run_task.html\nFor boto3 credential management, see\nhttps://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html\n",
"options": {
"conn_id": {
"description": "The Airflow connection (i.e. credentials) used by the ECS executor to make API calls to AWS ECS.\n",
"version_added": "8.10",
"type": "string",
"example": "aws_default",
"default": "aws_default",
},
"region_name": {
"description": "The name of the AWS Region where Amazon ECS is configured. Required.\n",
"version_added": "8.10",
"type": "string",
"example": "us-east-1",
"default": None,
},
"assign_public_ip": {
"description": "Whether to assign a public IP address to the containers launched by the ECS executor.\nFor more info see url to Boto3 docs above.\n",
"version_added": "8.10",
"type": "boolean",
"example": "True",
"default": "False",
},
"cluster": {
"description": "Name of the Amazon ECS Cluster. Required.\n",
"version_added": "8.10",
"type": "string",
"example": "ecs_executor_cluster",
"default": None,
},
"capacity_provider_strategy": {
"description": "The capacity provider strategy to use for the task.\n\nIf a Capacity Provider Strategy is specified, the Launch Type parameter must be omitted. If\nno Capacity Provider Strategy or Launch Type is specified, the Default CapacityProvider Strategy\nfor the cluster is used, if present.\n\nWhen you use cluster auto scaling, you must specify Capacity Provider Strategy and not Launch Type.\n",
"version_added": "8.17",
"type": "string",
"example": "[{'capacityProvider': 'cp1', 'weight': 5}, {'capacityProvider': 'cp2', 'weight': 1}]",
"default": None,
},
"container_name": {
"description": "Name of the container that will be used to execute Airflow tasks via the ECS executor.\nThe container should be specified in the ECS Task Definition and will receive an airflow\nCLI command as an additional parameter to its entrypoint. For more info see url to Boto3\ndocs above. Required.\n",
"version_added": "8.10",
"type": "string",
"example": "ecs_executor_container",
"default": None,
},
"launch_type": {
"description": "Launch type can either be 'FARGATE' OR 'EC2'. For more info see url to\nBoto3 docs above.\n\nIf a Launch Type is specified, the Capacity Provider Strategy parameter must be omitted. If\nno Capacity Provider Strategy or Launch Type is specified, the Default Capacity Provider Strategy\nfor the cluster is used, if present.\n\nIf the launch type is EC2, the executor will attempt to place tasks on\nempty EC2 instances. If there are no EC2 instances available, no task\nis placed and this function will be called again in the next heart-beat.\n\nIf the launch type is FARGATE, this will run the tasks on new AWS Fargate\ninstances.\n",
"version_added": "8.10",
"type": "string",
"example": "FARGATE",
"default": None,
},
"platform_version": {
"description": "The platform version the task uses. A platform version is only specified\nfor tasks hosted on Fargate. If one isn't specified, the LATEST platform\nversion is used.\n",
"version_added": "8.10",
"type": "string",
"example": "1.4.0",
"default": "LATEST",
},
"security_groups": {
"description": "The comma-separated IDs of the security groups associated with the task. If you\ndon't specify a security group, the default security group for the VPC is used.\nThere's a limit of 5 security groups. For more info see url to Boto3 docs above.\n",
"version_added": "8.10",
"type": "string",
"example": "sg-XXXX,sg-YYYY",
"default": None,
},
"subnets": {
"description": "The comma-separated IDs of the subnets associated with the task or service.\nThere's a limit of 16 subnets. For more info see url to Boto3 docs above.\n",
"version_added": "8.10",
"type": "string",
"example": "subnet-XXXXXXXX,subnet-YYYYYYYY",
"default": None,
},
"task_definition": {
"description": "The family and revision (family:revision) or full ARN of the task definition\nto run. If a revision isn't specified, the latest ACTIVE revision is used.\nFor more info see url to Boto3 docs above.\n",
"version_added": "8.10",
"type": "string",
"example": "executor_task_definition:LATEST",
"default": None,
},
"max_run_task_attempts": {
"description": "The maximum number of times the ECS Executor should attempt to run a task.\n",
"version_added": "8.10",
"type": "integer",
"example": "3",
"default": "3",
},
"run_task_kwargs": {
"description": "A JSON string containing arguments to provide the ECS `run_task` API (see url above).\n",
"version_added": "8.10",
"type": "string",
"example": '{"tags": {"key": "schema", "value": "1.0"}}',
"default": None,
},
"check_health_on_startup": {
"description": "Whether or not to check the ECS Executor health on startup.\n",
"version_added": "8.11",
"type": "boolean",
"example": "True",
"default": "True",
},
},
},
"aws_auth_manager": {
"description": "This section only applies if you are using the AwsAuthManager. In other words, if you set\n``[core] auth_manager = airflow.providers.amazon.aws.auth_manager.aws_auth_manager.AwsAuthManager`` in\nAirflow's configuration.\n",
"options": {
"enable": {
"description": "AWS auth manager is not ready to be used. Turn on this flag to use it anyway.\nDo that at your own risk since the AWS auth manager is not in an usable state.\n",
"version_added": "8.12.0",
"type": "boolean",
"example": "True",
"default": "False",
},
"conn_id": {
"description": "The Airflow connection (i.e. credentials) used by the AWS auth manager to make API calls to AWS\nIdentity Center and Amazon Verified Permissions.\n",
"version_added": "8.12.0",
"type": "string",
"example": "aws_default",
"default": "aws_default",
},
"region_name": {
"description": "The name of the AWS Region where Amazon Verified Permissions is configured. Required.\n",
"version_added": "8.10",
"type": "string",
"example": "us-east-1",
"default": None,
},
"saml_metadata_url": {
"description": "SAML metadata XML file provided by AWS Identity Center.\nThis URL can be found in the AWS Identity Center console. Required.\n",
"version_added": "8.12.0",
"type": "string",
"example": "https://portal.sso.<region>.amazonaws.com/saml/metadata/XXXXXXXXXX",
"default": None,
},
"avp_policy_store_id": {
"description": "Amazon Verified Permissions' policy store ID where all the policies defining user permissions\nin Airflow are stored. Required.\n",
"version_added": "8.12.0",
"type": "string",
"example": None,
"default": None,
},
},
},
},
"executors": ["airflow.providers.amazon.aws.executors.ecs.ecs_executor.AwsEcsExecutor"],
"dependencies": [
"apache-airflow>=2.9.0",
"apache-airflow-providers-common-compat>=1.3.0",
"apache-airflow-providers-common-sql>=1.20.0",
"apache-airflow-providers-http",
"boto3>=1.34.90",
"botocore>=1.34.90",
"inflection>=0.5.1",
"watchtower>=3.0.0,!=3.3.0,<4",
"jsonpath_ng>=1.5.3",
"redshift_connector>=2.0.918",
"asgiref>=2.3.0",
"PyAthena>=3.0.10",
"jmespath>=0.7.0",
"python3-saml>=1.16.0",
],
"optional-dependencies": {
"pandas": ["pandas>=2.1.2,<2.2"],
"aiobotocore": ["aiobotocore[boto3]>=2.13.0"],
"cncf.kubernetes": ["apache-airflow-providers-cncf-kubernetes>=7.2.0"],
"s3fs": ["s3fs>=2023.10.0"],
"python3-saml": ["python3-saml>=1.16.0"],
"apache.hive": ["apache-airflow-providers-apache-hive"],
"exasol": ["apache-airflow-providers-exasol"],
"ftp": ["apache-airflow-providers-ftp"],
"google": ["apache-airflow-providers-google"],
"imap": ["apache-airflow-providers-imap"],
"microsoft.azure": ["apache-airflow-providers-microsoft-azure"],
"mongo": ["apache-airflow-providers-mongo"],
"openlineage": ["apache-airflow-providers-openlineage"],
"salesforce": ["apache-airflow-providers-salesforce"],
"ssh": ["apache-airflow-providers-ssh"],
},
"devel-dependencies": [
"aiobotocore>=2.13.0",
"aws_xray_sdk>=2.12.0",
"moto[cloudformation,glue]>=5.0.0",
"mypy-boto3-appflow>=1.35.39",
"mypy-boto3-rds>=1.34.90",
"mypy-boto3-redshift-data>=1.34.0",
"mypy-boto3-s3>=1.34.90",
"s3fs>=2023.10.0",
"openapi-schema-validator>=0.6.2",
"openapi-spec-validator>=0.7.1",
],
}