[core] # The folder where your airflow pipelines live, most likely a # subfolder in a code repository. This path must be absolute. # # Variable: AIRFLOW__CORE__DAGS_FOLDER # dags_folder = /root/airflow/dags # Hostname by providing a path to a callable, which will resolve the hostname. # The format is "package.function". # # For example, default value ``airflow.utils.net.getfqdn`` means that result from patched # version of `socket.getfqdn() `__, # see related `CPython Issue `__. # # No argument should be required in the function specified. # If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` # # Variable: AIRFLOW__CORE__HOSTNAME_CALLABLE # hostname_callable = airflow.utils.net.getfqdn # A callable to check if a python file has airflow dags defined or not and should # return ``True`` if it has dags otherwise ``False``. # If this is not provided, Airflow uses its own heuristic rules. # # The function should have the following signature # # .. code-block:: python # # def func_name(file_path: str, zip_file: zipfile.ZipFile | None = None) -> bool: ... # # Variable: AIRFLOW__CORE__MIGHT_CONTAIN_DAG_CALLABLE # might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_heuristic # Default timezone in case supplied date times are naive # can be `UTC` (default), `system`, or any `IANA ` # timezone string (e.g. Europe/Amsterdam) # # Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE # default_timezone = utc # The executor class that airflow should use. Choices include # ``LocalExecutor``, ``CeleryExecutor``, # ``KubernetesExecutor`` or the full import path to the class when using a custom executor. # # Variable: AIRFLOW__CORE__EXECUTOR # executor = LocalExecutor # The auth manager class that airflow should use. Full import path to the auth manager class. # # Variable: AIRFLOW__CORE__AUTH_MANAGER # auth_manager = airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager # The list of users and their associated role in simple auth manager. If the simple auth manager is # used in your environment, this list controls who can access the environment. # # List of user-role delimited with a comma. Each user-role is a colon delimited couple of username and # role. Roles are predefined in simple auth managers: viewer, user, op, admin. # # Example: simple_auth_manager_users = bob:admin,peter:viewer # # Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS # simple_auth_manager_users = admin:admin,mo:admin,Bart:admin # Whether to disable authentication and allow everyone as admin in the environment. # # Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS # simple_auth_manager_all_admins = False # The json file where the simple auth manager stores passwords for the configured users. # By default this is ``AIRFLOW_HOME/simple_auth_manager_passwords.json.generated``. # # Example: simple_auth_manager_passwords_file = /path/to/passwords.json # # Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE # # simple_auth_manager_passwords_file = # This defines the maximum number of task instances that can run concurrently per scheduler in # Airflow, regardless of the worker count. Generally this value, multiplied by the number of # schedulers in your cluster, is the maximum number of task instances with the running # state in the metadata database. The value must be larger or equal 1. # # Variable: AIRFLOW__CORE__PARALLELISM # parallelism = 32 # The maximum number of task instances allowed to run concurrently in each DAG. To calculate # the number of tasks that is running concurrently for a DAG, add up the number of running # tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``, # which is defaulted as ``[core] max_active_tasks_per_dag``. # # An example scenario when this would be useful is when you want to stop a new dag with an early # start date from stealing all the executor slots in a cluster. # # Variable: AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG # max_active_tasks_per_dag = 16 # Are DAGs paused by default at creation # # Variable: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION # dags_are_paused_at_creation = True # The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs # if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, # which is defaulted as ``[core] max_active_runs_per_dag``. # # Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG # max_active_runs_per_dag = 16 # (experimental) The maximum number of consecutive DAG failures before DAG is automatically paused. # This is also configurable per DAG level with ``max_consecutive_failed_dag_runs``, # which is defaulted as ``[core] max_consecutive_failed_dag_runs_per_dag``. # If not specified, then the value is considered as 0, # meaning that the dags are never paused out by default. # # Variable: AIRFLOW__CORE__MAX_CONSECUTIVE_FAILED_DAG_RUNS_PER_DAG # max_consecutive_failed_dag_runs_per_dag = 0 # The name of the method used in order to start Python processes via the multiprocessing module. # This corresponds directly with the options available in the Python docs: # `multiprocessing.set_start_method # `__ # must be one of the values returned by `multiprocessing.get_all_start_methods() # `__. # # Example: mp_start_method = fork # # Variable: AIRFLOW__CORE__MP_START_METHOD # # mp_start_method = # Whether to load the DAG examples that ship with Airflow. It's good to # get started, but you probably want to set this to ``False`` in a production # environment # # Variable: AIRFLOW__CORE__LOAD_EXAMPLES # load_examples = False # Path to the folder containing Airflow plugins # # Variable: AIRFLOW__CORE__PLUGINS_FOLDER # plugins_folder = /root/airflow/plugins # Should tasks be executed via forking of the parent process # # * ``False``: Execute via forking of the parent process # * ``True``: Spawning a new python process, slower than fork, but means plugin changes picked # up by tasks straight away # # Variable: AIRFLOW__CORE__EXECUTE_TASKS_NEW_PYTHON_INTERPRETER # execute_tasks_new_python_interpreter = False # Secret key to save connection passwords in the db # # Variable: AIRFLOW__CORE__FERNET_KEY # fernet_key = REDACTED # Whether to disable pickling dags # # Variable: AIRFLOW__CORE__DONOT_PICKLE # donot_pickle = True # How long before timing out a python file import # # Variable: AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT # dagbag_import_timeout = 30.0 # Should a traceback be shown in the UI for dagbag import errors, # instead of just the exception message # # Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACKS # dagbag_import_error_tracebacks = True # If tracebacks are shown, how many entries from the traceback should be shown # # Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACK_DEPTH # dagbag_import_error_traceback_depth = 2 # If set, tasks without a ``run_as_user`` argument will be run with this user # Can be used to de-elevate a sudo user running Airflow when executing tasks # # Variable: AIRFLOW__CORE__DEFAULT_IMPERSONATION # default_impersonation = # What security module to use (for example kerberos) # # Variable: AIRFLOW__CORE__SECURITY # security = # Turn unit test mode on (overwrites many configuration options with test # values at runtime) # # Variable: AIRFLOW__CORE__UNIT_TEST_MODE # unit_test_mode = False # Space-separated list of classes that may be imported during deserialization. Items can be glob # expressions. Python built-in classes (like dict) are always allowed. # # Example: allowed_deserialization_classes = airflow.* my_mod.my_other_mod.TheseClasses* # # Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES # allowed_deserialization_classes = airflow.* # Space-separated list of classes that may be imported during deserialization. Items are processed # as regex expressions. Python built-in classes (like dict) are always allowed. # This is a secondary option to ``[core] allowed_deserialization_classes``. # # Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES_REGEXP # allowed_deserialization_classes_regexp = # When a task is killed forcefully, this is the amount of time in seconds that # it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED # # Variable: AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME # killed_task_cleanup_time = 60 # Whether to override params with dag_run.conf. If you pass some key-value pairs # through ``airflow dags backfill -c`` or # ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. # # Variable: AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS # dag_run_conf_overrides_params = True # If enabled, Airflow will only scan files containing both ``DAG`` and ``airflow`` (case-insensitive). # # Variable: AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE # dag_discovery_safe_mode = True # The pattern syntax used in the # `.airflowignore # `__ # files in the DAG directories. Valid values are ``regexp`` or ``glob``. # # Variable: AIRFLOW__CORE__DAG_IGNORE_FILE_SYNTAX # dag_ignore_file_syntax = glob # The number of retries each task is going to have by default. Can be overridden at dag or task level. # # Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRIES # default_task_retries = 0 # The number of seconds each task is going to wait by default between retries. Can be overridden at # dag or task level. # # Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRY_DELAY # default_task_retry_delay = 300 # The maximum delay (in seconds) each task is going to wait by default between retries. # This is a global setting and cannot be overridden at task or DAG level. # # Variable: AIRFLOW__CORE__MAX_TASK_RETRY_DELAY # max_task_retry_delay = 86400 # The weighting method used for the effective total priority weight of the task # # Variable: AIRFLOW__CORE__DEFAULT_TASK_WEIGHT_RULE # default_task_weight_rule = downstream # Maximum possible time (in seconds) that task will have for execution of auxiliary processes # (like listeners, mini scheduler...) after task is marked as success.. # # Variable: AIRFLOW__CORE__TASK_SUCCESS_OVERTIME # task_success_overtime = 20 # The default task execution_timeout value for the operators. Expected an integer value to # be passed into timedelta as seconds. If not specified, then the value is considered as None, # meaning that the operators are never timed out by default. # # Variable: AIRFLOW__CORE__DEFAULT_TASK_EXECUTION_TIMEOUT # default_task_execution_timeout = # Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. # # Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL # min_serialized_dag_update_interval = 30 # If ``True``, serialized DAGs are compressed before writing to DB. # # .. note:: # # This will disable the DAG dependencies view # # Variable: AIRFLOW__CORE__COMPRESS_SERIALIZED_DAGS # compress_serialized_dags = False # Fetching serialized DAG can not be faster than a minimum interval to reduce database # read rate. This config controls when your DAGs are updated in the Webserver # # Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_FETCH_INTERVAL # min_serialized_dag_fetch_interval = 10 # Maximum number of Rendered Task Instance Fields (Template Fields) per task to store # in the Database. # All the template_fields for each of Task Instance are stored in the Database. # Keeping this number small may cause an error when you try to view ``Rendered`` tab in # TaskInstance view for older tasks. # # Variable: AIRFLOW__CORE__MAX_NUM_RENDERED_TI_FIELDS_PER_TASK # max_num_rendered_ti_fields_per_task = 30 # Path to custom XCom class that will be used to store and resolve operators results # # Example: xcom_backend = path.to.CustomXCom # # Variable: AIRFLOW__CORE__XCOM_BACKEND # xcom_backend = airflow.sdk.execution_time.xcom.BaseXCom # By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, # if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. # # Variable: AIRFLOW__CORE__LAZY_LOAD_PLUGINS # lazy_load_plugins = True # By default Airflow providers are lazily-discovered (discovery and imports happen only when required). # Set it to ``False``, if you want to discover providers whenever 'airflow' is invoked via cli or # loaded from module. # # Variable: AIRFLOW__CORE__LAZY_DISCOVER_PROVIDERS # lazy_discover_providers = True # Hide sensitive **Variables** or **Connection extra json keys** from UI # and task logs when set to ``True`` # # .. note:: # # Connection passwords are always hidden in logs # # Variable: AIRFLOW__CORE__HIDE_SENSITIVE_VAR_CONN_FIELDS # hide_sensitive_var_conn_fields = True # A comma-separated list of extra sensitive keywords to look for in variables names or connection's # extra JSON. # # Variable: AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES # sensitive_var_conn_names = # Task Slot counts for ``default_pool``. This setting would not have any effect in an existing # deployment where the ``default_pool`` is already created. For existing deployments, users can # change the number of slots using Webserver, API or the CLI # # Variable: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT # default_pool_task_slot_count = 128 # The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a # length exceeding this value, the task pushing the XCom will be failed automatically to prevent the # mapped tasks from clogging the scheduler. # # Variable: AIRFLOW__CORE__MAX_MAP_LENGTH # max_map_length = 1024 # The default umask to use for process when run in daemon mode (scheduler, worker, etc.) # # This controls the file-creation mode mask which determines the initial value of file permission bits # for newly created files. # # This value is treated as an octal-integer. # # Variable: AIRFLOW__CORE__DAEMON_UMASK # daemon_umask = 0o077 # Class to use as asset manager. # # Example: asset_manager_class = airflow.assets.manager.AssetManager # # Variable: AIRFLOW__CORE__ASSET_MANAGER_CLASS # # asset_manager_class = # Kwargs to supply to asset manager. # # Example: asset_manager_kwargs = {"some_param": "some_value"} # # Variable: AIRFLOW__CORE__ASSET_MANAGER_KWARGS # # asset_manager_kwargs = # (experimental) Whether components should use Airflow Internal API for DB connectivity. # # Variable: AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION # database_access_isolation = False # (experimental) Airflow Internal API url. # Only used if ``[core] database_access_isolation`` is ``True``. # # Example: internal_api_url = http://localhost:8080 # # Variable: AIRFLOW__CORE__INTERNAL_API_URL # # internal_api_url = # Secret key used to authenticate internal API clients to core. It should be as random as possible. # However, when running more than 1 instances of webserver / internal API services, make sure all # of them use the same ``secret_key`` otherwise calls will fail on authentication. # The authentication token generated using the secret key has a short expiry time though - make # sure that time on ALL the machines that you run airflow components on is synchronized # (for example using ntpd) otherwise you might get "forbidden" errors when the logs are accessed. # # Variable: AIRFLOW__CORE__INTERNAL_API_SECRET_KEY # internal_api_secret_key = REDACTED # The ability to allow testing connections across Airflow UI, API and CLI. # Supported options: ``Disabled``, ``Enabled``, ``Hidden``. Default: Disabled # Disabled - Disables the test connection functionality and disables the Test Connection button in UI. # Enabled - Enables the test connection functionality and shows the Test Connection button in UI. # Hidden - Disables the test connection functionality and hides the Test Connection button in UI. # Before setting this to Enabled, make sure that you review the users who are able to add/edit # connections and ensure they are trusted. Connection testing can be done maliciously leading to # undesired and insecure outcomes. # See `Airflow Security Model: Capabilities of authenticated UI users # `__ # for more details. # # Variable: AIRFLOW__CORE__TEST_CONNECTION # test_connection = Disabled # The maximum length of the rendered template field. If the value to be stored in the # rendered template field exceeds this size, it's redacted. # # Variable: AIRFLOW__CORE__MAX_TEMPLATED_FIELD_LENGTH # max_templated_field_length = 4096 # The url of the execution api server. Default is ``{BASE_URL}/execution/`` # where ``{BASE_URL}`` is the base url of the API Server. If ``{BASE_URL}`` is not set, # it will use ``http://localhost:8080`` as the default base url. # # Variable: AIRFLOW__CORE__EXECUTION_API_SERVER_URL # # execution_api_server_url = [database] # Path to the ``alembic.ini`` file. You can either provide the file path relative # to the Airflow home directory or the absolute path if it is located elsewhere. # # Variable: AIRFLOW__DATABASE__ALEMBIC_INI_FILE_PATH # alembic_ini_file_path = alembic.ini # The SQLAlchemy connection string to the metadata database. # SQLAlchemy supports many different database engines. # See: `Set up a Database Backend: Database URI # `__ # for more details. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN # sql_alchemy_conn = sqlite:////root/airflow/airflow.db # Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value # # Example: sql_alchemy_engine_args = {"arg1": true} # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS # # sql_alchemy_engine_args = # The encoding for the databases # # Variable: AIRFLOW__DATABASE__SQL_ENGINE_ENCODING # sql_engine_encoding = utf-8 # Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id`` columns # in case they have different encoding. # By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb`` # the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed # the maximum size of allowed index when collation is set to ``utf8mb4`` variant, see # `GitHub Issue Comment `__ # for more details. # # Variable: AIRFLOW__DATABASE__SQL_ENGINE_COLLATION_FOR_IDS # # sql_engine_collation_for_ids = # If SQLAlchemy should pool database connections. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_ENABLED # sql_alchemy_pool_enabled = True # The SQLAlchemy pool size is the maximum number of database connections # in the pool. 0 indicates no limit. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE # sql_alchemy_pool_size = 5 # The maximum overflow size of the pool. # When the number of checked-out connections reaches the size set in pool_size, # additional connections will be returned up to this limit. # When those additional connections are returned to the pool, they are disconnected and discarded. # It follows then that the total number of simultaneous connections the pool will allow # is **pool_size** + **max_overflow**, # and the total number of "sleeping" connections the pool will allow is pool_size. # max_overflow can be set to ``-1`` to indicate no overflow limit; # no limit will be placed on the total number of concurrent connections. Defaults to ``10``. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW # sql_alchemy_max_overflow = 10 # The SQLAlchemy pool recycle is the number of seconds a connection # can be idle in the pool before it is invalidated. This config does # not apply to sqlite. If the number of DB connections is ever exceeded, # a lower config value will allow the system to recover faster. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE # sql_alchemy_pool_recycle = 1800 # Check connection at the start of each connection pool checkout. # Typically, this is a simple statement like "SELECT 1". # See `SQLAlchemy Pooling: Disconnect Handling - Pessimistic # `__ # for more details. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_PRE_PING # sql_alchemy_pool_pre_ping = True # The schema to use for the metadata database. # SQLAlchemy supports databases with the concept of multiple schemas. # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA # sql_alchemy_schema = # Import path for connect args in SQLAlchemy. Defaults to an empty dict. # This is useful when you want to configure db engine args that SQLAlchemy won't parse # in connection string. This can be set by passing a dictionary containing the create engine parameters. # For more details about passing create engine parameters (keepalives variables, timeout etc) # in Postgres DB Backend see `Setting up a PostgreSQL Database # `__ # e.g ``connect_args={"timeout":30}`` can be defined in ``airflow_local_settings.py`` and # can be imported as shown below # # Example: sql_alchemy_connect_args = airflow_local_settings.connect_args # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONNECT_ARGS # # sql_alchemy_connect_args = # Important Warning: Use of sql_alchemy_session_maker Highly Discouraged # Import path for function which returns 'sqlalchemy.orm.sessionmaker'. # Improper configuration of sql_alchemy_session_maker can lead to serious issues, # including data corruption, unrecoverable application crashes. Please review the SQLAlchemy # documentation for detailed guidance on proper configuration and best practices. # # Example: sql_alchemy_session_maker = airflow_local_settings._sessionmaker # # Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SESSION_MAKER # # sql_alchemy_session_maker = # Number of times the code should be retried in case of DB Operational Errors. # Not all transactions will be retried as it can cause undesired state. # Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. # # Variable: AIRFLOW__DATABASE__MAX_DB_RETRIES # max_db_retries = 3 # Whether to run alembic migrations during Airflow start up. Sometimes this operation can be expensive, # and the users can assert the correct version through other means (e.g. through a Helm chart). # Accepts ``True`` or ``False``. # # Variable: AIRFLOW__DATABASE__CHECK_MIGRATIONS # check_migrations = True # List of DB managers to use to migrate external tables in airflow database. The managers must inherit # from BaseDBManager. If ``FabAuthManager`` is configured in the environment, # ``airflow.providers.fab.auth_manager.models.db.FABDBManager`` is automatically added. # # Example: external_db_managers = airflow.providers.fab.auth_manager.models.db.FABDBManager # # Variable: AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS # # external_db_managers = # The number of rows to process in each batch when performing a migration. # This is useful for large tables to avoid locking and failure due to query timeouts. # # Variable: AIRFLOW__DATABASE__MIGRATION_BATCH_SIZE # migration_batch_size = 10000 [logging] # The folder where airflow should store its log files. # This path must be absolute. # There are a few existing configurations that assume this is set to the default. # If you choose to override this you may need to update the # ``[logging] dag_processor_manager_log_location`` and # ``[logging] dag_processor_child_process_log_directory settings`` as well. # # Variable: AIRFLOW__LOGGING__BASE_LOG_FOLDER # base_log_folder = /root/airflow/logs # Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. # Set this to ``True`` if you want to enable remote logging. # # Variable: AIRFLOW__LOGGING__REMOTE_LOGGING # remote_logging = False # Users must supply an Airflow connection id that provides access to the storage # location. Depending on your remote logging service, this may only be used for # reading logs, not writing them. # # Variable: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID # remote_log_conn_id = # Whether the local log files for GCS, S3, WASB, HDFS and OSS remote logging should be deleted after # they are uploaded to the remote location. # # Variable: AIRFLOW__LOGGING__DELETE_LOCAL_LOGS # delete_local_logs = False # Path to Google Credential JSON file. If omitted, authorization based on `the Application Default # Credentials # `__ will # be used. # # Variable: AIRFLOW__LOGGING__GOOGLE_KEY_PATH # google_key_path = # Storage bucket URL for remote logging # S3 buckets should start with **s3://** # Cloudwatch log groups should start with **cloudwatch://** # GCS buckets should start with **gs://** # WASB buckets should start with **wasb** just to help Airflow select correct handler # Stackdriver logs should start with **stackdriver://** # # Variable: AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER # remote_base_log_folder = # The remote_task_handler_kwargs param is loaded into a dictionary and passed to the ``__init__`` # of remote task handler and it overrides the values provided by Airflow config. For example if you set # ``delete_local_logs=False`` and you provide ``{"delete_local_copy": true}``, then the local # log files will be deleted after they are uploaded to remote location. # # Example: remote_task_handler_kwargs = {"delete_local_copy": true} # # Variable: AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS # remote_task_handler_kwargs = # Use server-side encryption for logs stored in S3 # # Variable: AIRFLOW__LOGGING__ENCRYPT_S3_LOGS # encrypt_s3_logs = False # Logging level. # # Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. # # Variable: AIRFLOW__LOGGING__LOGGING_LEVEL # logging_level = INFO # Logging level for celery. If not set, it uses the value of logging_level # # Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. # # Variable: AIRFLOW__LOGGING__CELERY_LOGGING_LEVEL # celery_logging_level = # Logging level for Flask-appbuilder UI. # # Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. # # Variable: AIRFLOW__LOGGING__FAB_LOGGING_LEVEL # fab_logging_level = WARNING # Logging class # Specify the class that will specify the logging configuration # This class has to be on the python classpath # # Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG # # Variable: AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS # logging_config_class = # Flag to enable/disable Colored logs in Console # Colour the logs when the controlling terminal is a TTY. # # Variable: AIRFLOW__LOGGING__COLORED_CONSOLE_LOG # colored_console_log = True # Log format for when Colored logs is enabled # # Variable: AIRFLOW__LOGGING__COLORED_LOG_FORMAT # colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s # Specifies the class utilized by Airflow to implement colored logging # # Variable: AIRFLOW__LOGGING__COLORED_FORMATTER_CLASS # colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter # Format of Log line # # Variable: AIRFLOW__LOGGING__LOG_FORMAT # log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s # Defines the format of log messages for simple logging configuration # # Variable: AIRFLOW__LOGGING__SIMPLE_LOG_FORMAT # simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s # Where to send dag parser logs. If "file", logs are sent to log files defined by child_process_log_directory. # # Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_TARGET # dag_processor_log_target = file # Format of Dag Processor Log line # # Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_FORMAT # dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s # Determines the directory where logs for the child processes of the dag processor will be stored # # Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_CHILD_PROCESS_LOG_DIRECTORY # dag_processor_child_process_log_directory = /root/airflow/logs/dag_processor # Determines the formatter class used by Airflow for structuring its log messages # The default formatter class is timezone-aware, which means that timestamps attached to log entries # will be adjusted to reflect the local timezone of the Airflow instance # # Variable: AIRFLOW__LOGGING__LOG_FORMATTER_CLASS # log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware # An import path to a function to add adaptations of each secret added with # ``airflow.sdk.execution_time.secrets_masker.mask_secret`` to be masked in log messages. # The given function is expected to require a single parameter: the secret to be adapted. # It may return a single adaptation of the secret or an iterable of adaptations to each be # masked as secrets. The original secret will be masked as well as any adaptations returned. # # Example: secret_mask_adapter = urllib.parse.quote # # Variable: AIRFLOW__LOGGING__SECRET_MASK_ADAPTER # secret_mask_adapter = # The minimum length of a secret to be masked in log messages. # Secrets shorter than this length will not be masked. # # Variable: AIRFLOW__LOGGING__MIN_LENGTH_MASKED_SECRET # min_length_masked_secret = 5 # Specify prefix pattern like mentioned below with stream handler ``TaskHandlerWithCustomFormatter`` # # Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{logical_date}}-{{ti.try_number}} # # Variable: AIRFLOW__LOGGING__TASK_LOG_PREFIX_TEMPLATE # task_log_prefix_template = # Formatting for how airflow generates file names/paths for each task run. # # Variable: AIRFLOW__LOGGING__LOG_FILENAME_TEMPLATE # log_filename_template = dag_id={{ ti.dag_id }}/run_id={{ ti.run_id }}/task_id={{ ti.task_id }}/{%% if ti.map_index >= 0 %%}map_index={{ ti.map_index }}/{%% endif %%}attempt={{ try_number|default(ti.try_number) }}.log # Name of handler to read task instance logs. # Defaults to use ``task`` handler. # # Variable: AIRFLOW__LOGGING__TASK_LOG_READER # task_log_reader = task # A comma\-separated list of third-party logger names that will be configured to print messages to # consoles\. # # Example: extra_logger_names = fastapi,sqlalchemy # # Variable: AIRFLOW__LOGGING__EXTRA_LOGGER_NAMES # extra_logger_names = # When you start an Airflow worker, Airflow starts a tiny web server # subprocess to serve the workers local log files to the airflow main # web server, who then builds pages and sends them to users. This defines # the port on which the logs are served. It needs to be unused, and open # visible from the main web server to connect into the workers. # # Variable: AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT # worker_log_server_port = 8793 # Port to serve logs from for triggerer. # See ``[logging] worker_log_server_port`` description for more info. # # Variable: AIRFLOW__LOGGING__TRIGGER_LOG_SERVER_PORT # trigger_log_server_port = 8794 # We must parse timestamps to interleave logs between trigger and task. To do so, # we need to parse timestamps in log files. In case your log format is non-standard, # you may provide import path to callable which takes a string log line and returns # the timestamp (datetime.datetime compatible). # # Example: interleave_timestamp_parser = path.to.my_func # # Variable: AIRFLOW__LOGGING__INTERLEAVE_TIMESTAMP_PARSER # # interleave_timestamp_parser = # Permissions in the form or of octal string as understood by chmod. The permissions are important # when you use impersonation, when logs are written by a different user than airflow. The most secure # way of configuring it in this case is to add both users to the same group and make it the default # group of both users. Group-writeable logs are default in airflow, but you might decide that you are # OK with having the logs other-writeable, in which case you should set it to ``0o777``. You might # decide to add more security if you do not use impersonation and change it to ``0o755`` to make it # only owner-writeable. You can also make it just readable only for owner by changing it to ``0o700`` # if all the access (read/write) for your logs happens from the same user. # # Example: file_task_handler_new_folder_permissions = 0o775 # # Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FOLDER_PERMISSIONS # file_task_handler_new_folder_permissions = 0o775 # Permissions in the form or of octal string as understood by chmod. The permissions are important # when you use impersonation, when logs are written by a different user than airflow. The most secure # way of configuring it in this case is to add both users to the same group and make it the default # group of both users. Group-writeable logs are default in airflow, but you might decide that you are # OK with having the logs other-writeable, in which case you should set it to ``0o666``. You might # decide to add more security if you do not use impersonation and change it to ``0o644`` to make it # only owner-writeable. You can also make it just readable only for owner by changing it to ``0o600`` # if all the access (read/write) for your logs happens from the same user. # # Example: file_task_handler_new_file_permissions = 0o664 # # Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FILE_PERMISSIONS # file_task_handler_new_file_permissions = 0o664 # By default Celery sends all logs into stderr. # If enabled any previous logging handlers will get *removed*. # With this option AirFlow will create new handlers # and send low level logs like INFO and WARNING to stdout, # while sending higher severity logs to stderr. # # Variable: AIRFLOW__LOGGING__CELERY_STDOUT_STDERR_SEPARATION # celery_stdout_stderr_separation = False # A comma separated list of keywords related to errors whose presence should display the line in red # color in UI # # Variable: AIRFLOW__LOGGING__COLOR_LOG_ERROR_KEYWORDS # color_log_error_keywords = error,exception # A comma separated list of keywords related to warning whose presence should display the line in yellow # color in UI # # Variable: AIRFLOW__LOGGING__COLOR_LOG_WARNING_KEYWORDS # color_log_warning_keywords = warn [metrics] # `StatsD `__ integration settings. # Configure an allow list (comma separated regex patterns to match) to send only certain metrics. # # Example: metrics_allow_list = "scheduler,executor,dagrun,pool,triggerer,celery" or "^scheduler,^executor,heartbeat|timeout" # # Variable: AIRFLOW__METRICS__METRICS_ALLOW_LIST # metrics_allow_list = # Configure a block list (comma separated regex patterns to match) to block certain metrics # from being emitted. # If ``[metrics] metrics_allow_list`` and ``[metrics] metrics_block_list`` are both configured, # ``[metrics] metrics_block_list`` is ignored. # # Example: metrics_block_list = "scheduler,executor,dagrun,pool,triggerer,celery" or "^scheduler,^executor,heartbeat|timeout" # # Variable: AIRFLOW__METRICS__METRICS_BLOCK_LIST # metrics_block_list = # Enables sending metrics to StatsD. # # Variable: AIRFLOW__METRICS__STATSD_ON # statsd_on = False # Specifies the host address where the StatsD daemon (or server) is running # # Variable: AIRFLOW__METRICS__STATSD_HOST # statsd_host = localhost # Enables the statsd host to be resolved into IPv6 address # # Variable: AIRFLOW__METRICS__STATSD_IPV6 # statsd_ipv6 = False # Specifies the port on which the StatsD daemon (or server) is listening to # # Variable: AIRFLOW__METRICS__STATSD_PORT # statsd_port = 8125 # Defines the namespace for all metrics sent from Airflow to StatsD # # Variable: AIRFLOW__METRICS__STATSD_PREFIX # statsd_prefix = airflow # A function that validate the StatsD stat name, apply changes to the stat name if necessary and return # the transformed stat name. # # The function should have the following signature # # .. code-block:: python # # def func_name(stat_name: str) -> str: ... # # Variable: AIRFLOW__METRICS__STAT_NAME_HANDLER # stat_name_handler = # To enable datadog integration to send airflow metrics. # # Variable: AIRFLOW__METRICS__STATSD_DATADOG_ENABLED # statsd_datadog_enabled = False # List of datadog tags attached to all metrics(e.g: ``key1:value1,key2:value2``) # # Variable: AIRFLOW__METRICS__STATSD_DATADOG_TAGS # statsd_datadog_tags = # Set to ``False`` to disable metadata tags for some of the emitted metrics # # Variable: AIRFLOW__METRICS__STATSD_DATADOG_METRICS_TAGS # statsd_datadog_metrics_tags = True # If you want to utilise your own custom StatsD client set the relevant # module path below. # Note: The module path must exist on your # `PYTHONPATH ` # for Airflow to pick it up # # Variable: AIRFLOW__METRICS__STATSD_CUSTOM_CLIENT_PATH # # statsd_custom_client_path = # If you want to avoid sending all the available metrics tags to StatsD, # you can configure a block list of prefixes (comma separated) to filter out metric tags # that start with the elements of the list (e.g: ``job_id,run_id``) # # Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id # # Variable: AIRFLOW__METRICS__STATSD_DISABLED_TAGS # statsd_disabled_tags = job_id,run_id # To enable sending Airflow metrics with StatsD-Influxdb tagging convention. # # Variable: AIRFLOW__METRICS__STATSD_INFLUXDB_ENABLED # statsd_influxdb_enabled = False # Enables sending metrics to OpenTelemetry. # # Variable: AIRFLOW__METRICS__OTEL_ON # otel_on = False # Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends # metrics and traces. # # Variable: AIRFLOW__METRICS__OTEL_HOST # otel_host = localhost # Specifies the port of the OpenTelemetry Collector that is listening to. # # Variable: AIRFLOW__METRICS__OTEL_PORT # otel_port = 8889 # The prefix for the Airflow metrics. # # Variable: AIRFLOW__METRICS__OTEL_PREFIX # otel_prefix = airflow # Defines the interval, in milliseconds, at which Airflow sends batches of metrics and traces # to the configured OpenTelemetry Collector. # # Variable: AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS # otel_interval_milliseconds = 60000 # If ``True``, all metrics are also emitted to the console. Defaults to ``False``. # # Variable: AIRFLOW__METRICS__OTEL_DEBUGGING_ON # otel_debugging_on = False # The default service name of traces. # # Variable: AIRFLOW__METRICS__OTEL_SERVICE # otel_service = Airflow # If ``True``, SSL will be enabled. Defaults to ``False``. # To establish an HTTPS connection to the OpenTelemetry collector, # you need to configure the SSL certificate and key within the OpenTelemetry collector's # ``config.yml`` file. # # Variable: AIRFLOW__METRICS__OTEL_SSL_ACTIVE # otel_ssl_active = False [traces] # Distributed traces integration settings. # Enables sending traces to OpenTelemetry. # # Variable: AIRFLOW__TRACES__OTEL_ON # otel_on = False # Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends # traces. # # Variable: AIRFLOW__TRACES__OTEL_HOST # otel_host = localhost # Specifies the port of the OpenTelemetry Collector that is listening to. # # Variable: AIRFLOW__TRACES__OTEL_PORT # otel_port = 8889 # The default service name of traces. # # Variable: AIRFLOW__TRACES__OTEL_SERVICE # otel_service = Airflow # If True, all traces are also emitted to the console. Defaults to False. # # Variable: AIRFLOW__TRACES__OTEL_DEBUGGING_ON # otel_debugging_on = False # If True, SSL will be enabled. Defaults to False. # To establish an HTTPS connection to the OpenTelemetry collector, # you need to configure the SSL certificate and key within the OpenTelemetry collector's # config.yml file. # # Variable: AIRFLOW__TRACES__OTEL_SSL_ACTIVE # otel_ssl_active = False [secrets] # Full class name of secrets backend to enable (will precede env vars and metastore in search path) # # Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend # # Variable: AIRFLOW__SECRETS__BACKEND # backend = # The backend_kwargs param is loaded into a dictionary and passed to ``__init__`` # of secrets backend class. See documentation for the secrets backend you are using. # JSON is expected. # # Example for AWS Systems Manager ParameterStore: # ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` # # Variable: AIRFLOW__SECRETS__BACKEND_KWARGS # backend_kwargs = # .. note:: |experimental| # # Enables local caching of Variables, when parsing DAGs only. # Using this option can make dag parsing faster if Variables are used in top level code, at the expense # of longer propagation time for changes. # Please note that this cache concerns only the DAG parsing step. There is no caching in place when DAG # tasks are run. # # Variable: AIRFLOW__SECRETS__USE_CACHE # use_cache = False # .. note:: |experimental| # # When the cache is enabled, this is the duration for which we consider an entry in the cache to be # valid. Entries are refreshed if they are older than this many seconds. # It means that when the cache is enabled, this is the maximum amount of time you need to wait to see a # Variable change take effect. # # Variable: AIRFLOW__SECRETS__CACHE_TTL_SECONDS # cache_ttl_seconds = 900 [api] # Boolean for running SwaggerUI in the webserver. # # Variable: AIRFLOW__API__ENABLE_SWAGGER_UI # enable_swagger_ui = True # Secret key used to run your api server. It should be as random as possible. However, when running # more than 1 instances of the api, make sure all of them use the same ``secret_key`` otherwise # one of them will error with "CSRF session token is missing". # The api key is also used to authorize requests to Celery workers when logs are retrieved. # The token generated using the secret key has a short expiry time though - make sure that time on # ALL the machines that you run airflow components on is synchronized (for example using ntpd) # otherwise you might get "forbidden" errors when the logs are accessed. # # Variable: AIRFLOW__API__SECRET_KEY # secret_key = 010RL08807/JBjH4cWzNaw== # Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values # except those that have security implications. ``True`` shows all values. ``False`` hides the # configuration completely. # # Variable: AIRFLOW__API__EXPOSE_CONFIG # expose_config = True # Expose stacktrace in the web server # # Variable: AIRFLOW__API__EXPOSE_STACKTRACE # expose_stacktrace = False # The base url of the API server. Airflow cannot guess what domain or CNAME you are using. # If the Airflow console (the front-end) and the API server are on a different domain, this config # should contain the API server endpoint. # # Example: base_url = https://my-airflow.company.com # # Variable: AIRFLOW__API__BASE_URL # # base_url = # The ip specified when starting the api server # # Variable: AIRFLOW__API__HOST # host = 0.0.0.0 # The port on which to run the api server # # Variable: AIRFLOW__API__PORT # port = 8080 # Number of workers to run on the API server # # Variable: AIRFLOW__API__WORKERS # workers = 4 # Number of seconds the API server waits before timing out on a worker # # Variable: AIRFLOW__API__WORKER_TIMEOUT # worker_timeout = 120 # Log files for the api server. '-' means log to stderr. # # Variable: AIRFLOW__API__ACCESS_LOGFILE # access_logfile = - # Paths to the SSL certificate and key for the api server. When both are # provided SSL will be enabled. This does not change the api server port. # The same SSL certificate will also be loaded into the worker to enable # it to be trusted when a self-signed certificate is used. # # Variable: AIRFLOW__API__SSL_CERT # ssl_cert = # Paths to the SSL certificate and key for the api server. When both are # provided SSL will be enabled. This does not change the api server port. # # Variable: AIRFLOW__API__SSL_KEY # ssl_key = # Used to set the maximum page limit for API requests. If limit passed as param # is greater than maximum page limit, it will be ignored and maximum page limit value # will be set as the limit # # Variable: AIRFLOW__API__MAXIMUM_PAGE_LIMIT # maximum_page_limit = 100 # Used to set the default page limit when limit param is zero or not provided in API # requests. Otherwise if positive integer is passed in the API requests as limit, the # smallest number of user given limit or maximum page limit is taken as limit. # # Variable: AIRFLOW__API__FALLBACK_PAGE_LIMIT # fallback_page_limit = 50 # Used in response to a preflight request to indicate which HTTP # headers can be used when making the actual request. This header is # the server side response to the browser's # Access-Control-Request-Headers header. # # Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS # access_control_allow_headers = # Specifies the method or methods allowed when accessing the resource. # # Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS # access_control_allow_methods = # Indicates whether the response can be shared with requesting code from the given origins. # Separate URLs with space. # # Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGINS # access_control_allow_origins = # Indicates whether the **xcomEntries** endpoint supports the **deserialize** # flag. If set to ``False``, setting this flag in a request would result in a # 400 Bad Request error. # # Variable: AIRFLOW__API__ENABLE_XCOM_DESERIALIZE_SUPPORT # enable_xcom_deserialize_support = False [workers] # Configuration related to workers that run Airflow tasks. # Full class name of secrets backend to enable for workers (will precede env vars backend) # # Example: secrets_backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend # # Variable: AIRFLOW__WORKERS__SECRETS_BACKEND # secrets_backend = # The secrets_backend_kwargs param is loaded into a dictionary and passed to ``__init__`` # of secrets backend class. See documentation for the secrets backend you are using. # JSON is expected. # # Example for AWS Systems Manager ParameterStore: # ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` # # Variable: AIRFLOW__WORKERS__SECRETS_BACKEND_KWARGS # secrets_backend_kwargs = # The minimum interval (in seconds) at which the worker checks the task instance's # heartbeat status with the API server to confirm it is still alive. # # Variable: AIRFLOW__WORKERS__MIN_HEARTBEAT_INTERVAL # min_heartbeat_interval = 5 # The maximum number of consecutive failed heartbeats before terminating the task instance process. # # Variable: AIRFLOW__WORKERS__MAX_FAILED_HEARTBEATS # max_failed_heartbeats = 3 # The maximum number of retry attempts to the execution API server. # # Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRIES # execution_api_retries = 5 # The minimum amount of time (in seconds) to wait before retrying a failed API request. # # Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRY_WAIT_MIN # execution_api_retry_wait_min = 1.0 # The maximum amount of time (in seconds) to wait before retrying a failed API request. # # Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRY_WAIT_MAX # execution_api_retry_wait_max = 90.0 # Number of seconds to wait after a task process exits before forcibly closing any # remaining communication sockets. This helps prevent the task supervisor from hanging # indefinitely due to missed EOF signals. # # Variable: AIRFLOW__WORKERS__SOCKET_CLEANUP_TIMEOUT # socket_cleanup_timeout = 60.0 [api_auth] # Settings relating to authentication on the Airflow APIs # The audience claim to use when generating and validating JWTs for the API. # # This variable can be a single value, or a comma-separated string, in which case the first value is the # one that will be used when generating, and the others are accepted at validation time. # # Not required, but strongly encouraged. # # See also :ref:`config:execution_api__jwt_audience` # # Example: jwt_audience = my-unique-airflow-id # # Variable: AIRFLOW__API_AUTH__JWT_AUDIENCE # # jwt_audience = # Issuer claim to use when generating and validating JWTs for the API. # # Variable: AIRFLOW__API_AUTH__JWT_ISSUER # jwt_issuer = airflow # Number in seconds until the JWTs used for authentication expires. When the token expires, # all API calls using this token will fail on authentication. # # Make sure that time on ALL the machines that you run airflow components on is synchronized # (for example using ntpd) otherwise you might get "forbidden" errors. # # See also :ref:`config:execution_api__jwt_expiration_time` # # Variable: AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME # jwt_expiration_time = 86400 # Number in seconds until the JWTs used for authentication expires for CLI commands. # When the token expires, all CLI calls using this token will fail on authentication. # # Make sure that time on ALL the machines that you run airflow components on is synchronized # (for example using ntpd) otherwise you might get "forbidden" errors. # # Variable: AIRFLOW__API_AUTH__JWT_CLI_EXPIRATION_TIME # jwt_cli_expiration_time = 3600 # Secret key used to encode and decode JWTs to authenticate to public and private APIs. # # It should be as random as possible. However, when running more than 1 instances of API services, # make sure all of them use the same ``jwt_secret`` otherwise calls will fail on authentication. # # Mutually exclusive with ``jwt_private_key_path``. # # Variable: AIRFLOW__API_AUTH__JWT_SECRET # jwt_secret = LUPyvA9pvI084a6b0PKUwA== # The path to a file containing a PEM-encoded private key use when generating Task Identity tokens in # the executor. # # Mutually exclusive with ``jwt_secret``. # # Example: jwt_private_key_path = /path/to/private_key.pem # # Variable: AIRFLOW__API_AUTH__JWT_PRIVATE_KEY_PATH # # jwt_private_key_path = # The algorithm name use when generating and validating JWT Task Identities. # # This value must be appropriate for the given private key type. # # If this is not specified Airflow makes some guesses as what algorithm is best based on the key type. # # ("HS512" if ``jwt_secret`` is set, otherwise a key-type specific guess) # # Example: jwt_algorithm = "EdDSA" or "HS512" # # Variable: AIRFLOW__API_AUTH__JWT_ALGORITHM # # jwt_algorithm = # The Key ID to place in header when generating JWTs. Not used in the validation path. # # If this is not specified the RFC7638 thumbprint of the private key will be used. # # Ignored when ``jwt_secret`` is used. # # Example: jwt_kid = my-key-id # # Variable: AIRFLOW__API_AUTH__JWT_KID # # jwt_kid = # The public signing keys of Task Execution token issuers to trust. It must contain the public key # related to ``jwt_private_key_path`` else tasks will be unlikely to execute successfully. # # Can be a local file path (without the ``file://`` prefix) or an http or https URL. # # If a remote URL is given it will be polled periodically for changes. # # Mutually exclusive with ``jwt_secret``. # # If a ``jwt_private_key_path`` is given but this settings is not set then the private key will be # trusted. If this is provided it is your responsibility to ensure that the private key used for # generation is in this list. # # Example: trusted_jwks_url = "/path/to/public-jwks.json" or "https://my-issuer/.well-known/jwks.json" # # Variable: AIRFLOW__API_AUTH__TRUSTED_JWKS_URL # # trusted_jwks_url = # Issuer of the JWT. This becomes the ``iss`` claim of generated tokens, and is validated on incoming # requests. # # Ideally this should be unique per individual airflow deployment # # Not required, but strongly recommended to be set. # # See also :ref:`config:api_auth__jwt_audience` # # Example: jwt_issuer = http://my-airflow.mycompany.com # # Variable: AIRFLOW__API_AUTH__JWT_ISSUER # # jwt_issuer = # Number of seconds leeway in validating expiry time of JWTs to account for clock skew between # client and server # # Variable: AIRFLOW__API_AUTH__JWT_LEEWAY # jwt_leeway = 10 [execution_api] # Settings related to the Execution API server. # # The ExecutionAPI also uses a lot of settings from the :ref:`config:api_auth` section. # Number in seconds until the JWT used for authentication expires. When the token expires, # all API calls using this token will fail on authentication. # # Make sure that time on ALL the machines that you run airflow components on is synchronized # (for example using ntpd) otherwise you might get "forbidden" errors. # # Variable: AIRFLOW__EXECUTION_API__JWT_EXPIRATION_TIME # jwt_expiration_time = 600 # The audience claim to use when generating and validating JWTs for the Execution API. # # This variable can be a single value, or a comma-separated string, in which case the first value is the # one that will be used when generating, and the others are accepted at validation time. # # Not required, but strongly encouraged # # See also :ref:`config:api_auth__jwt_audience` # # Variable: AIRFLOW__EXECUTION_API__JWT_AUDIENCE # jwt_audience = urn:airflow.apache.org:task [lineage] # what lineage backend to use # # Variable: AIRFLOW__LINEAGE__BACKEND # backend = [operators] # The default owner assigned to each new operator, unless # provided explicitly or passed via ``default_args`` # # Variable: AIRFLOW__OPERATORS__DEFAULT_OWNER # default_owner = airflow # The default value of attribute "deferrable" in operators and sensors. # # Variable: AIRFLOW__OPERATORS__DEFAULT_DEFERRABLE # default_deferrable = false # Indicates the default number of CPU units allocated to each operator when no specific CPU request # is specified in the operator's configuration # # Variable: AIRFLOW__OPERATORS__DEFAULT_CPUS # default_cpus = 1 # Indicates the default number of RAM allocated to each operator when no specific RAM request # is specified in the operator's configuration # # Variable: AIRFLOW__OPERATORS__DEFAULT_RAM # default_ram = 512 # Indicates the default number of disk storage allocated to each operator when no specific disk request # is specified in the operator's configuration # # Variable: AIRFLOW__OPERATORS__DEFAULT_DISK # default_disk = 512 # Indicates the default number of GPUs allocated to each operator when no specific GPUs request # is specified in the operator's configuration # # Variable: AIRFLOW__OPERATORS__DEFAULT_GPUS # default_gpus = 0 # Default queue that tasks get assigned to and that worker listen on. # # Variable: AIRFLOW__OPERATORS__DEFAULT_QUEUE # default_queue = default [webserver] # Sorting order in grid view. Valid values are: ``topological``, ``hierarchical_alphabetical`` # # Variable: AIRFLOW__WEBSERVER__GRID_VIEW_SORTING_ORDER # grid_view_sorting_order = topological # The amount of time (in secs) webserver will wait for initial handshake # while fetching logs from other worker machine # # Variable: AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC # log_fetch_timeout_sec = 5 # By default, the webserver shows paused DAGs. Flip this to hide paused # DAGs by default # # Variable: AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT # hide_paused_dags_by_default = False # Consistent page size across all listing views in the UI # # Variable: AIRFLOW__WEBSERVER__PAGE_SIZE # page_size = 50 # Default setting for wrap toggle on DAG code and TI log views. # # Variable: AIRFLOW__WEBSERVER__DEFAULT_WRAP # default_wrap = False # Sets a custom page title for the DAGs overview page and site title for all pages # # Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME # # instance_name = # Whether the custom page title for the DAGs overview page contains any Markup language # # Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME_HAS_MARKUP # instance_name_has_markup = False # How frequently, in seconds, the DAG data will auto-refresh in graph or grid view # when auto-refresh is turned on # # Variable: AIRFLOW__WEBSERVER__AUTO_REFRESH_INTERVAL # auto_refresh_interval = 3 # Boolean for displaying warning for publicly viewable deployment # # Variable: AIRFLOW__WEBSERVER__WARN_DEPLOYMENT_EXPOSURE # warn_deployment_exposure = True # Comma separated string of view events to exclude from dag audit view. # All other events will be added minus the ones passed here. # The audit logs in the db will not be affected by this parameter. # # Example: audit_view_excluded_events = cli_task_run,running,success # # Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_EXCLUDED_EVENTS # # audit_view_excluded_events = # Comma separated string of view events to include in dag audit view. # If passed, only these events will populate the dag audit view. # The audit logs in the db will not be affected by this parameter. # # Example: audit_view_included_events = dagrun_cleared,failed # # Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_INCLUDED_EVENTS # # audit_view_included_events = # Require confirmation when changing a DAG in the web UI. This is to prevent accidental changes # to a DAG that may be running on sensitive environments like production. # When set to ``True``, confirmation dialog will be shown when a user tries to Pause/Unpause, # Trigger a DAG # # Variable: AIRFLOW__WEBSERVER__REQUIRE_CONFIRMATION_DAG_CHANGE # require_confirmation_dag_change = False [email] # Configuration email backend and whether to # send email alerts on retry or failure # Email backend to use # # Variable: AIRFLOW__EMAIL__EMAIL_BACKEND # email_backend = airflow.utils.email.send_email_smtp # Email connection to use # # Variable: AIRFLOW__EMAIL__EMAIL_CONN_ID # email_conn_id = smtp_default # Whether email alerts should be sent when a task is retried # # Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY # default_email_on_retry = True # Whether email alerts should be sent when a task failed # # Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE # default_email_on_failure = True # File that will be used as the template for Email subject (which will be rendered using Jinja2). # If not set, Airflow uses a base template. # # Example: subject_template = /path/to/my_subject_template_file # # Variable: AIRFLOW__EMAIL__SUBJECT_TEMPLATE # # subject_template = # File that will be used as the template for Email content (which will be rendered using Jinja2). # If not set, Airflow uses a base template. # # Example: html_content_template = /path/to/my_html_content_template_file # # Variable: AIRFLOW__EMAIL__HTML_CONTENT_TEMPLATE # # html_content_template = # Email address that will be used as sender address. # It can either be raw email or the complete address in a format ``Sender Name `` # # Example: from_email = Airflow # # Variable: AIRFLOW__EMAIL__FROM_EMAIL # # from_email = # ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default" # which sets it to ``ssl.create_default_context()`` which provides the right balance between # compatibility and security, it however requires that certificates in your operating system are # updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public # keys installed on your machines. You can switch it to "none" if you want to disable checking # of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks # if your infrastructure is not sufficiently secured. It should only be set temporarily while you # are fixing your certificate configuration. This can be typically done by upgrading to newer # version of the operating system you run Airflow components on,by upgrading/refreshing proper # certificates in the OS or by updating certificates for your mail servers. # # Example: ssl_context = default # # Variable: AIRFLOW__EMAIL__SSL_CONTEXT # ssl_context = default [smtp] # If you want airflow to send emails on retries, failure, and you want to use # the airflow.utils.email.send_email_smtp function, you have to configure an # smtp server here # Specifies the host server address used by Airflow when sending out email notifications via SMTP. # # Variable: AIRFLOW__SMTP__SMTP_HOST # smtp_host = localhost # Determines whether to use the STARTTLS command when connecting to the SMTP server. # # Variable: AIRFLOW__SMTP__SMTP_STARTTLS # smtp_starttls = True # Determines whether to use an SSL connection when talking to the SMTP server. # # Variable: AIRFLOW__SMTP__SMTP_SSL # smtp_ssl = False # Defines the port number on which Airflow connects to the SMTP server to send email notifications. # # Variable: AIRFLOW__SMTP__SMTP_PORT # smtp_port = 25 # Specifies the default **from** email address used when Airflow sends email notifications. # # Variable: AIRFLOW__SMTP__SMTP_MAIL_FROM # smtp_mail_from = airflow@example.com # Determines the maximum time (in seconds) the Apache Airflow system will wait for a # connection to the SMTP server to be established. # # Variable: AIRFLOW__SMTP__SMTP_TIMEOUT # smtp_timeout = 30 # Defines the maximum number of times Airflow will attempt to connect to the SMTP server. # # Variable: AIRFLOW__SMTP__SMTP_RETRY_LIMIT # smtp_retry_limit = 5 [sentry] # `Sentry `__ integration. Here you can supply # additional configuration options based on the Python platform. # See `Python / Configuration / Basic Options # `__ for more details. # Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, # ``ignore_errors``, ``before_breadcrumb``, ``transport``. # Enable error reporting to Sentry # # Variable: AIRFLOW__SENTRY__SENTRY_ON # sentry_on = false # # Variable: AIRFLOW__SENTRY__SENTRY_DSN # sentry_dsn = # Dotted path to a before_send function that the sentry SDK should be configured to use. # # Variable: AIRFLOW__SENTRY__BEFORE_SEND # # before_send = [scheduler] # Task instances listen for external kill signal (when you clear tasks # from the CLI or the UI), this defines the frequency at which they should # listen (in seconds). # # Variable: AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC # job_heartbeat_sec = 5 # The scheduler constantly tries to trigger new tasks (look at the # scheduler section in the docs for more information). This defines # how often the scheduler should run (in seconds). # # Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC # scheduler_heartbeat_sec = 5 # The frequency (in seconds) at which the LocalTaskJob should send heartbeat signals to the # scheduler to notify it's still alive. If this value is set to 0, the heartbeat interval will default # to the value of ``[scheduler] task_instance_heartbeat_timeout``. # # Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_SEC # task_instance_heartbeat_sec = 0 # The number of times to try to schedule each DAG file # -1 indicates unlimited number # # Variable: AIRFLOW__SCHEDULER__NUM_RUNS # num_runs = -1 # Controls how long the scheduler will sleep between loops, but if there was nothing to do # in the loop. i.e. if it scheduled something then it will start the next loop # iteration straight away. # # Variable: AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME # scheduler_idle_sleep_time = 1 # How often (in seconds) to check for stale DAGs (DAGs which are no longer present in # the expected files) which should be deactivated, as well as assets that are no longer # referenced and should be marked as orphaned. # # Variable: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL # parsing_cleanup_interval = 60 # How often (in seconds) should pool usage stats be sent to StatsD (if statsd_on is enabled) # # Variable: AIRFLOW__SCHEDULER__POOL_METRICS_INTERVAL # pool_metrics_interval = 5.0 # How often (in seconds) should running task instance stats be sent to StatsD (if statsd_on is enabled) # # Variable: AIRFLOW__SCHEDULER__RUNNING_METRICS_INTERVAL # running_metrics_interval = 30.0 # If the last scheduler heartbeat happened more than ``[scheduler] scheduler_health_check_threshold`` # ago (in seconds), scheduler is considered unhealthy. # This is used by the health check in the **/health** endpoint and in ``airflow jobs check`` CLI # for SchedulerJob. # # Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_THRESHOLD # scheduler_health_check_threshold = 30 # When you start a scheduler, airflow starts a tiny web server # subprocess to serve a health check if this is set to ``True`` # # Variable: AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK # enable_health_check = False # When you start a scheduler, airflow starts a tiny web server # subprocess to serve a health check on this host # # Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_HOST # scheduler_health_check_server_host = 0.0.0.0 # When you start a scheduler, airflow starts a tiny web server # subprocess to serve a health check on this port # # Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_PORT # scheduler_health_check_server_port = 8974 # How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs # # Variable: AIRFLOW__SCHEDULER__ORPHANED_TASKS_CHECK_INTERVAL # orphaned_tasks_check_interval = 300.0 # Local task jobs periodically heartbeat to the DB. If the job has # not heartbeat in this many seconds, the scheduler will mark the # associated task instance as failed and will re-schedule the task. # # Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_TIMEOUT # task_instance_heartbeat_timeout = 300 # How often (in seconds) should the scheduler check for task instances whose heartbeats have timed out. # # Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_TIMEOUT_DETECTION_INTERVAL # task_instance_heartbeat_timeout_detection_interval = 10.0 # Turn on scheduler catchup by setting this to ``True``. # Default behavior is unchanged and # Command Line Backfills still work, but the scheduler # will not do scheduler catchup if this is ``False``, # however it can be set on a per DAG basis in the # DAG definition (catchup) # # Variable: AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT # catchup_by_default = False # Setting this to ``True`` will make first task instance of a task # ignore depends_on_past setting. A task instance will be considered # as the first task instance of a task when there is no task instance # in the DB with a logical_date earlier than it., i.e. no manual marking # success will be needed for a newly added task to be scheduled. # # Variable: AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT # ignore_first_depends_on_past_by_default = True # This determines the number of task instances to be evaluated for scheduling # during each scheduler loop. # Set this to 0 to use the value of ``[core] parallelism`` # # Variable: AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY # max_tis_per_query = 16 # Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. # If this is set to ``False`` then you should not run more than a single # scheduler at once # # Variable: AIRFLOW__SCHEDULER__USE_ROW_LEVEL_LOCKING # use_row_level_locking = True # Max number of DAGs to create DagRuns for per scheduler loop. # # Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_TO_CREATE_PER_LOOP # max_dagruns_to_create_per_loop = 10 # How many DagRuns should a scheduler examine (and lock) when scheduling # and queuing tasks. # # Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_PER_LOOP_TO_SCHEDULE # max_dagruns_per_loop_to_schedule = 20 # Time in seconds after which dags, which were not updated by Dag Processor are deactivated. # # Variable: AIRFLOW__SCHEDULER__DAG_STALE_NOT_SEEN_DURATION # dag_stale_not_seen_duration = 600 # Turn off scheduler use of cron intervals by setting this to ``False``. # DAGs submitted manually in the web UI or with trigger_dag will still run. # # Variable: AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE # use_job_schedule = True # How often to check for expired trigger requests that have not run yet. # # Variable: AIRFLOW__SCHEDULER__TRIGGER_TIMEOUT_CHECK_INTERVAL # trigger_timeout_check_interval = 15 # Amount of time a task can be in the queued state before being retried or set to failed. # # Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT # task_queued_timeout = 600.0 # How often to check for tasks that have been in the queued state for # longer than ``[scheduler] task_queued_timeout``. # # Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT_CHECK_INTERVAL # task_queued_timeout_check_interval = 120.0 # The run_id pattern used to verify the validity of user input to the run_id parameter when # triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id # for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. # # Variable: AIRFLOW__SCHEDULER__ALLOWED_RUN_ID_PATTERN # allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$ # Whether to create DAG runs that span an interval or one single point in time for cron schedules, when # a cron string is provided to ``schedule`` argument of a DAG. # # * ``True``: **CronDataIntervalTimetable** is used, which is suitable # for DAGs with well-defined data interval. You get contiguous intervals from the end of the previous # interval up to the scheduled datetime. # * ``False``: **CronTriggerTimetable** is used, which is closer to the behavior of cron itself. # # Notably, for **CronTriggerTimetable**, the logical date is the same as the time the DAG Run will # try to schedule, while for **CronDataIntervalTimetable**, the logical date is the beginning of # the data interval, but the DAG Run will try to schedule at the end of the data interval. # # Variable: AIRFLOW__SCHEDULER__CREATE_CRON_DATA_INTERVALS # create_cron_data_intervals = False # Whether to create DAG runs that span an interval or one single point in time when a timedelta or # relativedelta is provided to ``schedule`` argument of a DAG. # # * ``True``: **DeltaDataIntervalTimetable** is used, which is suitable for DAGs with well-defined data # interval. You get contiguous intervals from the end of the previous interval up to the scheduled # datetime. # * ``False``: **DeltaTriggerTimetable** is used, which is suitable for DAGs that simply want to say # e.g. "run this every day" and do not care about the data interval. # # Notably, for **DeltaTriggerTimetable**, the logical date is the same as the time the DAG Run will # try to schedule, while for **DeltaDataIntervalTimetable**, the logical date is the beginning of # the data interval, but the DAG Run will try to schedule at the end of the data interval. # # Variable: AIRFLOW__SCHEDULER__CREATE_DELTA_DATA_INTERVALS # create_delta_data_intervals = False # Whether to enable memory allocation tracing in the scheduler. If enabled, Airflow will start # tracing memory allocation and log the top 10 memory usages at the error level upon receiving the # signal SIGUSR1. # This is an expensive operation and generally should not be used except for debugging purposes. # # Variable: AIRFLOW__SCHEDULER__ENABLE_TRACEMALLOC # enable_tracemalloc = False [triggerer] # How many triggers a single Triggerer will run at once, by default. # # Variable: AIRFLOW__TRIGGERER__CAPACITY # capacity = 1000 # How often to heartbeat the Triggerer job to ensure it hasn't been killed. # # Variable: AIRFLOW__TRIGGERER__JOB_HEARTBEAT_SEC # job_heartbeat_sec = 5 # If the last triggerer heartbeat happened more than ``[triggerer] triggerer_health_check_threshold`` # ago (in seconds), triggerer is considered unhealthy. # This is used by the health check in the **/health** endpoint and in ``airflow jobs check`` CLI # for TriggererJob. # # Variable: AIRFLOW__TRIGGERER__TRIGGERER_HEALTH_CHECK_THRESHOLD # triggerer_health_check_threshold = 30 [kerberos] # Location of your ccache file once kinit has been performed. # # Variable: AIRFLOW__KERBEROS__CCACHE # ccache = /tmp/airflow_krb5_ccache # gets augmented with fqdn # # Variable: AIRFLOW__KERBEROS__PRINCIPAL # principal = airflow # Determines the frequency at which initialization or re-initialization processes occur. # # Variable: AIRFLOW__KERBEROS__REINIT_FREQUENCY # reinit_frequency = 3600 # Path to the kinit executable # # Variable: AIRFLOW__KERBEROS__KINIT_PATH # kinit_path = kinit # Designates the path to the Kerberos keytab file for the Airflow user # # Variable: AIRFLOW__KERBEROS__KEYTAB # keytab = airflow.keytab # Allow to disable ticket forwardability. # # Variable: AIRFLOW__KERBEROS__FORWARDABLE # forwardable = True # Allow to remove source IP from token, useful when using token behind NATted Docker host. # # Variable: AIRFLOW__KERBEROS__INCLUDE_IP # include_ip = True [sensors] # Sensor default timeout, 7 days by default (7 * 24 * 60 * 60). # # Variable: AIRFLOW__SENSORS__DEFAULT_TIMEOUT # default_timeout = 604800 [dag_processor] # Configuration for the Airflow DAG processor. This includes, for example: # - DAG bundles, which allows Airflow to load DAGs from different sources # - Parsing configuration, like: # - how often to refresh DAGs from those sources # - how many files to parse concurrently # String path to folder where Airflow bundles can store files locally. Not templated. # If no path is provided, Airflow will use ``Path(tempfile.gettempdir()) / "airflow"``. # This path must be absolute. # # Example: dag_bundle_storage_path = /tmp/some-place # # Variable: AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH # # dag_bundle_storage_path = # List of backend configs. Must supply name, classpath, and kwargs for each backend. # # By default, ``refresh_interval`` is set to ``[dag_processor] refresh_interval``, but that can # also be overridden in kwargs if desired. # # The default is the dags folder dag bundle. # # Note: As shown below, you can split your json config over multiple lines by indenting. # See configparser documentation for an example: # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure. # # Example: dag_bundle_config_list = [ # { # "name": "my-git-repo", # "classpath": "airflow.providers.git.bundles.git.GitDagBundle", # "kwargs": { # "subdir": "dags", # "tracking_ref": "main", # "refresh_interval": 0 # } # } # ] # # Variable: AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST # dag_bundle_config_list = [ { "name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {} } ] # How often (in seconds) to refresh, or look for new files, in a DAG bundle. # # Variable: AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL # refresh_interval = 300 # The DAG processor can run multiple processes in parallel to parse dags. # This defines how many processes will run. # # Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PROCESSES # parsing_processes = 2 # One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. # The DAG processor will list and sort the dag files to decide the parsing order. # # * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the # recently modified DAGs first. # * ``random_seeded_by_host``: Sort randomly across multiple DAG processors but with same order on the # same host, allowing each processor to parse the files in a different order. # * ``alphabetical``: Sort by filename # # Variable: AIRFLOW__DAG_PROCESSOR__FILE_PARSING_SORT_MODE # file_parsing_sort_mode = modified_time # The maximum number of callbacks that are fetched during a single loop. # # Variable: AIRFLOW__DAG_PROCESSOR__MAX_CALLBACKS_PER_LOOP # max_callbacks_per_loop = 20 # Number of seconds after which a DAG file is parsed. The DAG file is parsed every # ``[dag_processor] min_file_process_interval`` number of seconds. Updates to DAGs are reflected after # this interval. Keeping this number low will increase CPU usage. # # Variable: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL # min_file_process_interval = 30 # How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale # DAGs (DAGs which are no longer present in the expected files). The reason why we need # this threshold is to account for the time between when the file is parsed and when the # DAG is loaded. The absolute maximum that this could take is # ``[dag_processor] dag_file_processor_timeout``, but when you have a long timeout configured, # it results in a significant delay in the deactivation of stale dags. # # Variable: AIRFLOW__DAG_PROCESSOR__STALE_DAG_THRESHOLD # stale_dag_threshold = 50 # How long before timing out a DagFileProcessor, which processes a dag file # # Variable: AIRFLOW__DAG_PROCESSOR__DAG_FILE_PROCESSOR_TIMEOUT # dag_file_processor_timeout = 50 # How often should DAG processor stats be printed to the logs. Setting to 0 will disable printing stats # # Variable: AIRFLOW__DAG_PROCESSOR__PRINT_STATS_INTERVAL # print_stats_interval = 30 # Always run tasks with the latest code. If set to True, the bundle version will not # be stored on the dag run and therefore, the latest code will always be used. # # Variable: AIRFLOW__DAG_PROCESSOR__DISABLE_BUNDLE_VERSIONING # disable_bundle_versioning = False # How often the DAG processor should check if any DAG bundles are ready for a refresh, either by hitting # the bundles refresh_interval or because another DAG processor has seen a newer version of the bundle. # A low value means we check more frequently, and have a smaller window of time where DAG processors are # out of sync with each other, parsing different versions of the same bundle. # # Variable: AIRFLOW__DAG_PROCESSOR__BUNDLE_REFRESH_CHECK_INTERVAL # bundle_refresh_check_interval = 5 # On shared workers, bundle copies accumulate in local storage as tasks run # and version of the bundle changes. # This setting represents the delta in seconds between checks for these stale bundles. # Bundles which are older than `stale_bundle_cleanup_age_threshold` may be removed. But # we always keep `stale_bundle_cleanup_min_versions` versions locally. # Set to 0 or negative to disable. # # Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_INTERVAL # stale_bundle_cleanup_interval = 1800 # Bundle versions used more recently than this threshold will not be removed. # Recency of use is determined by when the task began running on the worker, # that age is compared with this setting, given as time delta in seconds. # # Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_AGE_THRESHOLD # stale_bundle_cleanup_age_threshold = 21600 # Minimum number of local bundle versions to retain on disk. # Local bundle versions older than `stale_bundle_cleanup_age_threshold` will # only be deleted we have more than `stale_bundle_cleanup_min_versions` versions # accumulated on the worker. # # Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_MIN_VERSIONS # stale_bundle_cleanup_min_versions = 10 # The dag_processor reads dag files to extract the airflow modules that are going to be used, # and imports them ahead of time to avoid having to re-do it for each parsing process. # This flag can be set to ``False`` to disable this behavior in case an airflow module needs # to be freshly imported each time (at the cost of increased DAG parsing time). # # Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PRE_IMPORT_MODULES # parsing_pre_import_modules = True [common.io] # Common IO configuration section # Path to a location on object storage where XComs can be stored in url format. # # Example: xcom_objectstorage_path = s3://conn_id@bucket/path # # Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_PATH # xcom_objectstorage_path = # Threshold in bytes for storing XComs in object storage. -1 means always store in the # database. 0 means always store in object storage. Any positive number means # it will be stored in object storage if the size of the value is greater than the threshold. # # Example: xcom_objectstorage_threshold = 1000000 # # Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_THRESHOLD # xcom_objectstorage_threshold = -1 # Compression algorithm to use when storing XComs in object storage. Supported algorithms # are a.o.: snappy, zip, gzip, bz2, and lzma. If not specified, no compression will be used. # Note that the compression algorithm must be available in the Python installation (e.g. # python-snappy for snappy). Zip, gz, bz2 are available by default. # # Example: xcom_objectstorage_compression = gz # # Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_COMPRESSION # xcom_objectstorage_compression = [imap] # Options for IMAP provider. # ssl_context = [standard] # Options for the standard provider operators. # Which python tooling should be used to install the virtual environment. # # The following options are available: # - ``auto``: Automatically select, use ``uv`` if available, otherwise use ``pip``. # - ``pip``: Use pip to install the virtual environment. # - ``uv``: Use uv to install the virtual environment. Must be available in environment PATH. # # Example: venv_install_method = uv # # Variable: AIRFLOW__STANDARD__VENV_INSTALL_METHOD # venv_install_method = auto