diff --git a/config/airflow/airflow.cfg b/config/airflow/airflow.cfg new file mode 100644 index 0000000..aa8f716 --- /dev/null +++ b/config/airflow/airflow.cfg @@ -0,0 +1,2358 @@ +[core] +# The folder where your airflow pipelines live, most likely a +# subfolder in a code repository. This path must be absolute. +# +# Variable: AIRFLOW__CORE__DAGS_FOLDER +# +dags_folder = /root/airflow/dags + +# Hostname by providing a path to a callable, which will resolve the hostname. +# The format is "package.function". +# +# For example, default value ``airflow.utils.net.getfqdn`` means that result from patched +# version of `socket.getfqdn() `__, +# see related `CPython Issue `__. +# +# No argument should be required in the function specified. +# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` +# +# Variable: AIRFLOW__CORE__HOSTNAME_CALLABLE +# +hostname_callable = airflow.utils.net.getfqdn + +# A callable to check if a python file has airflow dags defined or not and should +# return ``True`` if it has dags otherwise ``False``. +# If this is not provided, Airflow uses its own heuristic rules. +# +# The function should have the following signature +# +# .. code-block:: python +# +# def func_name(file_path: str, zip_file: zipfile.ZipFile | None = None) -> bool: ... +# +# Variable: AIRFLOW__CORE__MIGHT_CONTAIN_DAG_CALLABLE +# +might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_heuristic + +# Default timezone in case supplied date times are naive +# can be `UTC` (default), `system`, or any `IANA ` +# timezone string (e.g. Europe/Amsterdam) +# +# Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE +# +default_timezone = utc + +# The executor class that airflow should use. Choices include +# ``LocalExecutor``, ``CeleryExecutor``, +# ``KubernetesExecutor`` or the full import path to the class when using a custom executor. +# +# Variable: AIRFLOW__CORE__EXECUTOR +# +executor = LocalExecutor + +# The auth manager class that airflow should use. Full import path to the auth manager class. +# +# Variable: AIRFLOW__CORE__AUTH_MANAGER +# +auth_manager = airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager + +# The list of users and their associated role in simple auth manager. If the simple auth manager is +# used in your environment, this list controls who can access the environment. +# +# List of user-role delimited with a comma. Each user-role is a colon delimited couple of username and +# role. Roles are predefined in simple auth managers: viewer, user, op, admin. +# +# Example: simple_auth_manager_users = bob:admin,peter:viewer +# +# Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS +# +simple_auth_manager_users = admin:admin,mo:admin,Bart:admin + +# Whether to disable authentication and allow everyone as admin in the environment. +# +# Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS +# +simple_auth_manager_all_admins = False + +# The json file where the simple auth manager stores passwords for the configured users. +# By default this is ``AIRFLOW_HOME/simple_auth_manager_passwords.json.generated``. +# +# Example: simple_auth_manager_passwords_file = /path/to/passwords.json +# +# Variable: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE +# +# simple_auth_manager_passwords_file = + +# This defines the maximum number of task instances that can run concurrently per scheduler in +# Airflow, regardless of the worker count. Generally this value, multiplied by the number of +# schedulers in your cluster, is the maximum number of task instances with the running +# state in the metadata database. The value must be larger or equal 1. +# +# Variable: AIRFLOW__CORE__PARALLELISM +# +parallelism = 32 + +# The maximum number of task instances allowed to run concurrently in each DAG. To calculate +# the number of tasks that is running concurrently for a DAG, add up the number of running +# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``, +# which is defaulted as ``[core] max_active_tasks_per_dag``. +# +# An example scenario when this would be useful is when you want to stop a new dag with an early +# start date from stealing all the executor slots in a cluster. +# +# Variable: AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG +# +max_active_tasks_per_dag = 16 + +# Are DAGs paused by default at creation +# +# Variable: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION +# +dags_are_paused_at_creation = True + +# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs +# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, +# which is defaulted as ``[core] max_active_runs_per_dag``. +# +# Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG +# +max_active_runs_per_dag = 16 + +# (experimental) The maximum number of consecutive DAG failures before DAG is automatically paused. +# This is also configurable per DAG level with ``max_consecutive_failed_dag_runs``, +# which is defaulted as ``[core] max_consecutive_failed_dag_runs_per_dag``. +# If not specified, then the value is considered as 0, +# meaning that the dags are never paused out by default. +# +# Variable: AIRFLOW__CORE__MAX_CONSECUTIVE_FAILED_DAG_RUNS_PER_DAG +# +max_consecutive_failed_dag_runs_per_dag = 0 + +# The name of the method used in order to start Python processes via the multiprocessing module. +# This corresponds directly with the options available in the Python docs: +# `multiprocessing.set_start_method +# `__ +# must be one of the values returned by `multiprocessing.get_all_start_methods() +# `__. +# +# Example: mp_start_method = fork +# +# Variable: AIRFLOW__CORE__MP_START_METHOD +# +# mp_start_method = + +# Whether to load the DAG examples that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production +# environment +# +# Variable: AIRFLOW__CORE__LOAD_EXAMPLES +# +load_examples = False + +# Path to the folder containing Airflow plugins +# +# Variable: AIRFLOW__CORE__PLUGINS_FOLDER +# +plugins_folder = /root/airflow/plugins + +# Should tasks be executed via forking of the parent process +# +# * ``False``: Execute via forking of the parent process +# * ``True``: Spawning a new python process, slower than fork, but means plugin changes picked +# up by tasks straight away +# +# Variable: AIRFLOW__CORE__EXECUTE_TASKS_NEW_PYTHON_INTERPRETER +# +execute_tasks_new_python_interpreter = False + +# Secret key to save connection passwords in the db +# +# Variable: AIRFLOW__CORE__FERNET_KEY +# +fernet_key = f4n7uT66HPrl8yxqKHVtme_qINN9YlZ1xUQcZy761MY= + +# Whether to disable pickling dags +# +# Variable: AIRFLOW__CORE__DONOT_PICKLE +# +donot_pickle = True + +# How long before timing out a python file import +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT +# +dagbag_import_timeout = 30.0 + +# Should a traceback be shown in the UI for dagbag import errors, +# instead of just the exception message +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACKS +# +dagbag_import_error_tracebacks = True + +# If tracebacks are shown, how many entries from the traceback should be shown +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACK_DEPTH +# +dagbag_import_error_traceback_depth = 2 + +# If set, tasks without a ``run_as_user`` argument will be run with this user +# Can be used to de-elevate a sudo user running Airflow when executing tasks +# +# Variable: AIRFLOW__CORE__DEFAULT_IMPERSONATION +# +default_impersonation = + +# What security module to use (for example kerberos) +# +# Variable: AIRFLOW__CORE__SECURITY +# +security = + +# Turn unit test mode on (overwrites many configuration options with test +# values at runtime) +# +# Variable: AIRFLOW__CORE__UNIT_TEST_MODE +# +unit_test_mode = False + +# Space-separated list of classes that may be imported during deserialization. Items can be glob +# expressions. Python built-in classes (like dict) are always allowed. +# +# Example: allowed_deserialization_classes = airflow.* my_mod.my_other_mod.TheseClasses* +# +# Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES +# +allowed_deserialization_classes = airflow.* + +# Space-separated list of classes that may be imported during deserialization. Items are processed +# as regex expressions. Python built-in classes (like dict) are always allowed. +# This is a secondary option to ``[core] allowed_deserialization_classes``. +# +# Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES_REGEXP +# +allowed_deserialization_classes_regexp = + +# When a task is killed forcefully, this is the amount of time in seconds that +# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED +# +# Variable: AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME +# +killed_task_cleanup_time = 60 + +# Whether to override params with dag_run.conf. If you pass some key-value pairs +# through ``airflow dags backfill -c`` or +# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. +# +# Variable: AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS +# +dag_run_conf_overrides_params = True + +# If enabled, Airflow will only scan files containing both ``DAG`` and ``airflow`` (case-insensitive). +# +# Variable: AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE +# +dag_discovery_safe_mode = True + +# The pattern syntax used in the +# `.airflowignore +# `__ +# files in the DAG directories. Valid values are ``regexp`` or ``glob``. +# +# Variable: AIRFLOW__CORE__DAG_IGNORE_FILE_SYNTAX +# +dag_ignore_file_syntax = glob + +# The number of retries each task is going to have by default. Can be overridden at dag or task level. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRIES +# +default_task_retries = 0 + +# The number of seconds each task is going to wait by default between retries. Can be overridden at +# dag or task level. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRY_DELAY +# +default_task_retry_delay = 300 + +# The maximum delay (in seconds) each task is going to wait by default between retries. +# This is a global setting and cannot be overridden at task or DAG level. +# +# Variable: AIRFLOW__CORE__MAX_TASK_RETRY_DELAY +# +max_task_retry_delay = 86400 + +# The weighting method used for the effective total priority weight of the task +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_WEIGHT_RULE +# +default_task_weight_rule = downstream + +# Maximum possible time (in seconds) that task will have for execution of auxiliary processes +# (like listeners, mini scheduler...) after task is marked as success.. +# +# Variable: AIRFLOW__CORE__TASK_SUCCESS_OVERTIME +# +task_success_overtime = 20 + +# The default task execution_timeout value for the operators. Expected an integer value to +# be passed into timedelta as seconds. If not specified, then the value is considered as None, +# meaning that the operators are never timed out by default. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_EXECUTION_TIMEOUT +# +default_task_execution_timeout = + +# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. +# +# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL +# +min_serialized_dag_update_interval = 30 + +# If ``True``, serialized DAGs are compressed before writing to DB. +# +# .. note:: +# +# This will disable the DAG dependencies view +# +# Variable: AIRFLOW__CORE__COMPRESS_SERIALIZED_DAGS +# +compress_serialized_dags = False + +# Fetching serialized DAG can not be faster than a minimum interval to reduce database +# read rate. This config controls when your DAGs are updated in the Webserver +# +# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_FETCH_INTERVAL +# +min_serialized_dag_fetch_interval = 10 + +# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store +# in the Database. +# All the template_fields for each of Task Instance are stored in the Database. +# Keeping this number small may cause an error when you try to view ``Rendered`` tab in +# TaskInstance view for older tasks. +# +# Variable: AIRFLOW__CORE__MAX_NUM_RENDERED_TI_FIELDS_PER_TASK +# +max_num_rendered_ti_fields_per_task = 30 + +# Path to custom XCom class that will be used to store and resolve operators results +# +# Example: xcom_backend = path.to.CustomXCom +# +# Variable: AIRFLOW__CORE__XCOM_BACKEND +# +xcom_backend = airflow.sdk.execution_time.xcom.BaseXCom + +# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, +# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. +# +# Variable: AIRFLOW__CORE__LAZY_LOAD_PLUGINS +# +lazy_load_plugins = True + +# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). +# Set it to ``False``, if you want to discover providers whenever 'airflow' is invoked via cli or +# loaded from module. +# +# Variable: AIRFLOW__CORE__LAZY_DISCOVER_PROVIDERS +# +lazy_discover_providers = True + +# Hide sensitive **Variables** or **Connection extra json keys** from UI +# and task logs when set to ``True`` +# +# .. note:: +# +# Connection passwords are always hidden in logs +# +# Variable: AIRFLOW__CORE__HIDE_SENSITIVE_VAR_CONN_FIELDS +# +hide_sensitive_var_conn_fields = True + +# A comma-separated list of extra sensitive keywords to look for in variables names or connection's +# extra JSON. +# +# Variable: AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES +# +sensitive_var_conn_names = + +# Task Slot counts for ``default_pool``. This setting would not have any effect in an existing +# deployment where the ``default_pool`` is already created. For existing deployments, users can +# change the number of slots using Webserver, API or the CLI +# +# Variable: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT +# +default_pool_task_slot_count = 128 + +# The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a +# length exceeding this value, the task pushing the XCom will be failed automatically to prevent the +# mapped tasks from clogging the scheduler. +# +# Variable: AIRFLOW__CORE__MAX_MAP_LENGTH +# +max_map_length = 1024 + +# The default umask to use for process when run in daemon mode (scheduler, worker, etc.) +# +# This controls the file-creation mode mask which determines the initial value of file permission bits +# for newly created files. +# +# This value is treated as an octal-integer. +# +# Variable: AIRFLOW__CORE__DAEMON_UMASK +# +daemon_umask = 0o077 + +# Class to use as asset manager. +# +# Example: asset_manager_class = airflow.assets.manager.AssetManager +# +# Variable: AIRFLOW__CORE__ASSET_MANAGER_CLASS +# +# asset_manager_class = + +# Kwargs to supply to asset manager. +# +# Example: asset_manager_kwargs = {"some_param": "some_value"} +# +# Variable: AIRFLOW__CORE__ASSET_MANAGER_KWARGS +# +# asset_manager_kwargs = + +# (experimental) Whether components should use Airflow Internal API for DB connectivity. +# +# Variable: AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION +# +database_access_isolation = False + +# (experimental) Airflow Internal API url. +# Only used if ``[core] database_access_isolation`` is ``True``. +# +# Example: internal_api_url = http://localhost:8080 +# +# Variable: AIRFLOW__CORE__INTERNAL_API_URL +# +# internal_api_url = + +# Secret key used to authenticate internal API clients to core. It should be as random as possible. +# However, when running more than 1 instances of webserver / internal API services, make sure all +# of them use the same ``secret_key`` otherwise calls will fail on authentication. +# The authentication token generated using the secret key has a short expiry time though - make +# sure that time on ALL the machines that you run airflow components on is synchronized +# (for example using ntpd) otherwise you might get "forbidden" errors when the logs are accessed. +# +# Variable: AIRFLOW__CORE__INTERNAL_API_SECRET_KEY +# +internal_api_secret_key = 010RL08807/JBjH4cWzNaw== + +# The ability to allow testing connections across Airflow UI, API and CLI. +# Supported options: ``Disabled``, ``Enabled``, ``Hidden``. Default: Disabled +# Disabled - Disables the test connection functionality and disables the Test Connection button in UI. +# Enabled - Enables the test connection functionality and shows the Test Connection button in UI. +# Hidden - Disables the test connection functionality and hides the Test Connection button in UI. +# Before setting this to Enabled, make sure that you review the users who are able to add/edit +# connections and ensure they are trusted. Connection testing can be done maliciously leading to +# undesired and insecure outcomes. +# See `Airflow Security Model: Capabilities of authenticated UI users +# `__ +# for more details. +# +# Variable: AIRFLOW__CORE__TEST_CONNECTION +# +test_connection = Disabled + +# The maximum length of the rendered template field. If the value to be stored in the +# rendered template field exceeds this size, it's redacted. +# +# Variable: AIRFLOW__CORE__MAX_TEMPLATED_FIELD_LENGTH +# +max_templated_field_length = 4096 + +# The url of the execution api server. Default is ``{BASE_URL}/execution/`` +# where ``{BASE_URL}`` is the base url of the API Server. If ``{BASE_URL}`` is not set, +# it will use ``http://localhost:8080`` as the default base url. +# +# Variable: AIRFLOW__CORE__EXECUTION_API_SERVER_URL +# +# execution_api_server_url = + +[database] +# Path to the ``alembic.ini`` file. You can either provide the file path relative +# to the Airflow home directory or the absolute path if it is located elsewhere. +# +# Variable: AIRFLOW__DATABASE__ALEMBIC_INI_FILE_PATH +# +alembic_ini_file_path = alembic.ini + +# The SQLAlchemy connection string to the metadata database. +# SQLAlchemy supports many different database engines. +# See: `Set up a Database Backend: Database URI +# `__ +# for more details. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN +# +sql_alchemy_conn = sqlite:////root/airflow/airflow.db + +# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value +# +# Example: sql_alchemy_engine_args = {"arg1": true} +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS +# +# sql_alchemy_engine_args = + +# The encoding for the databases +# +# Variable: AIRFLOW__DATABASE__SQL_ENGINE_ENCODING +# +sql_engine_encoding = utf-8 + +# Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id`` columns +# in case they have different encoding. +# By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb`` +# the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed +# the maximum size of allowed index when collation is set to ``utf8mb4`` variant, see +# `GitHub Issue Comment `__ +# for more details. +# +# Variable: AIRFLOW__DATABASE__SQL_ENGINE_COLLATION_FOR_IDS +# +# sql_engine_collation_for_ids = + +# If SQLAlchemy should pool database connections. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_ENABLED +# +sql_alchemy_pool_enabled = True + +# The SQLAlchemy pool size is the maximum number of database connections +# in the pool. 0 indicates no limit. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE +# +sql_alchemy_pool_size = 5 + +# The maximum overflow size of the pool. +# When the number of checked-out connections reaches the size set in pool_size, +# additional connections will be returned up to this limit. +# When those additional connections are returned to the pool, they are disconnected and discarded. +# It follows then that the total number of simultaneous connections the pool will allow +# is **pool_size** + **max_overflow**, +# and the total number of "sleeping" connections the pool will allow is pool_size. +# max_overflow can be set to ``-1`` to indicate no overflow limit; +# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW +# +sql_alchemy_max_overflow = 10 + +# The SQLAlchemy pool recycle is the number of seconds a connection +# can be idle in the pool before it is invalidated. This config does +# not apply to sqlite. If the number of DB connections is ever exceeded, +# a lower config value will allow the system to recover faster. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE +# +sql_alchemy_pool_recycle = 1800 + +# Check connection at the start of each connection pool checkout. +# Typically, this is a simple statement like "SELECT 1". +# See `SQLAlchemy Pooling: Disconnect Handling - Pessimistic +# `__ +# for more details. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_PRE_PING +# +sql_alchemy_pool_pre_ping = True + +# The schema to use for the metadata database. +# SQLAlchemy supports databases with the concept of multiple schemas. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA +# +sql_alchemy_schema = + +# Import path for connect args in SQLAlchemy. Defaults to an empty dict. +# This is useful when you want to configure db engine args that SQLAlchemy won't parse +# in connection string. This can be set by passing a dictionary containing the create engine parameters. +# For more details about passing create engine parameters (keepalives variables, timeout etc) +# in Postgres DB Backend see `Setting up a PostgreSQL Database +# `__ +# e.g ``connect_args={"timeout":30}`` can be defined in ``airflow_local_settings.py`` and +# can be imported as shown below +# +# Example: sql_alchemy_connect_args = airflow_local_settings.connect_args +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONNECT_ARGS +# +# sql_alchemy_connect_args = + +# Important Warning: Use of sql_alchemy_session_maker Highly Discouraged +# Import path for function which returns 'sqlalchemy.orm.sessionmaker'. +# Improper configuration of sql_alchemy_session_maker can lead to serious issues, +# including data corruption, unrecoverable application crashes. Please review the SQLAlchemy +# documentation for detailed guidance on proper configuration and best practices. +# +# Example: sql_alchemy_session_maker = airflow_local_settings._sessionmaker +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SESSION_MAKER +# +# sql_alchemy_session_maker = + +# Number of times the code should be retried in case of DB Operational Errors. +# Not all transactions will be retried as it can cause undesired state. +# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. +# +# Variable: AIRFLOW__DATABASE__MAX_DB_RETRIES +# +max_db_retries = 3 + +# Whether to run alembic migrations during Airflow start up. Sometimes this operation can be expensive, +# and the users can assert the correct version through other means (e.g. through a Helm chart). +# Accepts ``True`` or ``False``. +# +# Variable: AIRFLOW__DATABASE__CHECK_MIGRATIONS +# +check_migrations = True + +# List of DB managers to use to migrate external tables in airflow database. The managers must inherit +# from BaseDBManager. If ``FabAuthManager`` is configured in the environment, +# ``airflow.providers.fab.auth_manager.models.db.FABDBManager`` is automatically added. +# +# Example: external_db_managers = airflow.providers.fab.auth_manager.models.db.FABDBManager +# +# Variable: AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS +# +# external_db_managers = + +# The number of rows to process in each batch when performing a migration. +# This is useful for large tables to avoid locking and failure due to query timeouts. +# +# Variable: AIRFLOW__DATABASE__MIGRATION_BATCH_SIZE +# +migration_batch_size = 10000 + +[logging] +# The folder where airflow should store its log files. +# This path must be absolute. +# There are a few existing configurations that assume this is set to the default. +# If you choose to override this you may need to update the +# ``[logging] dag_processor_manager_log_location`` and +# ``[logging] dag_processor_child_process_log_directory settings`` as well. +# +# Variable: AIRFLOW__LOGGING__BASE_LOG_FOLDER +# +base_log_folder = /root/airflow/logs + +# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. +# Set this to ``True`` if you want to enable remote logging. +# +# Variable: AIRFLOW__LOGGING__REMOTE_LOGGING +# +remote_logging = False + +# Users must supply an Airflow connection id that provides access to the storage +# location. Depending on your remote logging service, this may only be used for +# reading logs, not writing them. +# +# Variable: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID +# +remote_log_conn_id = + +# Whether the local log files for GCS, S3, WASB, HDFS and OSS remote logging should be deleted after +# they are uploaded to the remote location. +# +# Variable: AIRFLOW__LOGGING__DELETE_LOCAL_LOGS +# +delete_local_logs = False + +# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default +# Credentials +# `__ will +# be used. +# +# Variable: AIRFLOW__LOGGING__GOOGLE_KEY_PATH +# +google_key_path = + +# Storage bucket URL for remote logging +# S3 buckets should start with **s3://** +# Cloudwatch log groups should start with **cloudwatch://** +# GCS buckets should start with **gs://** +# WASB buckets should start with **wasb** just to help Airflow select correct handler +# Stackdriver logs should start with **stackdriver://** +# +# Variable: AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER +# +remote_base_log_folder = + +# The remote_task_handler_kwargs param is loaded into a dictionary and passed to the ``__init__`` +# of remote task handler and it overrides the values provided by Airflow config. For example if you set +# ``delete_local_logs=False`` and you provide ``{"delete_local_copy": true}``, then the local +# log files will be deleted after they are uploaded to remote location. +# +# Example: remote_task_handler_kwargs = {"delete_local_copy": true} +# +# Variable: AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS +# +remote_task_handler_kwargs = + +# Use server-side encryption for logs stored in S3 +# +# Variable: AIRFLOW__LOGGING__ENCRYPT_S3_LOGS +# +encrypt_s3_logs = False + +# Logging level. +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__LOGGING_LEVEL +# +logging_level = INFO + +# Logging level for celery. If not set, it uses the value of logging_level +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__CELERY_LOGGING_LEVEL +# +celery_logging_level = + +# Logging level for Flask-appbuilder UI. +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__FAB_LOGGING_LEVEL +# +fab_logging_level = WARNING + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +# +# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG +# +# Variable: AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS +# +logging_config_class = + +# Flag to enable/disable Colored logs in Console +# Colour the logs when the controlling terminal is a TTY. +# +# Variable: AIRFLOW__LOGGING__COLORED_CONSOLE_LOG +# +colored_console_log = True + +# Log format for when Colored logs is enabled +# +# Variable: AIRFLOW__LOGGING__COLORED_LOG_FORMAT +# +colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s + +# Specifies the class utilized by Airflow to implement colored logging +# +# Variable: AIRFLOW__LOGGING__COLORED_FORMATTER_CLASS +# +colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter + +# Format of Log line +# +# Variable: AIRFLOW__LOGGING__LOG_FORMAT +# +log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s + +# Defines the format of log messages for simple logging configuration +# +# Variable: AIRFLOW__LOGGING__SIMPLE_LOG_FORMAT +# +simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s + +# Where to send dag parser logs. If "file", logs are sent to log files defined by child_process_log_directory. +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_TARGET +# +dag_processor_log_target = file + +# Format of Dag Processor Log line +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_FORMAT +# +dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s + +# Determines the directory where logs for the child processes of the dag processor will be stored +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_CHILD_PROCESS_LOG_DIRECTORY +# +dag_processor_child_process_log_directory = /root/airflow/logs/dag_processor + +# Determines the formatter class used by Airflow for structuring its log messages +# The default formatter class is timezone-aware, which means that timestamps attached to log entries +# will be adjusted to reflect the local timezone of the Airflow instance +# +# Variable: AIRFLOW__LOGGING__LOG_FORMATTER_CLASS +# +log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware + +# An import path to a function to add adaptations of each secret added with +# ``airflow.sdk.execution_time.secrets_masker.mask_secret`` to be masked in log messages. +# The given function is expected to require a single parameter: the secret to be adapted. +# It may return a single adaptation of the secret or an iterable of adaptations to each be +# masked as secrets. The original secret will be masked as well as any adaptations returned. +# +# Example: secret_mask_adapter = urllib.parse.quote +# +# Variable: AIRFLOW__LOGGING__SECRET_MASK_ADAPTER +# +secret_mask_adapter = + +# The minimum length of a secret to be masked in log messages. +# Secrets shorter than this length will not be masked. +# +# Variable: AIRFLOW__LOGGING__MIN_LENGTH_MASKED_SECRET +# +min_length_masked_secret = 5 + +# Specify prefix pattern like mentioned below with stream handler ``TaskHandlerWithCustomFormatter`` +# +# Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{logical_date}}-{{ti.try_number}} +# +# Variable: AIRFLOW__LOGGING__TASK_LOG_PREFIX_TEMPLATE +# +task_log_prefix_template = + +# Formatting for how airflow generates file names/paths for each task run. +# +# Variable: AIRFLOW__LOGGING__LOG_FILENAME_TEMPLATE +# +log_filename_template = dag_id={{ ti.dag_id }}/run_id={{ ti.run_id }}/task_id={{ ti.task_id }}/{%% if ti.map_index >= 0 %%}map_index={{ ti.map_index }}/{%% endif %%}attempt={{ try_number|default(ti.try_number) }}.log + +# Name of handler to read task instance logs. +# Defaults to use ``task`` handler. +# +# Variable: AIRFLOW__LOGGING__TASK_LOG_READER +# +task_log_reader = task + +# A comma\-separated list of third-party logger names that will be configured to print messages to +# consoles\. +# +# Example: extra_logger_names = fastapi,sqlalchemy +# +# Variable: AIRFLOW__LOGGING__EXTRA_LOGGER_NAMES +# +extra_logger_names = + +# When you start an Airflow worker, Airflow starts a tiny web server +# subprocess to serve the workers local log files to the airflow main +# web server, who then builds pages and sends them to users. This defines +# the port on which the logs are served. It needs to be unused, and open +# visible from the main web server to connect into the workers. +# +# Variable: AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT +# +worker_log_server_port = 8793 + +# Port to serve logs from for triggerer. +# See ``[logging] worker_log_server_port`` description for more info. +# +# Variable: AIRFLOW__LOGGING__TRIGGER_LOG_SERVER_PORT +# +trigger_log_server_port = 8794 + +# We must parse timestamps to interleave logs between trigger and task. To do so, +# we need to parse timestamps in log files. In case your log format is non-standard, +# you may provide import path to callable which takes a string log line and returns +# the timestamp (datetime.datetime compatible). +# +# Example: interleave_timestamp_parser = path.to.my_func +# +# Variable: AIRFLOW__LOGGING__INTERLEAVE_TIMESTAMP_PARSER +# +# interleave_timestamp_parser = + +# Permissions in the form or of octal string as understood by chmod. The permissions are important +# when you use impersonation, when logs are written by a different user than airflow. The most secure +# way of configuring it in this case is to add both users to the same group and make it the default +# group of both users. Group-writeable logs are default in airflow, but you might decide that you are +# OK with having the logs other-writeable, in which case you should set it to ``0o777``. You might +# decide to add more security if you do not use impersonation and change it to ``0o755`` to make it +# only owner-writeable. You can also make it just readable only for owner by changing it to ``0o700`` +# if all the access (read/write) for your logs happens from the same user. +# +# Example: file_task_handler_new_folder_permissions = 0o775 +# +# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FOLDER_PERMISSIONS +# +file_task_handler_new_folder_permissions = 0o775 + +# Permissions in the form or of octal string as understood by chmod. The permissions are important +# when you use impersonation, when logs are written by a different user than airflow. The most secure +# way of configuring it in this case is to add both users to the same group and make it the default +# group of both users. Group-writeable logs are default in airflow, but you might decide that you are +# OK with having the logs other-writeable, in which case you should set it to ``0o666``. You might +# decide to add more security if you do not use impersonation and change it to ``0o644`` to make it +# only owner-writeable. You can also make it just readable only for owner by changing it to ``0o600`` +# if all the access (read/write) for your logs happens from the same user. +# +# Example: file_task_handler_new_file_permissions = 0o664 +# +# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FILE_PERMISSIONS +# +file_task_handler_new_file_permissions = 0o664 + +# By default Celery sends all logs into stderr. +# If enabled any previous logging handlers will get *removed*. +# With this option AirFlow will create new handlers +# and send low level logs like INFO and WARNING to stdout, +# while sending higher severity logs to stderr. +# +# Variable: AIRFLOW__LOGGING__CELERY_STDOUT_STDERR_SEPARATION +# +celery_stdout_stderr_separation = False + +# A comma separated list of keywords related to errors whose presence should display the line in red +# color in UI +# +# Variable: AIRFLOW__LOGGING__COLOR_LOG_ERROR_KEYWORDS +# +color_log_error_keywords = error,exception + +# A comma separated list of keywords related to warning whose presence should display the line in yellow +# color in UI +# +# Variable: AIRFLOW__LOGGING__COLOR_LOG_WARNING_KEYWORDS +# +color_log_warning_keywords = warn + +[metrics] +# `StatsD `__ integration settings. + +# Configure an allow list (comma separated regex patterns to match) to send only certain metrics. +# +# Example: metrics_allow_list = "scheduler,executor,dagrun,pool,triggerer,celery" or "^scheduler,^executor,heartbeat|timeout" +# +# Variable: AIRFLOW__METRICS__METRICS_ALLOW_LIST +# +metrics_allow_list = + +# Configure a block list (comma separated regex patterns to match) to block certain metrics +# from being emitted. +# If ``[metrics] metrics_allow_list`` and ``[metrics] metrics_block_list`` are both configured, +# ``[metrics] metrics_block_list`` is ignored. +# +# Example: metrics_block_list = "scheduler,executor,dagrun,pool,triggerer,celery" or "^scheduler,^executor,heartbeat|timeout" +# +# Variable: AIRFLOW__METRICS__METRICS_BLOCK_LIST +# +metrics_block_list = + +# Enables sending metrics to StatsD. +# +# Variable: AIRFLOW__METRICS__STATSD_ON +# +statsd_on = False + +# Specifies the host address where the StatsD daemon (or server) is running +# +# Variable: AIRFLOW__METRICS__STATSD_HOST +# +statsd_host = localhost + +# Enables the statsd host to be resolved into IPv6 address +# +# Variable: AIRFLOW__METRICS__STATSD_IPV6 +# +statsd_ipv6 = False + +# Specifies the port on which the StatsD daemon (or server) is listening to +# +# Variable: AIRFLOW__METRICS__STATSD_PORT +# +statsd_port = 8125 + +# Defines the namespace for all metrics sent from Airflow to StatsD +# +# Variable: AIRFLOW__METRICS__STATSD_PREFIX +# +statsd_prefix = airflow + +# A function that validate the StatsD stat name, apply changes to the stat name if necessary and return +# the transformed stat name. +# +# The function should have the following signature +# +# .. code-block:: python +# +# def func_name(stat_name: str) -> str: ... +# +# Variable: AIRFLOW__METRICS__STAT_NAME_HANDLER +# +stat_name_handler = + +# To enable datadog integration to send airflow metrics. +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_ENABLED +# +statsd_datadog_enabled = False + +# List of datadog tags attached to all metrics(e.g: ``key1:value1,key2:value2``) +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_TAGS +# +statsd_datadog_tags = + +# Set to ``False`` to disable metadata tags for some of the emitted metrics +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_METRICS_TAGS +# +statsd_datadog_metrics_tags = True + +# If you want to utilise your own custom StatsD client set the relevant +# module path below. +# Note: The module path must exist on your +# `PYTHONPATH ` +# for Airflow to pick it up +# +# Variable: AIRFLOW__METRICS__STATSD_CUSTOM_CLIENT_PATH +# +# statsd_custom_client_path = + +# If you want to avoid sending all the available metrics tags to StatsD, +# you can configure a block list of prefixes (comma separated) to filter out metric tags +# that start with the elements of the list (e.g: ``job_id,run_id``) +# +# Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id +# +# Variable: AIRFLOW__METRICS__STATSD_DISABLED_TAGS +# +statsd_disabled_tags = job_id,run_id + +# To enable sending Airflow metrics with StatsD-Influxdb tagging convention. +# +# Variable: AIRFLOW__METRICS__STATSD_INFLUXDB_ENABLED +# +statsd_influxdb_enabled = False + +# Enables sending metrics to OpenTelemetry. +# +# Variable: AIRFLOW__METRICS__OTEL_ON +# +otel_on = False + +# Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends +# metrics and traces. +# +# Variable: AIRFLOW__METRICS__OTEL_HOST +# +otel_host = localhost + +# Specifies the port of the OpenTelemetry Collector that is listening to. +# +# Variable: AIRFLOW__METRICS__OTEL_PORT +# +otel_port = 8889 + +# The prefix for the Airflow metrics. +# +# Variable: AIRFLOW__METRICS__OTEL_PREFIX +# +otel_prefix = airflow + +# Defines the interval, in milliseconds, at which Airflow sends batches of metrics and traces +# to the configured OpenTelemetry Collector. +# +# Variable: AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS +# +otel_interval_milliseconds = 60000 + +# If ``True``, all metrics are also emitted to the console. Defaults to ``False``. +# +# Variable: AIRFLOW__METRICS__OTEL_DEBUGGING_ON +# +otel_debugging_on = False + +# The default service name of traces. +# +# Variable: AIRFLOW__METRICS__OTEL_SERVICE +# +otel_service = Airflow + +# If ``True``, SSL will be enabled. Defaults to ``False``. +# To establish an HTTPS connection to the OpenTelemetry collector, +# you need to configure the SSL certificate and key within the OpenTelemetry collector's +# ``config.yml`` file. +# +# Variable: AIRFLOW__METRICS__OTEL_SSL_ACTIVE +# +otel_ssl_active = False + +[traces] +# Distributed traces integration settings. + +# Enables sending traces to OpenTelemetry. +# +# Variable: AIRFLOW__TRACES__OTEL_ON +# +otel_on = False + +# Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends +# traces. +# +# Variable: AIRFLOW__TRACES__OTEL_HOST +# +otel_host = localhost + +# Specifies the port of the OpenTelemetry Collector that is listening to. +# +# Variable: AIRFLOW__TRACES__OTEL_PORT +# +otel_port = 8889 + +# The default service name of traces. +# +# Variable: AIRFLOW__TRACES__OTEL_SERVICE +# +otel_service = Airflow + +# If True, all traces are also emitted to the console. Defaults to False. +# +# Variable: AIRFLOW__TRACES__OTEL_DEBUGGING_ON +# +otel_debugging_on = False + +# If True, SSL will be enabled. Defaults to False. +# To establish an HTTPS connection to the OpenTelemetry collector, +# you need to configure the SSL certificate and key within the OpenTelemetry collector's +# config.yml file. +# +# Variable: AIRFLOW__TRACES__OTEL_SSL_ACTIVE +# +otel_ssl_active = False + +[secrets] +# Full class name of secrets backend to enable (will precede env vars and metastore in search path) +# +# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend +# +# Variable: AIRFLOW__SECRETS__BACKEND +# +backend = + +# The backend_kwargs param is loaded into a dictionary and passed to ``__init__`` +# of secrets backend class. See documentation for the secrets backend you are using. +# JSON is expected. +# +# Example for AWS Systems Manager ParameterStore: +# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` +# +# Variable: AIRFLOW__SECRETS__BACKEND_KWARGS +# +backend_kwargs = + +# .. note:: |experimental| +# +# Enables local caching of Variables, when parsing DAGs only. +# Using this option can make dag parsing faster if Variables are used in top level code, at the expense +# of longer propagation time for changes. +# Please note that this cache concerns only the DAG parsing step. There is no caching in place when DAG +# tasks are run. +# +# Variable: AIRFLOW__SECRETS__USE_CACHE +# +use_cache = False + +# .. note:: |experimental| +# +# When the cache is enabled, this is the duration for which we consider an entry in the cache to be +# valid. Entries are refreshed if they are older than this many seconds. +# It means that when the cache is enabled, this is the maximum amount of time you need to wait to see a +# Variable change take effect. +# +# Variable: AIRFLOW__SECRETS__CACHE_TTL_SECONDS +# +cache_ttl_seconds = 900 + +[api] +# Boolean for running SwaggerUI in the webserver. +# +# Variable: AIRFLOW__API__ENABLE_SWAGGER_UI +# +enable_swagger_ui = True + +# Secret key used to run your api server. It should be as random as possible. However, when running +# more than 1 instances of the api, make sure all of them use the same ``secret_key`` otherwise +# one of them will error with "CSRF session token is missing". +# The api key is also used to authorize requests to Celery workers when logs are retrieved. +# The token generated using the secret key has a short expiry time though - make sure that time on +# ALL the machines that you run airflow components on is synchronized (for example using ntpd) +# otherwise you might get "forbidden" errors when the logs are accessed. +# +# Variable: AIRFLOW__API__SECRET_KEY +# +secret_key = 010RL08807/JBjH4cWzNaw== + +# Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values +# except those that have security implications. ``True`` shows all values. ``False`` hides the +# configuration completely. +# +# Variable: AIRFLOW__API__EXPOSE_CONFIG +# +expose_config = True + +# Expose stacktrace in the web server +# +# Variable: AIRFLOW__API__EXPOSE_STACKTRACE +# +expose_stacktrace = False + +# The base url of the API server. Airflow cannot guess what domain or CNAME you are using. +# If the Airflow console (the front-end) and the API server are on a different domain, this config +# should contain the API server endpoint. +# +# Example: base_url = https://my-airflow.company.com +# +# Variable: AIRFLOW__API__BASE_URL +# +# base_url = + +# The ip specified when starting the api server +# +# Variable: AIRFLOW__API__HOST +# +host = 0.0.0.0 + +# The port on which to run the api server +# +# Variable: AIRFLOW__API__PORT +# +port = 8080 + +# Number of workers to run on the API server +# +# Variable: AIRFLOW__API__WORKERS +# +workers = 4 + +# Number of seconds the API server waits before timing out on a worker +# +# Variable: AIRFLOW__API__WORKER_TIMEOUT +# +worker_timeout = 120 + +# Log files for the api server. '-' means log to stderr. +# +# Variable: AIRFLOW__API__ACCESS_LOGFILE +# +access_logfile = - + +# Paths to the SSL certificate and key for the api server. When both are +# provided SSL will be enabled. This does not change the api server port. +# The same SSL certificate will also be loaded into the worker to enable +# it to be trusted when a self-signed certificate is used. +# +# Variable: AIRFLOW__API__SSL_CERT +# +ssl_cert = + +# Paths to the SSL certificate and key for the api server. When both are +# provided SSL will be enabled. This does not change the api server port. +# +# Variable: AIRFLOW__API__SSL_KEY +# +ssl_key = + +# Used to set the maximum page limit for API requests. If limit passed as param +# is greater than maximum page limit, it will be ignored and maximum page limit value +# will be set as the limit +# +# Variable: AIRFLOW__API__MAXIMUM_PAGE_LIMIT +# +maximum_page_limit = 100 + +# Used to set the default page limit when limit param is zero or not provided in API +# requests. Otherwise if positive integer is passed in the API requests as limit, the +# smallest number of user given limit or maximum page limit is taken as limit. +# +# Variable: AIRFLOW__API__FALLBACK_PAGE_LIMIT +# +fallback_page_limit = 50 + +# Used in response to a preflight request to indicate which HTTP +# headers can be used when making the actual request. This header is +# the server side response to the browser's +# Access-Control-Request-Headers header. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS +# +access_control_allow_headers = + +# Specifies the method or methods allowed when accessing the resource. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS +# +access_control_allow_methods = + +# Indicates whether the response can be shared with requesting code from the given origins. +# Separate URLs with space. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGINS +# +access_control_allow_origins = + +# Indicates whether the **xcomEntries** endpoint supports the **deserialize** +# flag. If set to ``False``, setting this flag in a request would result in a +# 400 Bad Request error. +# +# Variable: AIRFLOW__API__ENABLE_XCOM_DESERIALIZE_SUPPORT +# +enable_xcom_deserialize_support = False + +[workers] +# Configuration related to workers that run Airflow tasks. + +# Full class name of secrets backend to enable for workers (will precede env vars backend) +# +# Example: secrets_backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend +# +# Variable: AIRFLOW__WORKERS__SECRETS_BACKEND +# +secrets_backend = + +# The secrets_backend_kwargs param is loaded into a dictionary and passed to ``__init__`` +# of secrets backend class. See documentation for the secrets backend you are using. +# JSON is expected. +# +# Example for AWS Systems Manager ParameterStore: +# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` +# +# Variable: AIRFLOW__WORKERS__SECRETS_BACKEND_KWARGS +# +secrets_backend_kwargs = + +# The minimum interval (in seconds) at which the worker checks the task instance's +# heartbeat status with the API server to confirm it is still alive. +# +# Variable: AIRFLOW__WORKERS__MIN_HEARTBEAT_INTERVAL +# +min_heartbeat_interval = 5 + +# The maximum number of consecutive failed heartbeats before terminating the task instance process. +# +# Variable: AIRFLOW__WORKERS__MAX_FAILED_HEARTBEATS +# +max_failed_heartbeats = 3 + +# The maximum number of retry attempts to the execution API server. +# +# Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRIES +# +execution_api_retries = 5 + +# The minimum amount of time (in seconds) to wait before retrying a failed API request. +# +# Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRY_WAIT_MIN +# +execution_api_retry_wait_min = 1.0 + +# The maximum amount of time (in seconds) to wait before retrying a failed API request. +# +# Variable: AIRFLOW__WORKERS__EXECUTION_API_RETRY_WAIT_MAX +# +execution_api_retry_wait_max = 90.0 + +# Number of seconds to wait after a task process exits before forcibly closing any +# remaining communication sockets. This helps prevent the task supervisor from hanging +# indefinitely due to missed EOF signals. +# +# Variable: AIRFLOW__WORKERS__SOCKET_CLEANUP_TIMEOUT +# +socket_cleanup_timeout = 60.0 + +[api_auth] +# Settings relating to authentication on the Airflow APIs + +# The audience claim to use when generating and validating JWTs for the API. +# +# This variable can be a single value, or a comma-separated string, in which case the first value is the +# one that will be used when generating, and the others are accepted at validation time. +# +# Not required, but strongly encouraged. +# +# See also :ref:`config:execution_api__jwt_audience` +# +# Example: jwt_audience = my-unique-airflow-id +# +# Variable: AIRFLOW__API_AUTH__JWT_AUDIENCE +# +# jwt_audience = + +# Issuer claim to use when generating and validating JWTs for the API. +# +# Variable: AIRFLOW__API_AUTH__JWT_ISSUER +# +jwt_issuer = airflow + +# Number in seconds until the JWTs used for authentication expires. When the token expires, +# all API calls using this token will fail on authentication. +# +# Make sure that time on ALL the machines that you run airflow components on is synchronized +# (for example using ntpd) otherwise you might get "forbidden" errors. +# +# See also :ref:`config:execution_api__jwt_expiration_time` +# +# Variable: AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME +# +jwt_expiration_time = 86400 + +# Number in seconds until the JWTs used for authentication expires for CLI commands. +# When the token expires, all CLI calls using this token will fail on authentication. +# +# Make sure that time on ALL the machines that you run airflow components on is synchronized +# (for example using ntpd) otherwise you might get "forbidden" errors. +# +# Variable: AIRFLOW__API_AUTH__JWT_CLI_EXPIRATION_TIME +# +jwt_cli_expiration_time = 3600 + +# Secret key used to encode and decode JWTs to authenticate to public and private APIs. +# +# It should be as random as possible. However, when running more than 1 instances of API services, +# make sure all of them use the same ``jwt_secret`` otherwise calls will fail on authentication. +# +# Mutually exclusive with ``jwt_private_key_path``. +# +# Variable: AIRFLOW__API_AUTH__JWT_SECRET +# +jwt_secret = LUPyvA9pvI084a6b0PKUwA== + +# The path to a file containing a PEM-encoded private key use when generating Task Identity tokens in +# the executor. +# +# Mutually exclusive with ``jwt_secret``. +# +# Example: jwt_private_key_path = /path/to/private_key.pem +# +# Variable: AIRFLOW__API_AUTH__JWT_PRIVATE_KEY_PATH +# +# jwt_private_key_path = + +# The algorithm name use when generating and validating JWT Task Identities. +# +# This value must be appropriate for the given private key type. +# +# If this is not specified Airflow makes some guesses as what algorithm is best based on the key type. +# +# ("HS512" if ``jwt_secret`` is set, otherwise a key-type specific guess) +# +# Example: jwt_algorithm = "EdDSA" or "HS512" +# +# Variable: AIRFLOW__API_AUTH__JWT_ALGORITHM +# +# jwt_algorithm = + +# The Key ID to place in header when generating JWTs. Not used in the validation path. +# +# If this is not specified the RFC7638 thumbprint of the private key will be used. +# +# Ignored when ``jwt_secret`` is used. +# +# Example: jwt_kid = my-key-id +# +# Variable: AIRFLOW__API_AUTH__JWT_KID +# +# jwt_kid = + +# The public signing keys of Task Execution token issuers to trust. It must contain the public key +# related to ``jwt_private_key_path`` else tasks will be unlikely to execute successfully. +# +# Can be a local file path (without the ``file://`` prefix) or an http or https URL. +# +# If a remote URL is given it will be polled periodically for changes. +# +# Mutually exclusive with ``jwt_secret``. +# +# If a ``jwt_private_key_path`` is given but this settings is not set then the private key will be +# trusted. If this is provided it is your responsibility to ensure that the private key used for +# generation is in this list. +# +# Example: trusted_jwks_url = "/path/to/public-jwks.json" or "https://my-issuer/.well-known/jwks.json" +# +# Variable: AIRFLOW__API_AUTH__TRUSTED_JWKS_URL +# +# trusted_jwks_url = + +# Issuer of the JWT. This becomes the ``iss`` claim of generated tokens, and is validated on incoming +# requests. +# +# Ideally this should be unique per individual airflow deployment +# +# Not required, but strongly recommended to be set. +# +# See also :ref:`config:api_auth__jwt_audience` +# +# Example: jwt_issuer = http://my-airflow.mycompany.com +# +# Variable: AIRFLOW__API_AUTH__JWT_ISSUER +# +# jwt_issuer = + +# Number of seconds leeway in validating expiry time of JWTs to account for clock skew between +# client and server +# +# Variable: AIRFLOW__API_AUTH__JWT_LEEWAY +# +jwt_leeway = 10 + +[execution_api] +# Settings related to the Execution API server. +# +# The ExecutionAPI also uses a lot of settings from the :ref:`config:api_auth` section. + +# Number in seconds until the JWT used for authentication expires. When the token expires, +# all API calls using this token will fail on authentication. +# +# Make sure that time on ALL the machines that you run airflow components on is synchronized +# (for example using ntpd) otherwise you might get "forbidden" errors. +# +# Variable: AIRFLOW__EXECUTION_API__JWT_EXPIRATION_TIME +# +jwt_expiration_time = 600 + +# The audience claim to use when generating and validating JWTs for the Execution API. +# +# This variable can be a single value, or a comma-separated string, in which case the first value is the +# one that will be used when generating, and the others are accepted at validation time. +# +# Not required, but strongly encouraged +# +# See also :ref:`config:api_auth__jwt_audience` +# +# Variable: AIRFLOW__EXECUTION_API__JWT_AUDIENCE +# +jwt_audience = urn:airflow.apache.org:task + +[lineage] +# what lineage backend to use +# +# Variable: AIRFLOW__LINEAGE__BACKEND +# +backend = + +[operators] +# The default owner assigned to each new operator, unless +# provided explicitly or passed via ``default_args`` +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_OWNER +# +default_owner = airflow + +# The default value of attribute "deferrable" in operators and sensors. +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_DEFERRABLE +# +default_deferrable = false + +# Indicates the default number of CPU units allocated to each operator when no specific CPU request +# is specified in the operator's configuration +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_CPUS +# +default_cpus = 1 + +# Indicates the default number of RAM allocated to each operator when no specific RAM request +# is specified in the operator's configuration +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_RAM +# +default_ram = 512 + +# Indicates the default number of disk storage allocated to each operator when no specific disk request +# is specified in the operator's configuration +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_DISK +# +default_disk = 512 + +# Indicates the default number of GPUs allocated to each operator when no specific GPUs request +# is specified in the operator's configuration +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_GPUS +# +default_gpus = 0 + +# Default queue that tasks get assigned to and that worker listen on. +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_QUEUE +# +default_queue = default + +[webserver] +# Sorting order in grid view. Valid values are: ``topological``, ``hierarchical_alphabetical`` +# +# Variable: AIRFLOW__WEBSERVER__GRID_VIEW_SORTING_ORDER +# +grid_view_sorting_order = topological + +# The amount of time (in secs) webserver will wait for initial handshake +# while fetching logs from other worker machine +# +# Variable: AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC +# +log_fetch_timeout_sec = 5 + +# By default, the webserver shows paused DAGs. Flip this to hide paused +# DAGs by default +# +# Variable: AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT +# +hide_paused_dags_by_default = False + +# Consistent page size across all listing views in the UI +# +# Variable: AIRFLOW__WEBSERVER__PAGE_SIZE +# +page_size = 50 + +# Default setting for wrap toggle on DAG code and TI log views. +# +# Variable: AIRFLOW__WEBSERVER__DEFAULT_WRAP +# +default_wrap = False + +# Sets a custom page title for the DAGs overview page and site title for all pages +# +# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME +# +# instance_name = + +# Whether the custom page title for the DAGs overview page contains any Markup language +# +# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME_HAS_MARKUP +# +instance_name_has_markup = False + +# How frequently, in seconds, the DAG data will auto-refresh in graph or grid view +# when auto-refresh is turned on +# +# Variable: AIRFLOW__WEBSERVER__AUTO_REFRESH_INTERVAL +# +auto_refresh_interval = 3 + +# Boolean for displaying warning for publicly viewable deployment +# +# Variable: AIRFLOW__WEBSERVER__WARN_DEPLOYMENT_EXPOSURE +# +warn_deployment_exposure = True + +# Comma separated string of view events to exclude from dag audit view. +# All other events will be added minus the ones passed here. +# The audit logs in the db will not be affected by this parameter. +# +# Example: audit_view_excluded_events = cli_task_run,running,success +# +# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_EXCLUDED_EVENTS +# +# audit_view_excluded_events = + +# Comma separated string of view events to include in dag audit view. +# If passed, only these events will populate the dag audit view. +# The audit logs in the db will not be affected by this parameter. +# +# Example: audit_view_included_events = dagrun_cleared,failed +# +# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_INCLUDED_EVENTS +# +# audit_view_included_events = + +# Require confirmation when changing a DAG in the web UI. This is to prevent accidental changes +# to a DAG that may be running on sensitive environments like production. +# When set to ``True``, confirmation dialog will be shown when a user tries to Pause/Unpause, +# Trigger a DAG +# +# Variable: AIRFLOW__WEBSERVER__REQUIRE_CONFIRMATION_DAG_CHANGE +# +require_confirmation_dag_change = False + +[email] +# Configuration email backend and whether to +# send email alerts on retry or failure + +# Email backend to use +# +# Variable: AIRFLOW__EMAIL__EMAIL_BACKEND +# +email_backend = airflow.utils.email.send_email_smtp + +# Email connection to use +# +# Variable: AIRFLOW__EMAIL__EMAIL_CONN_ID +# +email_conn_id = smtp_default + +# Whether email alerts should be sent when a task is retried +# +# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY +# +default_email_on_retry = True + +# Whether email alerts should be sent when a task failed +# +# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE +# +default_email_on_failure = True + +# File that will be used as the template for Email subject (which will be rendered using Jinja2). +# If not set, Airflow uses a base template. +# +# Example: subject_template = /path/to/my_subject_template_file +# +# Variable: AIRFLOW__EMAIL__SUBJECT_TEMPLATE +# +# subject_template = + +# File that will be used as the template for Email content (which will be rendered using Jinja2). +# If not set, Airflow uses a base template. +# +# Example: html_content_template = /path/to/my_html_content_template_file +# +# Variable: AIRFLOW__EMAIL__HTML_CONTENT_TEMPLATE +# +# html_content_template = + +# Email address that will be used as sender address. +# It can either be raw email or the complete address in a format ``Sender Name `` +# +# Example: from_email = Airflow +# +# Variable: AIRFLOW__EMAIL__FROM_EMAIL +# +# from_email = + +# ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default" +# which sets it to ``ssl.create_default_context()`` which provides the right balance between +# compatibility and security, it however requires that certificates in your operating system are +# updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public +# keys installed on your machines. You can switch it to "none" if you want to disable checking +# of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks +# if your infrastructure is not sufficiently secured. It should only be set temporarily while you +# are fixing your certificate configuration. This can be typically done by upgrading to newer +# version of the operating system you run Airflow components on,by upgrading/refreshing proper +# certificates in the OS or by updating certificates for your mail servers. +# +# Example: ssl_context = default +# +# Variable: AIRFLOW__EMAIL__SSL_CONTEXT +# +ssl_context = default + +[smtp] +# If you want airflow to send emails on retries, failure, and you want to use +# the airflow.utils.email.send_email_smtp function, you have to configure an +# smtp server here + +# Specifies the host server address used by Airflow when sending out email notifications via SMTP. +# +# Variable: AIRFLOW__SMTP__SMTP_HOST +# +smtp_host = localhost + +# Determines whether to use the STARTTLS command when connecting to the SMTP server. +# +# Variable: AIRFLOW__SMTP__SMTP_STARTTLS +# +smtp_starttls = True + +# Determines whether to use an SSL connection when talking to the SMTP server. +# +# Variable: AIRFLOW__SMTP__SMTP_SSL +# +smtp_ssl = False + +# Defines the port number on which Airflow connects to the SMTP server to send email notifications. +# +# Variable: AIRFLOW__SMTP__SMTP_PORT +# +smtp_port = 25 + +# Specifies the default **from** email address used when Airflow sends email notifications. +# +# Variable: AIRFLOW__SMTP__SMTP_MAIL_FROM +# +smtp_mail_from = airflow@example.com + +# Determines the maximum time (in seconds) the Apache Airflow system will wait for a +# connection to the SMTP server to be established. +# +# Variable: AIRFLOW__SMTP__SMTP_TIMEOUT +# +smtp_timeout = 30 + +# Defines the maximum number of times Airflow will attempt to connect to the SMTP server. +# +# Variable: AIRFLOW__SMTP__SMTP_RETRY_LIMIT +# +smtp_retry_limit = 5 + +[sentry] +# `Sentry `__ integration. Here you can supply +# additional configuration options based on the Python platform. +# See `Python / Configuration / Basic Options +# `__ for more details. +# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, +# ``ignore_errors``, ``before_breadcrumb``, ``transport``. + +# Enable error reporting to Sentry +# +# Variable: AIRFLOW__SENTRY__SENTRY_ON +# +sentry_on = false + +# +# Variable: AIRFLOW__SENTRY__SENTRY_DSN +# +sentry_dsn = + +# Dotted path to a before_send function that the sentry SDK should be configured to use. +# +# Variable: AIRFLOW__SENTRY__BEFORE_SEND +# +# before_send = + +[scheduler] +# Task instances listen for external kill signal (when you clear tasks +# from the CLI or the UI), this defines the frequency at which they should +# listen (in seconds). +# +# Variable: AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC +# +job_heartbeat_sec = 5 + +# The scheduler constantly tries to trigger new tasks (look at the +# scheduler section in the docs for more information). This defines +# how often the scheduler should run (in seconds). +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC +# +scheduler_heartbeat_sec = 5 + +# The frequency (in seconds) at which the LocalTaskJob should send heartbeat signals to the +# scheduler to notify it's still alive. If this value is set to 0, the heartbeat interval will default +# to the value of ``[scheduler] task_instance_heartbeat_timeout``. +# +# Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_SEC +# +task_instance_heartbeat_sec = 0 + +# The number of times to try to schedule each DAG file +# -1 indicates unlimited number +# +# Variable: AIRFLOW__SCHEDULER__NUM_RUNS +# +num_runs = -1 + +# Controls how long the scheduler will sleep between loops, but if there was nothing to do +# in the loop. i.e. if it scheduled something then it will start the next loop +# iteration straight away. +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME +# +scheduler_idle_sleep_time = 1 + +# How often (in seconds) to check for stale DAGs (DAGs which are no longer present in +# the expected files) which should be deactivated, as well as assets that are no longer +# referenced and should be marked as orphaned. +# +# Variable: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL +# +parsing_cleanup_interval = 60 + +# How often (in seconds) should pool usage stats be sent to StatsD (if statsd_on is enabled) +# +# Variable: AIRFLOW__SCHEDULER__POOL_METRICS_INTERVAL +# +pool_metrics_interval = 5.0 + +# How often (in seconds) should running task instance stats be sent to StatsD (if statsd_on is enabled) +# +# Variable: AIRFLOW__SCHEDULER__RUNNING_METRICS_INTERVAL +# +running_metrics_interval = 30.0 + +# If the last scheduler heartbeat happened more than ``[scheduler] scheduler_health_check_threshold`` +# ago (in seconds), scheduler is considered unhealthy. +# This is used by the health check in the **/health** endpoint and in ``airflow jobs check`` CLI +# for SchedulerJob. +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_THRESHOLD +# +scheduler_health_check_threshold = 30 + +# When you start a scheduler, airflow starts a tiny web server +# subprocess to serve a health check if this is set to ``True`` +# +# Variable: AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK +# +enable_health_check = False + +# When you start a scheduler, airflow starts a tiny web server +# subprocess to serve a health check on this host +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_HOST +# +scheduler_health_check_server_host = 0.0.0.0 + +# When you start a scheduler, airflow starts a tiny web server +# subprocess to serve a health check on this port +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_PORT +# +scheduler_health_check_server_port = 8974 + +# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs +# +# Variable: AIRFLOW__SCHEDULER__ORPHANED_TASKS_CHECK_INTERVAL +# +orphaned_tasks_check_interval = 300.0 + +# Local task jobs periodically heartbeat to the DB. If the job has +# not heartbeat in this many seconds, the scheduler will mark the +# associated task instance as failed and will re-schedule the task. +# +# Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_TIMEOUT +# +task_instance_heartbeat_timeout = 300 + +# How often (in seconds) should the scheduler check for task instances whose heartbeats have timed out. +# +# Variable: AIRFLOW__SCHEDULER__TASK_INSTANCE_HEARTBEAT_TIMEOUT_DETECTION_INTERVAL +# +task_instance_heartbeat_timeout_detection_interval = 10.0 + +# Turn on scheduler catchup by setting this to ``True``. +# Default behavior is unchanged and +# Command Line Backfills still work, but the scheduler +# will not do scheduler catchup if this is ``False``, +# however it can be set on a per DAG basis in the +# DAG definition (catchup) +# +# Variable: AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT +# +catchup_by_default = False + +# Setting this to ``True`` will make first task instance of a task +# ignore depends_on_past setting. A task instance will be considered +# as the first task instance of a task when there is no task instance +# in the DB with a logical_date earlier than it., i.e. no manual marking +# success will be needed for a newly added task to be scheduled. +# +# Variable: AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT +# +ignore_first_depends_on_past_by_default = True + +# This determines the number of task instances to be evaluated for scheduling +# during each scheduler loop. +# Set this to 0 to use the value of ``[core] parallelism`` +# +# Variable: AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY +# +max_tis_per_query = 16 + +# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. +# If this is set to ``False`` then you should not run more than a single +# scheduler at once +# +# Variable: AIRFLOW__SCHEDULER__USE_ROW_LEVEL_LOCKING +# +use_row_level_locking = True + +# Max number of DAGs to create DagRuns for per scheduler loop. +# +# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_TO_CREATE_PER_LOOP +# +max_dagruns_to_create_per_loop = 10 + +# How many DagRuns should a scheduler examine (and lock) when scheduling +# and queuing tasks. +# +# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_PER_LOOP_TO_SCHEDULE +# +max_dagruns_per_loop_to_schedule = 20 + +# Time in seconds after which dags, which were not updated by Dag Processor are deactivated. +# +# Variable: AIRFLOW__SCHEDULER__DAG_STALE_NOT_SEEN_DURATION +# +dag_stale_not_seen_duration = 600 + +# Turn off scheduler use of cron intervals by setting this to ``False``. +# DAGs submitted manually in the web UI or with trigger_dag will still run. +# +# Variable: AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE +# +use_job_schedule = True + +# How often to check for expired trigger requests that have not run yet. +# +# Variable: AIRFLOW__SCHEDULER__TRIGGER_TIMEOUT_CHECK_INTERVAL +# +trigger_timeout_check_interval = 15 + +# Amount of time a task can be in the queued state before being retried or set to failed. +# +# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT +# +task_queued_timeout = 600.0 + +# How often to check for tasks that have been in the queued state for +# longer than ``[scheduler] task_queued_timeout``. +# +# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT_CHECK_INTERVAL +# +task_queued_timeout_check_interval = 120.0 + +# The run_id pattern used to verify the validity of user input to the run_id parameter when +# triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id +# for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. +# +# Variable: AIRFLOW__SCHEDULER__ALLOWED_RUN_ID_PATTERN +# +allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$ + +# Whether to create DAG runs that span an interval or one single point in time for cron schedules, when +# a cron string is provided to ``schedule`` argument of a DAG. +# +# * ``True``: **CronDataIntervalTimetable** is used, which is suitable +# for DAGs with well-defined data interval. You get contiguous intervals from the end of the previous +# interval up to the scheduled datetime. +# * ``False``: **CronTriggerTimetable** is used, which is closer to the behavior of cron itself. +# +# Notably, for **CronTriggerTimetable**, the logical date is the same as the time the DAG Run will +# try to schedule, while for **CronDataIntervalTimetable**, the logical date is the beginning of +# the data interval, but the DAG Run will try to schedule at the end of the data interval. +# +# Variable: AIRFLOW__SCHEDULER__CREATE_CRON_DATA_INTERVALS +# +create_cron_data_intervals = False + +# Whether to create DAG runs that span an interval or one single point in time when a timedelta or +# relativedelta is provided to ``schedule`` argument of a DAG. +# +# * ``True``: **DeltaDataIntervalTimetable** is used, which is suitable for DAGs with well-defined data +# interval. You get contiguous intervals from the end of the previous interval up to the scheduled +# datetime. +# * ``False``: **DeltaTriggerTimetable** is used, which is suitable for DAGs that simply want to say +# e.g. "run this every day" and do not care about the data interval. +# +# Notably, for **DeltaTriggerTimetable**, the logical date is the same as the time the DAG Run will +# try to schedule, while for **DeltaDataIntervalTimetable**, the logical date is the beginning of +# the data interval, but the DAG Run will try to schedule at the end of the data interval. +# +# Variable: AIRFLOW__SCHEDULER__CREATE_DELTA_DATA_INTERVALS +# +create_delta_data_intervals = False + +# Whether to enable memory allocation tracing in the scheduler. If enabled, Airflow will start +# tracing memory allocation and log the top 10 memory usages at the error level upon receiving the +# signal SIGUSR1. +# This is an expensive operation and generally should not be used except for debugging purposes. +# +# Variable: AIRFLOW__SCHEDULER__ENABLE_TRACEMALLOC +# +enable_tracemalloc = False + +[triggerer] +# How many triggers a single Triggerer will run at once, by default. +# +# Variable: AIRFLOW__TRIGGERER__CAPACITY +# +capacity = 1000 + +# How often to heartbeat the Triggerer job to ensure it hasn't been killed. +# +# Variable: AIRFLOW__TRIGGERER__JOB_HEARTBEAT_SEC +# +job_heartbeat_sec = 5 + +# If the last triggerer heartbeat happened more than ``[triggerer] triggerer_health_check_threshold`` +# ago (in seconds), triggerer is considered unhealthy. +# This is used by the health check in the **/health** endpoint and in ``airflow jobs check`` CLI +# for TriggererJob. +# +# Variable: AIRFLOW__TRIGGERER__TRIGGERER_HEALTH_CHECK_THRESHOLD +# +triggerer_health_check_threshold = 30 + +[kerberos] +# Location of your ccache file once kinit has been performed. +# +# Variable: AIRFLOW__KERBEROS__CCACHE +# +ccache = /tmp/airflow_krb5_ccache + +# gets augmented with fqdn +# +# Variable: AIRFLOW__KERBEROS__PRINCIPAL +# +principal = airflow + +# Determines the frequency at which initialization or re-initialization processes occur. +# +# Variable: AIRFLOW__KERBEROS__REINIT_FREQUENCY +# +reinit_frequency = 3600 + +# Path to the kinit executable +# +# Variable: AIRFLOW__KERBEROS__KINIT_PATH +# +kinit_path = kinit + +# Designates the path to the Kerberos keytab file for the Airflow user +# +# Variable: AIRFLOW__KERBEROS__KEYTAB +# +keytab = airflow.keytab + +# Allow to disable ticket forwardability. +# +# Variable: AIRFLOW__KERBEROS__FORWARDABLE +# +forwardable = True + +# Allow to remove source IP from token, useful when using token behind NATted Docker host. +# +# Variable: AIRFLOW__KERBEROS__INCLUDE_IP +# +include_ip = True + +[sensors] +# Sensor default timeout, 7 days by default (7 * 24 * 60 * 60). +# +# Variable: AIRFLOW__SENSORS__DEFAULT_TIMEOUT +# +default_timeout = 604800 + +[dag_processor] +# Configuration for the Airflow DAG processor. This includes, for example: +# - DAG bundles, which allows Airflow to load DAGs from different sources +# - Parsing configuration, like: +# - how often to refresh DAGs from those sources +# - how many files to parse concurrently + +# String path to folder where Airflow bundles can store files locally. Not templated. +# If no path is provided, Airflow will use ``Path(tempfile.gettempdir()) / "airflow"``. +# This path must be absolute. +# +# Example: dag_bundle_storage_path = /tmp/some-place +# +# Variable: AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH +# +# dag_bundle_storage_path = + +# List of backend configs. Must supply name, classpath, and kwargs for each backend. +# +# By default, ``refresh_interval`` is set to ``[dag_processor] refresh_interval``, but that can +# also be overridden in kwargs if desired. +# +# The default is the dags folder dag bundle. +# +# Note: As shown below, you can split your json config over multiple lines by indenting. +# See configparser documentation for an example: +# https://docs.python.org/3/library/configparser.html#supported-ini-file-structure. +# +# Example: dag_bundle_config_list = [ +# { +# "name": "my-git-repo", +# "classpath": "airflow.providers.git.bundles.git.GitDagBundle", +# "kwargs": { +# "subdir": "dags", +# "tracking_ref": "main", +# "refresh_interval": 0 +# } +# } +# ] +# +# Variable: AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST +# +dag_bundle_config_list = [ + { + "name": "dags-folder", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {} + } + ] + + +# How often (in seconds) to refresh, or look for new files, in a DAG bundle. +# +# Variable: AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL +# +refresh_interval = 300 + +# The DAG processor can run multiple processes in parallel to parse dags. +# This defines how many processes will run. +# +# Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PROCESSES +# +parsing_processes = 2 + +# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. +# The DAG processor will list and sort the dag files to decide the parsing order. +# +# * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the +# recently modified DAGs first. +# * ``random_seeded_by_host``: Sort randomly across multiple DAG processors but with same order on the +# same host, allowing each processor to parse the files in a different order. +# * ``alphabetical``: Sort by filename +# +# Variable: AIRFLOW__DAG_PROCESSOR__FILE_PARSING_SORT_MODE +# +file_parsing_sort_mode = modified_time + +# The maximum number of callbacks that are fetched during a single loop. +# +# Variable: AIRFLOW__DAG_PROCESSOR__MAX_CALLBACKS_PER_LOOP +# +max_callbacks_per_loop = 20 + +# Number of seconds after which a DAG file is parsed. The DAG file is parsed every +# ``[dag_processor] min_file_process_interval`` number of seconds. Updates to DAGs are reflected after +# this interval. Keeping this number low will increase CPU usage. +# +# Variable: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL +# +min_file_process_interval = 30 + +# How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale +# DAGs (DAGs which are no longer present in the expected files). The reason why we need +# this threshold is to account for the time between when the file is parsed and when the +# DAG is loaded. The absolute maximum that this could take is +# ``[dag_processor] dag_file_processor_timeout``, but when you have a long timeout configured, +# it results in a significant delay in the deactivation of stale dags. +# +# Variable: AIRFLOW__DAG_PROCESSOR__STALE_DAG_THRESHOLD +# +stale_dag_threshold = 50 + +# How long before timing out a DagFileProcessor, which processes a dag file +# +# Variable: AIRFLOW__DAG_PROCESSOR__DAG_FILE_PROCESSOR_TIMEOUT +# +dag_file_processor_timeout = 50 + +# How often should DAG processor stats be printed to the logs. Setting to 0 will disable printing stats +# +# Variable: AIRFLOW__DAG_PROCESSOR__PRINT_STATS_INTERVAL +# +print_stats_interval = 30 + +# Always run tasks with the latest code. If set to True, the bundle version will not +# be stored on the dag run and therefore, the latest code will always be used. +# +# Variable: AIRFLOW__DAG_PROCESSOR__DISABLE_BUNDLE_VERSIONING +# +disable_bundle_versioning = False + +# How often the DAG processor should check if any DAG bundles are ready for a refresh, either by hitting +# the bundles refresh_interval or because another DAG processor has seen a newer version of the bundle. +# A low value means we check more frequently, and have a smaller window of time where DAG processors are +# out of sync with each other, parsing different versions of the same bundle. +# +# Variable: AIRFLOW__DAG_PROCESSOR__BUNDLE_REFRESH_CHECK_INTERVAL +# +bundle_refresh_check_interval = 5 + +# On shared workers, bundle copies accumulate in local storage as tasks run +# and version of the bundle changes. +# This setting represents the delta in seconds between checks for these stale bundles. +# Bundles which are older than `stale_bundle_cleanup_age_threshold` may be removed. But +# we always keep `stale_bundle_cleanup_min_versions` versions locally. +# Set to 0 or negative to disable. +# +# Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_INTERVAL +# +stale_bundle_cleanup_interval = 1800 + +# Bundle versions used more recently than this threshold will not be removed. +# Recency of use is determined by when the task began running on the worker, +# that age is compared with this setting, given as time delta in seconds. +# +# Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_AGE_THRESHOLD +# +stale_bundle_cleanup_age_threshold = 21600 + +# Minimum number of local bundle versions to retain on disk. +# Local bundle versions older than `stale_bundle_cleanup_age_threshold` will +# only be deleted we have more than `stale_bundle_cleanup_min_versions` versions +# accumulated on the worker. +# +# Variable: AIRFLOW__DAG_PROCESSOR__STALE_BUNDLE_CLEANUP_MIN_VERSIONS +# +stale_bundle_cleanup_min_versions = 10 + +# The dag_processor reads dag files to extract the airflow modules that are going to be used, +# and imports them ahead of time to avoid having to re-do it for each parsing process. +# This flag can be set to ``False`` to disable this behavior in case an airflow module needs +# to be freshly imported each time (at the cost of increased DAG parsing time). +# +# Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PRE_IMPORT_MODULES +# +parsing_pre_import_modules = True + +[common.io] +# Common IO configuration section + +# Path to a location on object storage where XComs can be stored in url format. +# +# Example: xcom_objectstorage_path = s3://conn_id@bucket/path +# +# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_PATH +# +xcom_objectstorage_path = + +# Threshold in bytes for storing XComs in object storage. -1 means always store in the +# database. 0 means always store in object storage. Any positive number means +# it will be stored in object storage if the size of the value is greater than the threshold. +# +# Example: xcom_objectstorage_threshold = 1000000 +# +# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_THRESHOLD +# +xcom_objectstorage_threshold = -1 + +# Compression algorithm to use when storing XComs in object storage. Supported algorithms +# are a.o.: snappy, zip, gzip, bz2, and lzma. If not specified, no compression will be used. +# Note that the compression algorithm must be available in the Python installation (e.g. +# python-snappy for snappy). Zip, gz, bz2 are available by default. +# +# Example: xcom_objectstorage_compression = gz +# +# Variable: AIRFLOW__COMMON.IO__XCOM_OBJECTSTORAGE_COMPRESSION +# +xcom_objectstorage_compression = + +[imap] +# Options for IMAP provider. + +# ssl_context = + +[standard] +# Options for the standard provider operators. + +# Which python tooling should be used to install the virtual environment. +# +# The following options are available: +# - ``auto``: Automatically select, use ``uv`` if available, otherwise use ``pip``. +# - ``pip``: Use pip to install the virtual environment. +# - ``uv``: Use uv to install the virtual environment. Must be available in environment PATH. +# +# Example: venv_install_method = uv +# +# Variable: AIRFLOW__STANDARD__VENV_INSTALL_METHOD +# +venv_install_method = auto + diff --git a/config/airflow/generate_data_dag.py b/config/airflow/generate_data_dag.py new file mode 100644 index 0000000..3f0efea --- /dev/null +++ b/config/airflow/generate_data_dag.py @@ -0,0 +1,114 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from datetime import datetime, timedelta +import os + +# Default arguments for the DAG +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': datetime(2025, 1, 1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +# Define the DAG +dag = DAG( + 'generate_data_all_databases', + default_args=default_args, + description='Generate fake data for all databases', + schedule=None, + catchup=False, + tags=['data', 'generation', 'fake-data'], +) + +# Script paths +SCRIPTS_DIR = '/root/airflow/dags/scripts' +POSTGRES_SCRIPT = os.path.join(SCRIPTS_DIR, 'generate_postgres_sales_data.py') +MONGODB_SCRIPT = os.path.join(SCRIPTS_DIR, 'generate_mongodb_events_data.py') +MYSQL_SCRIPT = os.path.join(SCRIPTS_DIR, 'generate_mysql_employee_data.py') +NEO4J_SCRIPT = os.path.join(SCRIPTS_DIR, 'generate_neo4j_graph_data.py') +CASSANDRA_SCRIPT = os.path.join(SCRIPTS_DIR, 'generate_cassandra_telemetry_data.py') + +# Task functions +def run_postgres_script(): + import subprocess + result = subprocess.run(['python3', POSTGRES_SCRIPT], capture_output=True, text=True) + print(result.stdout) + if result.stderr: + print(result.stderr) + if result.returncode != 0: + raise Exception(f"PostgreSQL script failed with return code {result.returncode}") + +def run_mongodb_script(): + import subprocess + result = subprocess.run(['python3', MONGODB_SCRIPT], capture_output=True, text=True) + print(result.stdout) + if result.stderr: + print(result.stderr) + if result.returncode != 0: + raise Exception(f"MongoDB script failed with return code {result.returncode}") + +def run_mysql_script(): + import subprocess + result = subprocess.run(['python3', MYSQL_SCRIPT], capture_output=True, text=True) + print(result.stdout) + if result.stderr: + print(result.stderr) + if result.returncode != 0: + raise Exception(f"MySQL script failed with return code {result.returncode}") + +def run_neo4j_script(): + import subprocess + result = subprocess.run(['python3', NEO4J_SCRIPT], capture_output=True, text=True) + print(result.stdout) + if result.stderr: + print(result.stderr) + if result.returncode != 0: + raise Exception(f"Neo4j script failed with return code {result.returncode}") + +def run_cassandra_script(): + import subprocess + result = subprocess.run(['python3', CASSANDRA_SCRIPT], capture_output=True, text=True) + print(result.stdout) + if result.stderr: + print(result.stderr) + if result.returncode != 0: + raise Exception(f"Cassandra script failed with return code {result.returncode}") + +# Define tasks +generate_postgres_task = PythonOperator( + task_id='generate_postgres_sales_data', + python_callable=run_postgres_script, + dag=dag, +) + +generate_mongodb_task = PythonOperator( + task_id='generate_mongodb_events_data', + python_callable=run_mongodb_script, + dag=dag, +) + +generate_mysql_task = PythonOperator( + task_id='generate_mysql_employee_data', + python_callable=run_mysql_script, + dag=dag, +) + +generate_neo4j_task = PythonOperator( + task_id='generate_neo4j_graph_data', + python_callable=run_neo4j_script, + dag=dag, +) + +generate_cassandra_task = PythonOperator( + task_id='generate_cassandra_telemetry_data', + python_callable=run_cassandra_script, + dag=dag, +) + +# Set task dependencies - run all in parallel +[generate_postgres_task, generate_mongodb_task, generate_mysql_task, + generate_neo4j_task, generate_cassandra_task] diff --git a/config/homepage/custom.css b/config/homepage/custom.css index 29e8350..70f5e52 100644 --- a/config/homepage/custom.css +++ b/config/homepage/custom.css @@ -1,4 +1,4 @@ -/* Palantir-style Dark Theme with Colorful Accents */ +/* Lakehouse Dashboard - Colorful Palantir-style Theme */ :root { --background-color: #0a0e1a; --card-background: #111827; @@ -11,6 +11,8 @@ --accent-green: #10b981; --accent-orange: #f59e0b; --accent-cyan: #06b6d4; + --accent-red: #ef4444; + --accent-yellow: #fbbf24; } body { @@ -22,6 +24,7 @@ body { background: linear-gradient(135deg, var(--card-background) 0%, #1a2332 100%); border: 1px solid rgba(59, 130, 246, 0.2); transition: all 0.3s ease; + border-radius: 12px; } .service-card:hover { @@ -32,21 +35,61 @@ body { } .group-header { - background: linear-gradient(90deg, var(--accent-blue), var(--accent-purple)); + background: linear-gradient(90deg, var(--accent-blue), var(--accent-purple), var(--accent-pink)); -webkit-background-clip: text; -webkit-text-fill-color: transparent; font-weight: bold; text-transform: uppercase; letter-spacing: 1px; + font-size: 1.2em; } -/* Colorful icons based on service type */ -.icon-kafka { filter: drop-shadow(0 0 8px var(--accent-blue)); } +/* Colorful icon glow effects */ +.icon-kafka { filter: drop-shadow(0 0 8px var(--accent-red)); } .icon-spark { filter: drop-shadow(0 0 8px var(--accent-orange)); } .icon-trino { filter: drop-shadow(0 0 8px var(--accent-cyan)); } .icon-airflow { filter: drop-shadow(0 0 8px var(--accent-green)); } -.icon-elasticsearch { filter: drop-shadow(0 0 8px var(--accent-purple)); } +.icon-superset { filter: drop-shadow(0 0 8px var(--accent-purple)); } +.icon-elasticsearch { filter: drop-shadow(0 0 8px var(--accent-yellow)); } .icon-kibana { filter: drop-shadow(0 0 8px var(--accent-pink)); } .icon-grafana { filter: drop-shadow(0 0 8px var(--accent-orange)); } -.icon-proxmox { filter: drop-shadow(0 0 8px var(--accent-blue)); } -.icon-git { filter: drop-shadow(0 0 8px var(--accent-green)); } +.icon-proxmox { filter: drop-shadow(0 0 8px var(--accent-green)); } +.icon-git { filter: drop-shadow(0 0 8px var(--accent-blue)); } + +/* Animated background gradient */ +@keyframes gradient { + 0% { background-position: 0% 50%; } + 50% { background-position: 100% 50%; } + 100% { background-position: 0% 50%; } +} + +.animated-gradient { + background: linear-gradient(-45deg, #ee7752, #e73c7e, #23a6d5, #23d5ab); + background-size: 400% 400%; + animation: gradient 15s ease infinite; +} + +/* Colorful service cards based on category */ +[data-group*="Data Pipeline"] .service-card { + border-color: rgba(239, 68, 68, 0.3); +} + +[data-group*"Source Systems"] .service-card { + border-color: rgba(168, 85, 247, 0.3); +} + +[data-group*"Storage"] .service-card { + border-color: rgba(59, 130, 246, 0.3); +} + +[data-group*"Orchestration"] .service-card { + border-color: rgba(245, 158, 11, 0.3); +} + +[data-group*="Analytics"] .service-card { + border-color: rgba(139, 92, 246, 0.3); +} + +[data-group*="Management"] .service-card { + border-color: rgba(16, 185, 129, 0.3); +} diff --git a/config/homepage/icons/airflow.png b/config/homepage/icons/airflow.png new file mode 100644 index 0000000..ffb1272 --- /dev/null +++ b/config/homepage/icons/airflow.png @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/config/homepage/icons/kafka.png b/config/homepage/icons/kafka.png new file mode 100644 index 0000000..c515711 --- /dev/null +++ b/config/homepage/icons/kafka.png @@ -0,0 +1,2 @@ + +Apache Kafka icon \ No newline at end of file diff --git a/config/homepage/icons/spark.png b/config/homepage/icons/spark.png new file mode 100644 index 0000000..1becba2 --- /dev/null +++ b/config/homepage/icons/spark.png @@ -0,0 +1 @@ +404: Not Found \ No newline at end of file diff --git a/config/homepage/icons/superset.png b/config/homepage/icons/superset.png new file mode 100644 index 0000000..735b4b4 --- /dev/null +++ b/config/homepage/icons/superset.png @@ -0,0 +1 @@ +Apache Superset \ No newline at end of file diff --git a/config/homepage/icons/trino.png b/config/homepage/icons/trino.png new file mode 100644 index 0000000..1becba2 --- /dev/null +++ b/config/homepage/icons/trino.png @@ -0,0 +1 @@ +404: Not Found \ No newline at end of file diff --git a/config/homepage/services.yaml b/config/homepage/services.yaml index 78f0812..4f6da85 100644 --- a/config/homepage/services.yaml +++ b/config/homepage/services.yaml @@ -1,97 +1,117 @@ --- -# Palantir-style Data Pipeline Dashboard +# Lakehouse Data Pipeline Dashboard # Organized by system type with colorful layout - Data Pipeline: - Kafka UI: - icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kafka.svg + icon: /icons/kafka.png href: http://10.0.21.36:9000/ description: Kafka Topics & Management + color: '#ff6b6b' - Debezium Connect: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/debezium.svg href: http://10.0.21.50:8083/ description: CDC Connectors + color: '#4ecdc4' - Spark Master: - icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-spark.svg + icon: /icons/spark.png href: http://10.0.21.50:8080/ description: Spark Jobs & Monitoring + color: '#ffe66d' - Trino: - icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/trino.svg + icon: /icons/trino.png href: http://10.0.21.50:8089/ description: Distributed SQL Query Engine + color: '#95e1d3' - Source Systems: - MongoDB: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mongodb.svg href: mongodb://10.0.21.51:27017 description: MongoDB Supply Chain Data + color: '#f38181' - PostgreSQL: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/postgresql.svg href: postgresql://10.0.21.51:5432 description: PostgreSQL Sales Data + color: '#aa96da' - Cassandra: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-cassandra.svg href: cassandra://10.0.21.51:9042 description: Cassandra Telemetry Data + color: '#fcbad3' - MySQL: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mysql.svg href: mysql://10.0.21.51:3306 description: MySQL HR Data + color: '#a8d8ea' - Neo4j: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/neo4j.svg href: http://10.0.21.51:7474/ description: Neo4j Graph Database + color: '#ff9a9e' - Storage: - ObjectScale S3: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/minio.svg href: http://10.0.20.111:9020/ description: S3 Object Storage + color: '#a18cd1' - ObjectScale Web: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/object-storage.svg href: https://10.0.20.111/ description: ObjectScale Management UI + color: '#fbc2eb' - Orchestration: - Airflow: - icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-airflow.svg + icon: /icons/airflow.png href: http://10.0.21.55:8080/ description: Workflow Orchestration + color: '#fd9644' - Analytics & Visualization: - Superset: - icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-superset.svg + icon: /icons/superset.png href: http://10.0.21.45:8088/ description: Business Intelligence & Dashboards + color: '#6c5ce7' - Elasticsearch: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/elasticsearch.svg href: http://10.0.21.46:9200/ description: Search & Analytics Engine + color: '#00cec9' - Kibana: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kibana.svg href: http://10.0.21.46:5601/ description: Data Visualization + color: '#e84393' - Grafana: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/grafana.svg href: http://10.0.20.103/ description: Metrics & Monitoring + color: '#fd79a8' - Management: - Proxmox: icon: proxmox.png href: http://10.0.10.65:8006/ description: Virtualization Platform + color: '#00b894' - Git/Forgejo: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/git.svg href: http://atc-mgt01.dell-atc.lan:3001/ description: Git Repository + color: '#636e72' - ATC LDAP: icon: /icons/lam.png href: http://atc-mgt01.dell-atc.lan/lam/ description: LDAP Management + color: '#0984e3' - Development: - Homepage Config: icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/homepage.svg href: http://atc-docker01.dell-atc.lan/ description: Dashboard Configuration + color: '#b2bec3' diff --git a/config/spark/spark-defaults.conf b/config/spark/spark-defaults.conf new file mode 100644 index 0000000..19cba6e --- /dev/null +++ b/config/spark/spark-defaults.conf @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +# Example: +# spark.master spark://master:7077 +# spark.eventLog.enabled true +# spark.eventLog.dir hdfs://namenode:8021/directory +# spark.serializer org.apache.spark.serializer.KryoSerializer +# spark.driver.memory 5g +# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" diff --git a/config/trino/config.properties b/config/trino/config.properties new file mode 100644 index 0000000..a11cba3 --- /dev/null +++ b/config/trino/config.properties @@ -0,0 +1,5 @@ +#single node install config +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080