diff --git a/deploy/openmetadata/ingest/es_meta.yaml b/deploy/openmetadata/ingest/es_meta.yaml new file mode 100644 index 0000000..544cb86 --- /dev/null +++ b/deploy/openmetadata/ingest/es_meta.yaml @@ -0,0 +1,26 @@ +source: + type: elasticsearch + serviceName: atc_elasticsearch + serviceConnection: + config: + type: ElasticSearch + hostPort: https://10.0.21.46:9200 + authType: + username: elastic + password: __ES_PASS__ + connectionArguments: + verify_certs: "false" + ssl_show_warn: "false" + sourceConfig: + config: + type: SearchMetadata + includeSampleData: false +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/kafka_meta.yaml b/deploy/openmetadata/ingest/kafka_meta.yaml new file mode 100644 index 0000000..6774d1b --- /dev/null +++ b/deploy/openmetadata/ingest/kafka_meta.yaml @@ -0,0 +1,24 @@ +source: + type: kafka + serviceName: atc_kafka + serviceConnection: + config: + type: Kafka + bootstrapServers: 10.0.21.36:9092 + sourceConfig: + config: + type: MessagingMetadata + generateSampleData: false + topicFilterPattern: + excludes: + - "^__.*" + - "^_confluent.*" +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/s3_meta.yaml b/deploy/openmetadata/ingest/s3_meta.yaml new file mode 100644 index 0000000..3646a07 --- /dev/null +++ b/deploy/openmetadata/ingest/s3_meta.yaml @@ -0,0 +1,23 @@ +source: + type: s3 + serviceName: atc_s3 + serviceConnection: + config: + type: S3 + awsConfig: + awsAccessKeyId: __S3_KEY__ + awsSecretAccessKey: __S3_SECRET__ + awsRegion: us-east-1 + endPointURL: http://10.0.20.111:9020 + sourceConfig: + config: + type: StorageMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/superset_meta.yaml b/deploy/openmetadata/ingest/superset_meta.yaml new file mode 100644 index 0000000..3c6ab1a --- /dev/null +++ b/deploy/openmetadata/ingest/superset_meta.yaml @@ -0,0 +1,23 @@ +source: + type: superset + serviceName: atc_superset + serviceConnection: + config: + type: Superset + hostPort: http://10.0.21.45:8088 + connection: + provider: db + username: mo + password: Dell2026! + sourceConfig: + config: + type: DashboardMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/trino_meta.yaml b/deploy/openmetadata/ingest/trino_meta.yaml index b1a2889..bac6b6b 100644 --- a/deploy/openmetadata/ingest/trino_meta.yaml +++ b/deploy/openmetadata/ingest/trino_meta.yaml @@ -6,10 +6,17 @@ source: type: Trino username: mo hostPort: 10.0.21.50:8089 - catalog: iceberg sourceConfig: config: type: DatabaseMetadata + markDeletedTables: true + databaseFilterPattern: + excludes: + - system + - jmx + - tpch + - tpcds + - memory sink: type: metadata-rest config: {}