diff --git a/deploy/openmetadata/ingest/mysql_profile.yaml b/deploy/openmetadata/ingest/mysql_profile.yaml new file mode 100644 index 0000000..03c9d47 --- /dev/null +++ b/deploy/openmetadata/ingest/mysql_profile.yaml @@ -0,0 +1,30 @@ +source: + type: mysql + serviceName: atc_mysql + serviceConnection: + config: + type: Mysql + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:3306 + databaseSchema: hr + sourceConfig: + config: + type: Profiler + computeTableMetrics: true + computeColumnMetrics: false + useStatistics: true + threadCount: 5 +processor: + type: orm-profiler + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/pg_profile.yaml b/deploy/openmetadata/ingest/pg_profile.yaml new file mode 100644 index 0000000..9f2afc9 --- /dev/null +++ b/deploy/openmetadata/ingest/pg_profile.yaml @@ -0,0 +1,40 @@ +source: + type: postgres + serviceName: atc_postgres + serviceConnection: + config: + type: Postgres + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:5432 + database: postgres + # Hard server-side cap so no profiler query can run away on huge tables. + connectionArguments: + options: "-c statement_timeout=90000" + sourceConfig: + config: + type: Profiler + # Table-level metrics from DB statistics only. The source tables hold tens + # of millions of rows; OM's column profiler full-scans per column (it does + # not push down TABLESAMPLE), which would hammer the live CDC source. Row + # counts come from pg_class stats instantly. + computeTableMetrics: true + computeColumnMetrics: false + useStatistics: true + threadCount: 5 + schemaFilterPattern: + includes: + - public +processor: + type: orm-profiler + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/trino_profile.yaml b/deploy/openmetadata/ingest/trino_profile.yaml new file mode 100644 index 0000000..550283d --- /dev/null +++ b/deploy/openmetadata/ingest/trino_profile.yaml @@ -0,0 +1,34 @@ +source: + type: trino + serviceName: atc_trino + serviceConnection: + config: + type: Trino + username: mo + hostPort: 10.0.21.50:8089 + sourceConfig: + config: + type: Profiler + # Iceberg lakehouse tables are small (20k-25k rows), so full column-level + # profiling is cheap here and gives rich stats + sample data on the + # governed/masked data. Source catalogs are profiled natively; the + # kafka/cassandra-system catalogs are skipped. + computeTableMetrics: true + computeColumnMetrics: true + useStatistics: true + threadCount: 5 + databaseFilterPattern: + includes: + - iceberg +processor: + type: orm-profiler + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/ts_emp_masked.yaml b/deploy/openmetadata/ingest/ts_emp_masked.yaml new file mode 100644 index 0000000..09a8e62 --- /dev/null +++ b/deploy/openmetadata/ingest/ts_emp_masked.yaml @@ -0,0 +1,39 @@ +source: + type: TestSuite + serviceName: atc_data_quality + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.employee_events_masked" +processor: + type: orm-test-runner + config: + testCases: + - name: employee_events_masked_has_rows + testDefinitionName: tableRowCountToBeBetween + parameterValues: + - name: minValue + value: "1" + - name: maxValue + value: "100000000" + - name: employee_events_masked_event_id_unique + columnName: event_id + testDefinitionName: columnValuesToBeUnique + - name: employee_events_masked_event_id_not_null + columnName: event_id + testDefinitionName: columnValuesToBeNotNull + - name: employee_events_masked_email_not_null + columnName: employee_email_masked + testDefinitionName: columnValuesToBeNotNull + - name: employee_events_masked_natid_not_null + columnName: national_id_masked + testDefinitionName: columnValuesToBeNotNull +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/ts_historical.yaml b/deploy/openmetadata/ingest/ts_historical.yaml new file mode 100644 index 0000000..19b0232 --- /dev/null +++ b/deploy/openmetadata/ingest/ts_historical.yaml @@ -0,0 +1,46 @@ +source: + type: TestSuite + serviceName: atc_data_quality + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: "atc_trino.iceberg.hadoop.historical_sales" +processor: + type: orm-test-runner + config: + testCases: + - name: historical_sales_has_rows + testDefinitionName: tableRowCountToBeBetween + parameterValues: + - name: minValue + value: "1" + - name: maxValue + value: "100000000" + - name: historical_sales_order_id_not_null + columnName: order_id + testDefinitionName: columnValuesToBeNotNull + - name: historical_sales_amount_range + columnName: amount + testDefinitionName: columnValuesToBeBetween + parameterValues: + - name: minValue + value: "0" + - name: maxValue + value: "100000000" + - name: historical_sales_quantity_range + columnName: quantity + testDefinitionName: columnValuesToBeBetween + parameterValues: + - name: minValue + value: "0" + - name: maxValue + value: "100000" +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/ts_mysql_emp.yaml b/deploy/openmetadata/ingest/ts_mysql_emp.yaml new file mode 100644 index 0000000..4866fab --- /dev/null +++ b/deploy/openmetadata/ingest/ts_mysql_emp.yaml @@ -0,0 +1,28 @@ +source: + type: TestSuite + serviceName: atc_data_quality + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: "atc_mysql.default.hr.employee_events" +processor: + type: orm-test-runner + config: + testCases: + # Cheap row-count check only: this is the 24M-row live CDC source table. + - name: employee_events_has_rows + testDefinitionName: tableRowCountToBeBetween + parameterValues: + - name: minValue + value: "1" + - name: maxValue + value: "10000000000" +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/ts_pg_sales.yaml b/deploy/openmetadata/ingest/ts_pg_sales.yaml new file mode 100644 index 0000000..6ff0e6f --- /dev/null +++ b/deploy/openmetadata/ingest/ts_pg_sales.yaml @@ -0,0 +1,28 @@ +source: + type: TestSuite + serviceName: atc_data_quality + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: "atc_postgres.postgres.public.sales_orders" +processor: + type: orm-test-runner + config: + testCases: + # Cheap row-count check only: this is the 53M-row live CDC source table. + - name: sales_orders_has_rows + testDefinitionName: tableRowCountToBeBetween + parameterValues: + - name: minValue + value: "1" + - name: maxValue + value: "10000000000" +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/ts_sales_masked.yaml b/deploy/openmetadata/ingest/ts_sales_masked.yaml new file mode 100644 index 0000000..98730b3 --- /dev/null +++ b/deploy/openmetadata/ingest/ts_sales_masked.yaml @@ -0,0 +1,44 @@ +source: + type: TestSuite + serviceName: atc_data_quality + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.sales_orders_masked" +processor: + type: orm-test-runner + config: + testCases: + - name: sales_orders_masked_has_rows + testDefinitionName: tableRowCountToBeBetween + parameterValues: + - name: minValue + value: "1" + - name: maxValue + value: "100000000" + - name: sales_orders_masked_order_id_unique + columnName: order_id + testDefinitionName: columnValuesToBeUnique + - name: sales_orders_masked_order_id_not_null + columnName: order_id + testDefinitionName: columnValuesToBeNotNull + - name: sales_orders_masked_email_not_null + columnName: customer_email_masked + testDefinitionName: columnValuesToBeNotNull + - name: sales_orders_masked_amount_range + columnName: amount + testDefinitionName: columnValuesToBeBetween + parameterValues: + - name: minValue + value: "0" + - name: maxValue + value: "100000000" +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__"