ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(openmetadata): profiler + data-quality tests for data insight

Browse Source 
Profiler:
- pg/mysql: table-level metrics via DB statistics (exact-ish row counts) only;
  source tables hold 24-53M rows and OM's column profiler full-scans per column
  (no TABLESAMPLE pushdown), which would hammer the live CDC source.
- trino/iceberg: full column metrics + sample data on the small (20-25k row)
  curated_masked + hadoop tables.

Data quality: 16 test cases across 5 tables (row counts on big sources; row
count + uniqueness/not-null/range on curated/masked + historical), all passing.
This populates OM's Data Quality dashboard (coverage, healthy assets,
dimensions, test results).
This commit is contained in:
mo
2026-06-27 14:55:48 +02:00
parent 8aa2324211
commit 21af36a591
8 changed files with 289 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/openmetadata/ingest/mysql_profile.yaml
+30
View File
@@ -0,0 +1,30 @@
source:
type: mysql
serviceName: atc_mysql
serviceConnection:
config:
type: Mysql
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:3306
databaseSchema: hr
sourceConfig:
config:
type: Profiler
computeTableMetrics: true
computeColumnMetrics: false
useStatistics: true
threadCount: 5
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/pg_profile.yaml
+40
View File
@@ -0,0 +1,40 @@
source:
type: postgres
serviceName: atc_postgres
serviceConnection:
config:
type: Postgres
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:5432
database: postgres
# Hard server-side cap so no profiler query can run away on huge tables.
connectionArguments:
options: "-c statement_timeout=90000"
sourceConfig:
config:
type: Profiler
# Table-level metrics from DB statistics only. The source tables hold tens
# of millions of rows; OM's column profiler full-scans per column (it does
# not push down TABLESAMPLE), which would hammer the live CDC source. Row
# counts come from pg_class stats instantly.
computeTableMetrics: true
computeColumnMetrics: false
useStatistics: true
threadCount: 5
schemaFilterPattern:
includes:
- public
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/trino_profile.yaml
+34
View File
@@ -0,0 +1,34 @@
source:
type: trino
serviceName: atc_trino
serviceConnection:
config:
type: Trino
username: mo
hostPort: 10.0.21.50:8089
sourceConfig:
config:
type: Profiler
# Iceberg lakehouse tables are small (20k-25k rows), so full column-level
# profiling is cheap here and gives rich stats + sample data on the
# governed/masked data. Source catalogs are profiled natively; the
# kafka/cassandra-system catalogs are skipped.
computeTableMetrics: true
computeColumnMetrics: true
useStatistics: true
threadCount: 5
databaseFilterPattern:
includes:
- iceberg
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/ts_emp_masked.yaml
+39
View File
@@ -0,0 +1,39 @@
source:
type: TestSuite
serviceName: atc_data_quality
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.employee_events_masked"
processor:
type: orm-test-runner
config:
testCases:
- name: employee_events_masked_has_rows
testDefinitionName: tableRowCountToBeBetween
parameterValues:
- name: minValue
value: "1"
- name: maxValue
value: "100000000"
- name: employee_events_masked_event_id_unique
columnName: event_id
testDefinitionName: columnValuesToBeUnique
- name: employee_events_masked_event_id_not_null
columnName: event_id
testDefinitionName: columnValuesToBeNotNull
- name: employee_events_masked_email_not_null
columnName: employee_email_masked
testDefinitionName: columnValuesToBeNotNull
- name: employee_events_masked_natid_not_null
columnName: national_id_masked
testDefinitionName: columnValuesToBeNotNull
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/ts_historical.yaml
+46
View File
@@ -0,0 +1,46 @@
source:
type: TestSuite
serviceName: atc_data_quality
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: "atc_trino.iceberg.hadoop.historical_sales"
processor:
type: orm-test-runner
config:
testCases:
- name: historical_sales_has_rows
testDefinitionName: tableRowCountToBeBetween
parameterValues:
- name: minValue
value: "1"
- name: maxValue
value: "100000000"
- name: historical_sales_order_id_not_null
columnName: order_id
testDefinitionName: columnValuesToBeNotNull
- name: historical_sales_amount_range
columnName: amount
testDefinitionName: columnValuesToBeBetween
parameterValues:
- name: minValue
value: "0"
- name: maxValue
value: "100000000"
- name: historical_sales_quantity_range
columnName: quantity
testDefinitionName: columnValuesToBeBetween
parameterValues:
- name: minValue
value: "0"
- name: maxValue
value: "100000"
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/ts_mysql_emp.yaml
+28
View File
@@ -0,0 +1,28 @@
source:
type: TestSuite
serviceName: atc_data_quality
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: "atc_mysql.default.hr.employee_events"
processor:
type: orm-test-runner
config:
testCases:
# Cheap row-count check only: this is the 24M-row live CDC source table.
- name: employee_events_has_rows
testDefinitionName: tableRowCountToBeBetween
parameterValues:
- name: minValue
value: "1"
- name: maxValue
value: "10000000000"
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/ts_pg_sales.yaml
+28
View File
@@ -0,0 +1,28 @@
source:
type: TestSuite
serviceName: atc_data_quality
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: "atc_postgres.postgres.public.sales_orders"
processor:
type: orm-test-runner
config:
testCases:
# Cheap row-count check only: this is the 53M-row live CDC source table.
- name: sales_orders_has_rows
testDefinitionName: tableRowCountToBeBetween
parameterValues:
- name: minValue
value: "1"
- name: maxValue
value: "10000000000"
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/ts_sales_masked.yaml
+44
View File
@@ -0,0 +1,44 @@
source:
type: TestSuite
serviceName: atc_data_quality
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.sales_orders_masked"
processor:
type: orm-test-runner
config:
testCases:
- name: sales_orders_masked_has_rows
testDefinitionName: tableRowCountToBeBetween
parameterValues:
- name: minValue
value: "1"
- name: maxValue
value: "100000000"
- name: sales_orders_masked_order_id_unique
columnName: order_id
testDefinitionName: columnValuesToBeUnique
- name: sales_orders_masked_order_id_not_null
columnName: order_id
testDefinitionName: columnValuesToBeNotNull
- name: sales_orders_masked_email_not_null
columnName: customer_email_masked
testDefinitionName: columnValuesToBeNotNull
- name: sales_orders_masked_amount_range
columnName: amount
testDefinitionName: columnValuesToBeBetween
parameterValues:
- name: minValue
value: "0"
- name: maxValue
value: "100000000"
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"