feat(openmetadata): profiler + data-quality tests for data insight

Profiler:
- pg/mysql: table-level metrics via DB statistics (exact-ish row counts) only;
  source tables hold 24-53M rows and OM's column profiler full-scans per column
  (no TABLESAMPLE pushdown), which would hammer the live CDC source.
- trino/iceberg: full column metrics + sample data on the small (20-25k row)
  curated_masked + hadoop tables.

Data quality: 16 test cases across 5 tables (row counts on big sources; row
count + uniqueness/not-null/range on curated/masked + historical), all passing.
This populates OM's Data Quality dashboard (coverage, healthy assets,
dimensions, test results).

deploy/openmetadata/ingest/mysql_profile.yaml

@@ -0,0 +1,30 @@
+source:
+  type: mysql
+  serviceName: atc_mysql
+  serviceConnection:
+    config:
+      type: Mysql
+      username: mo
+      authType:
+        password: Dell2026!
+      hostPort: 10.0.21.51:3306
+      databaseSchema: hr
+  sourceConfig:
+    config:
+      type: Profiler
+      computeTableMetrics: true
+      computeColumnMetrics: false
+      useStatistics: true
+      threadCount: 5
+processor:
+  type: orm-profiler
+  config: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/pg_profile.yaml

@@ -0,0 +1,40 @@
+source:
+  type: postgres
+  serviceName: atc_postgres
+  serviceConnection:
+    config:
+      type: Postgres
+      username: mo
+      authType:
+        password: Dell2026!
+      hostPort: 10.0.21.51:5432
+      database: postgres
+      # Hard server-side cap so no profiler query can run away on huge tables.
+      connectionArguments:
+        options: "-c statement_timeout=90000"
+  sourceConfig:
+    config:
+      type: Profiler
+      # Table-level metrics from DB statistics only. The source tables hold tens
+      # of millions of rows; OM's column profiler full-scans per column (it does
+      # not push down TABLESAMPLE), which would hammer the live CDC source. Row
+      # counts come from pg_class stats instantly.
+      computeTableMetrics: true
+      computeColumnMetrics: false
+      useStatistics: true
+      threadCount: 5
+      schemaFilterPattern:
+        includes:
+          - public
+processor:
+  type: orm-profiler
+  config: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/trino_profile.yaml

@@ -0,0 +1,34 @@
+source:
+  type: trino
+  serviceName: atc_trino
+  serviceConnection:
+    config:
+      type: Trino
+      username: mo
+      hostPort: 10.0.21.50:8089
+  sourceConfig:
+    config:
+      type: Profiler
+      # Iceberg lakehouse tables are small (20k-25k rows), so full column-level
+      # profiling is cheap here and gives rich stats + sample data on the
+      # governed/masked data. Source catalogs are profiled natively; the
+      # kafka/cassandra-system catalogs are skipped.
+      computeTableMetrics: true
+      computeColumnMetrics: true
+      useStatistics: true
+      threadCount: 5
+      databaseFilterPattern:
+        includes:
+          - iceberg
+processor:
+  type: orm-profiler
+  config: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/ts_emp_masked.yaml

@@ -0,0 +1,39 @@
+source:
+  type: TestSuite
+  serviceName: atc_data_quality
+  sourceConfig:
+    config:
+      type: TestSuite
+      entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.employee_events_masked"
+processor:
+  type: orm-test-runner
+  config:
+    testCases:
+      - name: employee_events_masked_has_rows
+        testDefinitionName: tableRowCountToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "1"
+          - name: maxValue
+            value: "100000000"
+      - name: employee_events_masked_event_id_unique
+        columnName: event_id
+        testDefinitionName: columnValuesToBeUnique
+      - name: employee_events_masked_event_id_not_null
+        columnName: event_id
+        testDefinitionName: columnValuesToBeNotNull
+      - name: employee_events_masked_email_not_null
+        columnName: employee_email_masked
+        testDefinitionName: columnValuesToBeNotNull
+      - name: employee_events_masked_natid_not_null
+        columnName: national_id_masked
+        testDefinitionName: columnValuesToBeNotNull
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/ts_historical.yaml

@@ -0,0 +1,46 @@
+source:
+  type: TestSuite
+  serviceName: atc_data_quality
+  sourceConfig:
+    config:
+      type: TestSuite
+      entityFullyQualifiedName: "atc_trino.iceberg.hadoop.historical_sales"
+processor:
+  type: orm-test-runner
+  config:
+    testCases:
+      - name: historical_sales_has_rows
+        testDefinitionName: tableRowCountToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "1"
+          - name: maxValue
+            value: "100000000"
+      - name: historical_sales_order_id_not_null
+        columnName: order_id
+        testDefinitionName: columnValuesToBeNotNull
+      - name: historical_sales_amount_range
+        columnName: amount
+        testDefinitionName: columnValuesToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "0"
+          - name: maxValue
+            value: "100000000"
+      - name: historical_sales_quantity_range
+        columnName: quantity
+        testDefinitionName: columnValuesToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "0"
+          - name: maxValue
+            value: "100000"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/ts_mysql_emp.yaml

@@ -0,0 +1,28 @@
+source:
+  type: TestSuite
+  serviceName: atc_data_quality
+  sourceConfig:
+    config:
+      type: TestSuite
+      entityFullyQualifiedName: "atc_mysql.default.hr.employee_events"
+processor:
+  type: orm-test-runner
+  config:
+    testCases:
+      # Cheap row-count check only: this is the 24M-row live CDC source table.
+      - name: employee_events_has_rows
+        testDefinitionName: tableRowCountToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "1"
+          - name: maxValue
+            value: "10000000000"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/ts_pg_sales.yaml

@@ -0,0 +1,28 @@
+source:
+  type: TestSuite
+  serviceName: atc_data_quality
+  sourceConfig:
+    config:
+      type: TestSuite
+      entityFullyQualifiedName: "atc_postgres.postgres.public.sales_orders"
+processor:
+  type: orm-test-runner
+  config:
+    testCases:
+      # Cheap row-count check only: this is the 53M-row live CDC source table.
+      - name: sales_orders_has_rows
+        testDefinitionName: tableRowCountToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "1"
+          - name: maxValue
+            value: "10000000000"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/ts_sales_masked.yaml

@@ -0,0 +1,44 @@
+source:
+  type: TestSuite
+  serviceName: atc_data_quality
+  sourceConfig:
+    config:
+      type: TestSuite
+      entityFullyQualifiedName: "atc_trino.iceberg.curated_masked.sales_orders_masked"
+processor:
+  type: orm-test-runner
+  config:
+    testCases:
+      - name: sales_orders_masked_has_rows
+        testDefinitionName: tableRowCountToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "1"
+          - name: maxValue
+            value: "100000000"
+      - name: sales_orders_masked_order_id_unique
+        columnName: order_id
+        testDefinitionName: columnValuesToBeUnique
+      - name: sales_orders_masked_order_id_not_null
+        columnName: order_id
+        testDefinitionName: columnValuesToBeNotNull
+      - name: sales_orders_masked_email_not_null
+        columnName: customer_email_masked
+        testDefinitionName: columnValuesToBeNotNull
+      - name: sales_orders_masked_amount_range
+        columnName: amount
+        testDefinitionName: columnValuesToBeBetween
+        parameterValues:
+          - name: minValue
+            value: "0"
+          - name: maxValue
+            value: "100000000"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"