Data Quality Engine

Profileer, valideer en verbeter je data met enterprise-grade tools

pandas great_expectations dbt scikit-learn

<% if (error) { %>

<%= error %>

<% } %> <% if (!uploaded && !profile) { %> <% } %> <% if (profile) { %> <% var qScore = profile.overall_quality_score || 0; var grade = qScore >= 90 ? 'A' : qScore >= 75 ? 'B' : qScore >= 50 ? 'C' : qScore >= 25 ? 'D' : 'F'; %>

<%= profile.rows.toLocaleString() %>

Rijen

<%= profile.columns %>

Kolommen

<%= profile.memory_mb %> MB

Geheugen

<%= qScore %>%

<%= grade %> Grade

<% profile.columns_profile.slice(0, 8).forEach(function(cp){ var status = cp.null_pct > 20 ? 'danger' : cp.null_pct > 5 ? 'warn' : cp.null_pct === 0 ? 'good' : ''; %>

<%= cp.name %>

Type<%= cp.dtype %>

Compleet<%= (100-cp.null_pct).toFixed(1) %>%

Uniek<%= cp.unique %> (<%= cp.unique_pct.toFixed(0) %>%)

<% if (cp.min !== undefined && cp.min !== null) { %>

Min → Max<%= cp.min %> → <%= cp.max %>

<% } %> <% if (cp.histogram) { %>

<% var maxH=Math.max(...cp.histogram.counts,1); cp.histogram.counts.forEach(function(h){ %>

<% }) %>

<% } %> <% if (cp.top_values) { var tvk=Object.keys(cp.top_values).slice(0,3); %>

Top: <% tvk.forEach(function(k,i){ %><%= k %><%= i<% }) %>

<% } %> <% if (cp.null_count>0) { %>

<%= cp.null_count %> nulls

<% } %>

<% }) %>

<% profile.columns_profile.forEach(function(cp){ var colIssues = []; if (cp.null_pct>20) colIssues.push({s:'high',t:'Hoge null-rate'}); else if (cp.null_pct>5) colIssues.push({s:'medium',t:'Enkele nulls'}); if (cp.negative>0) colIssues.push({s:'medium',t:'Negatieve waarden'}); if (cp.unique_pct>95) colIssues.push({s:'low',t:'Unieke ID kolom'}); if (cp.unique_pct<1) colIssues.push({s:'low',t:'Constante waarde'}); var colStat = cp.null_pct > 20 ? 'danger' : cp.null_pct > 5 ? 'warn' : 'good'; %>

<%= cp.name %>

Dtype<%= cp.dtype %>

Compleet<%= (100-cp.null_pct).toFixed(1) %>% · <%= cp.null_count %> null

Uniek<%= cp.unique %> / <%= cp.non_null %> (<%= cp.unique_pct.toFixed(1) %>%)

<% if (cp.min !== undefined && cp.min !== null) { %>

Range<%= cp.min %> — <%= cp.max %>

<% } %> <% if (cp.mean !== undefined && cp.mean !== null) { %>

μ ± σ<%= cp.mean %> ± <%= cp.std %>

<% } %> <% if (cp.q25 !== undefined) { %>

QuartielenQ1:<%= cp.q25 %> M:<%= cp.q50 %> Q3:<%= cp.q75 %>

<% } %> <% if (cp.histogram) { %>

<% var hMax=Math.max(...cp.histogram.counts,1); cp.histogram.counts.forEach(function(h){ %>

<% }) %>

<% } %> <% if (cp.top_values) { var tvk2=Object.keys(cp.top_values).slice(0,5); %>

Top: <% tvk2.forEach(function(k,i){ %><%= k %><%= i<% }) %>

<% } %> <% if (colIssues.length>0) { %>

<% colIssues.forEach(function(i){ %><%= i.t %><% }) %>

<% } %>

<% }) %>

<% if (profile.suggestions && profile.suggestions.length > 0) { %>

<% profile.suggestions.forEach(function(s){ %>

<%= s.column %>

<%= s.issue %>

→ <%= s.action %>

<% }) %>

<% } else { %>

Geen verbeterpunten — je data ziet er goed uit!

<% } %>

<% if (profile.correlation && profile.correlation.length > 0) { %>

Correlatie matrix

<% profile.correlation.forEach(function(c){ %>

<%= c.from %>↔<%= c.to %><%= c.direction==='positive'?'+':'' %><%= c.value %>

<% }) %>

<% } %> <% if (profile.distribution && Object.keys(profile.distribution).length > 0) { %>

Distributie analyse

<% Object.entries(profile.distribution).forEach(function(e){ var k=e[0],d=e[1]; %>

<%= k %>

Mean<%= d.mean %>

Median<%= d.median %>

Stdev<%= d.std %>

Skew<%= d.skew %>

Outliers (IQR)<%= d.outliers_iqr %>

<% }) %>

<% } %>

dbt Model validator

Valideer SQL modellen op best practices

dbt Test runner

Voer data tests uit (unique, not_null, accepted_values)

<% if (profile && profile.columns_profile) { %>

<% } %>

Nieuw bestand

<% } %>