import { useCallback, useEffect, useState, Fragment } from 'react' import { AlertTriangle, CheckCircle2, FileSearch, FileText, Image, Layers, Loader2, RefreshCw, Table2, Upload, XCircle, Gauge, } from 'lucide-react' import { cn } from '../../lib/utils' import { subTabActive, subTabIdle } from '../../lib/tabActive' import { DqMonitoringPanel } from './DqMonitoringPanel' type Dimension = { id: string label: string description: string score: number level: string findings: string[] recommended_actions?: string[] } type ColumnProfile = { name: string dtype: string null_pct: number unique_count: number quality_flags: string[] sample_values?: string[] numeric?: { min: number; max: number; mean: number; outliers: number } text?: { avg_length: number; empty_strings: number } top_values?: { value: string; count: number }[] } type Violations = { violation_count?: number violation_pct?: number sample_rows?: { row_index: number; values: Record }[] location_hint?: string affected_columns?: string[] } type DqCheck = { id?: string suite: string success?: boolean status?: string outcome?: string expectation?: string name?: string metric?: string check?: string test?: string rule?: string monitor?: string result?: string detail?: string column?: string violations?: Violations meta?: Record } type GxCheck = DqCheck & { expectation: string; success: boolean; result: string } type SodaCheck = DqCheck & { name: string; check: string; outcome: string; detail: string } type DocStructure = { pages: number pictures: number tables: number text_blocks: number headings: number paragraphs: number list_items?: number form_items: number key_value_pairs: number label_counts?: Record table_details?: { index: number; rows: number; cols: number; cells: number; preview?: string }[] picture_details?: { index: number; label: string; has_image: boolean; captions: number }[] outline?: { type: string; text: string; level?: number }[] } type AssessResult = { ok: boolean report_id: string overall_score: number maturity_level: string maturity_description?: string rows: number columns: number dimensions: Dimension[] column_profiles: ColumnProfile[] action_items: { priority: string; dimension: string; score: number; action: string }[] checks?: { great_expectations: GxCheck[] soda_core: SodaCheck[] deequ?: DqCheck[] pandera?: DqCheck[] dbt_expectations?: DqCheck[] monte_carlo?: DqCheck[] affirm?: DqCheck[] } checks_summary: Record executive_summary?: { headline: string dataset: string checks_run: number issues_found: number flagged_columns: number tools: string[] recommendation: string } docling?: { used: boolean; parse_id?: string; document_structure?: DocStructure; stats?: Record; images?: DocImage[] } rag_ingest?: { ok: boolean; duplicate?: boolean; chunks?: number; message?: string; error?: string } report_url: string } type DocImage = { index: number label: string available: boolean url?: string width?: number height?: number mimetype?: string dpi?: number bytes?: number captions?: string[] } type ParseResult = { ok: boolean parse_id: string filename: string status: string processing_time_sec?: number formats_available: string[] document_structure: DocStructure images?: DocImage[] stats: Record content: { preview_markdown?: string; preview_html?: string; markdown?: string; html?: string } table_preview?: string[] errors?: string[] parse_json_url?: string } type Capabilities = { maturity_dimensions: { id: string; label: string; description: string }[] maturity_levels: { min_score: number; label: string; description: string }[] supported_data_formats: string[] supported_document_formats: string[] tools: Record docling_online: boolean } type ReportSummary = { id: string filename: string ts: string overall_score: number maturity_level: string rows: number columns: number } type Tab = 'assess' | 'docling' | 'reports' | 'monitoring' const SCORE_COLOR = (s: number) => (s >= 80 ? 'text-success' : s >= 60 ? 'text-warning' : 'text-danger') const BAR_COLOR = (s: number) => (s >= 80 ? 'bg-success' : s >= 60 ? 'bg-warning' : 'bg-danger') export function DataQualityView() { const [tab, setTab] = useState('monitoring') const [caps, setCaps] = useState(null) const [loading, setLoading] = useState(false) const [assess, setAssess] = useState(null) const [parse, setParse] = useState(null) const [parseFormat, setParseFormat] = useState<'markdown' | 'html'>('markdown') const [reports, setReports] = useState([]) const [error, setError] = useState(null) const [expandedCol, setExpandedCol] = useState(null) const [expandedCheck, setExpandedCheck] = useState(null) const [expandedDim, setExpandedDim] = useState(null) const [activeTool, setActiveTool] = useState('great_expectations') const loadMeta = useCallback(async () => { try { const [c, r] = await Promise.all([fetch('/dq/capabilities'), fetch('/dq/reports')]) if (c.ok) setCaps(await c.json()) if (r.ok) { const j = await r.json() setReports(j.reports || []) } } catch { /* ignore */ } }, []) useEffect(() => { loadMeta() }, [loadMeta]) const onAssess = async (file: File) => { setLoading(true) setError(null) setAssess(null) const fd = new FormData() fd.append('file', file) try { const r = await fetch('/dq/assess', { method: 'POST', body: fd }) const j = await r.json() if (!r.ok || !j.ok) { setError(j.error || j.detail || 'Assessment failed') return } setAssess(j as AssessResult) loadMeta() } catch { setError('Connection failed — check DQ API') } finally { setLoading(false) } } const onParse = async (file: File) => { setLoading(true) setError(null) setParse(null) const fd = new FormData() fd.append('file', file) fd.append('to_formats', 'md,html,json') const ctrl = new AbortController() const timer = setTimeout(() => ctrl.abort(), 300000) try { const r = await fetch('/dq/parse', { method: 'POST', body: fd, signal: ctrl.signal }) const j = await r.json() if (!r.ok || !j.ok) { setError(typeof j.error === 'string' ? j.error : JSON.stringify(j.error || j).slice(0, 200) || 'Docling parse failed') return } setParse(j as ParseResult) loadMeta() } catch (e) { setError(e instanceof Error && e.name === 'AbortError' ? 'Timeout — document too large or Docling overloaded' : 'Docling unavailable') } finally { clearTimeout(timer) setLoading(false) } } const tabs: { id: Tab; label: string; icon: typeof FileSearch }[] = [ { id: 'monitoring', label: 'Live Monitoring', icon: Gauge }, { id: 'assess', label: 'Maturity Assessment', icon: FileSearch }, { id: 'docling', label: 'Docling Parser', icon: FileText }, { id: 'reports', label: 'Reports', icon: CheckCircle2 }, ] return (

Data Quality & Maturity Platform

Full data maturity assessment for customer data — Docling, Great Expectations, Soda Core

Docling {caps?.docling_online ? '● online' : '○ offline'} Docling UI ↗
{caps && (
d.label)} icon={Layers} />
)}
{tabs.map(({ id, label, icon: Icon }) => ( ))}
{error && (
{error}
)} {tab === 'monitoring' && } {tab === 'assess' && (
{loading && } {assess && (
s + (x.total || 0), 0))} sub="across 7 tools" /> 0} />
{assess.docling?.used && assess.docling.document_structure && ( <> {assess.docling.images && assess.docling.images.length > 0 && assess.docling.parse_id && ( )} )} {assess.rag_ingest && (

Knowledge Chat sync

{assess.rag_ingest.ok ? (assess.rag_ingest.duplicate ? `Already in Knowledge Chat — ${assess.rag_ingest.message || 'you can chat immediately.'}` : `Indexed for chat: ${assess.rag_ingest.chunks ?? '?'} text chunks. Open Knowledge Chat to ask questions.`) : (assess.rag_ingest.error || 'Could not sync to Knowledge Chat')}

)} {assess.executive_summary && (

Executive Summary — Customer Report

{assess.executive_summary.headline}

{assess.executive_summary.dataset}

{assess.executive_summary.checks_run} checks run 0 ? 'text-warning' : 'text-success'}>{assess.executive_summary.issues_found} issues found {assess.executive_summary.flagged_columns} flagged columns

Recommendation: {assess.executive_summary.recommendation}

)}

Data Quality Tools — click any row for error details

{([ ['great_expectations', 'Great Expectations'], ['soda_core', 'Soda Core'], ['deequ', 'AWS Deequ'], ['pandera', 'Pandera'], ['dbt_expectations', 'dbt Expectations'], ['monte_carlo', 'Monte Carlo'], ['affirm', 'Affirm'], ] as const).map(([key, label]) => { const items = assess.checks?.[key] || [] if (!items.length) return null const summary = assess.checks_summary?.[key] const badge = summary?.passed !== undefined ? `${summary.passed}/${summary.total}` : `${items.length}` return ( ) })}
{assess.checks && ( )}

6 Maturity Dimensions

{assess.dimensions.map((d) => ( setExpandedDim(expandedDim === d.id ? null : d.id)} /> ))}
{assess.action_items.length > 0 && (

Remediation Roadmap

{assess.action_items.map((a, i) => (
{a.priority} {' · '}{a.dimension} ({a.score}): {a.action}
))}
)}

Column profiles ({assess.column_profiles.length})

)}
)} {tab === 'docling' && (

Docling extracts text, tables, images and document structure from PDF, PowerPoint, Word, Excel and images. Output: Markdown, HTML, JSON with pages, images, tables and document outline.

{loading && } {parse && (
20 ? parse.filename.slice(0, 18) + '…' : parse.filename} sub={parse.status} />
{parse.images && parse.images.filter((i) => i.available).length > 0 && ( )} {parse.document_structure?.outline && parse.document_structure.outline.length > 0 && (

Document outline

    {parse.document_structure.outline.map((o, i) => (
  • {o.type} {o.text}
  • ))}
)}
{(['markdown', 'html'] as const).map((f) => ( ))} {parse.parse_json_url && ( Full JSON ↗ )}
{parse.table_preview && parse.table_preview.length > 0 && (

Tables (markdown preview)

                      {parse.table_preview.join('\n')}
                    
)}

Extracted content

{parseFormat === 'html' && (parse.content.preview_html || parse.content.html) ? (
) : (
                      {parse.content.preview_markdown || parse.content.markdown || '(no content)'}
                    
)}
)}
)} {tab === 'reports' && (
{reports.length === 0 ? (

No reports yet — upload customer data in Maturity Assessment.

) : ( reports.map((r) => (

{r.filename}

{r.ts} · {r.rows?.toLocaleString()} rows · {r.columns} cols

{r.overall_score}

{r.maturity_level}

)) )}
)}
) } function ImageGallery({ images, parseId }: { images: DocImage[]; parseId: string }) { const available = images.filter((i) => i.available) const [lightbox, setLightbox] = useState(null) if (!available.length) { return (

Images detected ({images.length}) — no embedded export

Re-upload the document to extract images (embedded mode).

) } return (

Images extracted by Docling ({available.length})

{available.map((img) => ( ))}
{lightbox !== null && (
setLightbox(null)}>
e.stopPropagation()}> {`Image
)}
) } function DocStructurePanel({ structure, title }: { structure: DocStructure; title: string }) { return (

{title}

{[ { label: 'Pagina\'s', value: structure.pages, icon: Layers }, { label: 'Images', value: structure.pictures, icon: Image }, { label: 'Tables', value: structure.tables, icon: Table2 }, { label: 'Headings', value: structure.headings }, { label: 'Paragraphs', value: structure.paragraphs }, { label: 'Text blocks', value: structure.text_blocks }, ].map(({ label, value, icon: Icon }) => (
{Icon && }

{value}

{label}

))}
{structure.picture_details && structure.picture_details.length > 0 && (

Images ({structure.picture_details.length})

{structure.picture_details.map((p) => ( #{p.index + 1} {p.label} {p.has_image ? '🖼' : ''} {p.captions > 0 ? `(${p.captions} captions)` : ''} ))}
)} {structure.table_details && structure.table_details.length > 0 && (

Tables ({structure.table_details.length})

{structure.table_details.map((t) => (
Table {t.index + 1}: {t.rows}×{t.cols} ({t.cells} cells) — {t.preview || '…'}
))}
)}
) } function normalizeChecks(items: DqCheck[]): DqCheck[] { return items.map((c, i) => ({ ...c, id: c.id || `${c.suite}_${i}` })) } function DimensionCard({ dimension: d, expanded, onToggle }: { dimension: Dimension; expanded?: boolean; onToggle?: () => void }) { return ( ) } function ColumnTable({ profiles, expandedCol, onToggle }: { profiles: ColumnProfile[]; expandedCol: string | null; onToggle: (n: string | null) => void }) { return (
{profiles.map((c) => ( onToggle(expandedCol === c.name ? null : c.name)}> {expandedCol === c.name && ( )} ))}
ColumnTypeNull% UniqueFlags
{c.name} {c.dtype} 10 && 'font-semibold text-warning')}>{c.null_pct}% {c.unique_count.toLocaleString()} {c.quality_flags.join(', ') || '—'}
{c.sample_values?.length ?

Samples: {c.sample_values.join(' · ')}

: null} {c.numeric &&

Range {c.numeric.min} – {c.numeric.max}, μ={c.numeric.mean}, {c.numeric.outliers} outliers

} {c.text &&

Avg len {c.text.avg_length}, {c.text.empty_strings} empty strings

} {c.top_values?.map((tv) => {tv.value} ({tv.count}))}
) } function checkLabel(c: DqCheck): string { const base = c.expectation || c.name || c.metric || c.check || c.test || c.rule || c.monitor || 'check' return c.column ? `${base} [${c.column}]` : base } function checkStatus(c: DqCheck): string { return c.status || (c.success === false ? 'fail' : c.success ? 'pass' : c.outcome || 'unknown') } function checkDetail(c: DqCheck): string { return c.result || c.detail || '' } function CheckTable({ title, checks, expandedId, onToggle }: { title: string; checks: DqCheck[]; expandedId: string | null; onToggle: (id: string | null) => void }) { return (

{title}

{checks.map((c) => { const id = c.id || checkLabel(c) const status = checkStatus(c) const viol = c.violations return ( onToggle(expandedId === id ? null : id)}> {expandedId === id && ( )} ) })}
{status} {checkLabel(c)} {viol?.violation_count ? {viol.violation_count} violations : null} {checkDetail(c)}

Error location & samples

{viol?.violation_count ? ( <>

{viol.violation_count} rows affected ({viol.violation_pct}%){viol.location_hint ? ` · ${viol.location_hint}` : ''}

{viol.sample_rows && viol.sample_rows.length > 0 && (
{Object.keys(viol.sample_rows[0].values).map((k) => )} {viol.sample_rows.map((s) => ( {Object.values(s.values).map((v, j) => )} ))}
Row #{k}
{s.row_index}{v === null ? 'NULL' : String(v)}
)} ) : (

No row-level violations — check passed.

)} {c.meta &&
{JSON.stringify(c.meta, null, 2)}
}
) } function CapCard({ title, items, icon: Icon }: { title: string; items: string[]; icon: typeof Layers }) { return (
{title}

{items.join(' · ')}

) } function UploadZone({ label, hint, accept, loading, onFile }: { label: string; hint: string; accept: string; loading: boolean; onFile: (f: File) => void }) { return ( ) } function LoadingMsg({ text }: { text: string }) { return (
{text}
) } function StatCard({ label, value, sub, accent, warn, icon: Icon }: { label: string; value: string; sub?: string; accent?: boolean; warn?: boolean; icon?: typeof Image }) { return (
{Icon && }

{label}

{value}

{sub &&

{sub}

}
) }