feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations
This commit is contained in:
mo
2026-06-27 19:37:50 +00:00
parent 5828113f53
commit 46b9c50e73
39 changed files with 5476 additions and 725 deletions
+6 -15
View File
@@ -11,17 +11,14 @@ import { ChatDrawer } from './components/features/ChatDrawer'
import { GpuMonitor } from './components/features/GpuMonitor'
import { InfraQuickAccess } from './components/features/InfraQuickAccess'
import { InspectorPanel } from './components/features/InspectorPanel'
import { PlatformTopology } from './components/features/PlatformTopology'
import { PresentationView } from './components/features/PresentationView'
import { PlatformView } from './components/features/PlatformView'
import { DataQualityView } from './components/features/DataQualityView'
import { KnowledgeChatView } from './components/features/KnowledgeChatView'
import { StorageView } from './components/features/StorageView'
import { HdfsView } from './components/features/HdfsView'
import { DataGenView } from './components/features/DataGenView'
import { ChangesView } from './components/features/ChangesView'
import { DataFlowView } from './components/features/DataFlowView'
import { SearchView } from './components/features/SearchView'
import { DataSourcesView } from './components/features/DataSourcesView'
import { DataHubView } from './components/features/DataHubView'
import { SshTerminal } from './components/features/SshTerminal'
import { TerminalDock } from './components/features/TerminalDock'
import { WorkbenchPanel } from './components/features/WorkbenchPanel'
@@ -39,7 +36,7 @@ export default function App() {
const mainScrollRef = useRef<HTMLDivElement>(null)
const isPlatform = cc.mainView === 'platform'
const isDataSources = cc.mainView === 'datasources'
const isDataSources = cc.mainView === 'datasources' || cc.mainView === 'hdfs'
const openApprovals = () => {
cc.setMainView('approvals')
@@ -123,31 +120,25 @@ export default function App() {
<div className={cn('flex min-h-0 flex-col', (isPlatform || isDataSources) ? 'min-h-0 flex-1 overflow-hidden' : 'min-h-0 flex-1')}>
{cc.mainView === 'platform' ? (
<PlatformTopology
<PlatformView
workload={cc.workload}
animations={cc.anims}
selectedNodeId={cc.selectedNodeId}
onNodeClick={cc.selectNode}
pulse={cc.genPulse}
/>
) : cc.mainView === 'datasources' ? (
<DataSourcesView focusEngine={cc.dataSourceFocus} />
) : cc.mainView === 'datagen' ? (
<DataGenView onPulse={cc.pulseFlow} onOpenPlatform={() => cc.setMainView('platform')} />
) : cc.mainView === 'datasources' || cc.mainView === 'hdfs' ? (
<DataHubView focusEngine={cc.dataSourceFocus} initialTab={cc.mainView === 'hdfs' ? 'hadoop' : 'sources'} onPulse={cc.pulseFlow} />
) : cc.mainView === 'changes' ? (
<ChangesView liveChanges={cc.changes} />
) : cc.mainView === 'dataflow' ? (
<DataFlowView />
) : cc.mainView === 'presentation' ? (
<PresentationView />
) : cc.mainView === 'dataquality' ? (
<DataQualityView />
) : cc.mainView === 'knowledge' ? (
<KnowledgeChatView onGpuActivity={setGpuChatActive} />
) : cc.mainView === 'storage' ? (
<StorageView />
) : cc.mainView === 'hdfs' ? (
<HdfsView />
) : cc.mainView === 'search' ? (
<SearchView />
) : (