SysOps: voice-agy-webbuilder-backup — 2026-06-23 10:04 UTC
This commit is contained in:
sysops
@@ -0,0 +1,212 @@
|
||||
"""Docling API routes for doc-ingest service."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.docling_service import capabilities, convert_nas_batch, convert_nas_file, get_result, list_history
|
||||
|
||||
router = APIRouter(prefix="/docling", tags=["docling"])
|
||||
|
||||
NAS_ROOT = Path(os.getenv("NAS_ROOT", "/nas"))
|
||||
|
||||
|
||||
class DoclingConvertBody(BaseModel):
|
||||
path: str = Field(..., min_length=1, description="Relatief NAS-pad")
|
||||
ocr: bool = True
|
||||
tables: bool = True
|
||||
page_images: bool = False
|
||||
picture_images: bool = True
|
||||
force_full_page_ocr: bool = False
|
||||
formats: list[str] = Field(default_factory=lambda: ["markdown", "html", "json", "text"])
|
||||
|
||||
|
||||
class DoclingBatchBody(BaseModel):
|
||||
paths: list[str] = Field(default_factory=list)
|
||||
limit: int = Field(default=10, ge=1, le=50)
|
||||
ext: Optional[str] = None
|
||||
ocr: bool = True
|
||||
tables: bool = True
|
||||
page_images: bool = False
|
||||
picture_images: bool = True
|
||||
force_full_page_ocr: bool = False
|
||||
formats: list[str] = Field(default_factory=lambda: ["markdown", "json"])
|
||||
|
||||
|
||||
def _options(body: DoclingConvertBody | DoclingBatchBody) -> dict[str, Any]:
|
||||
return {
|
||||
"ocr": body.ocr,
|
||||
"tables": body.tables,
|
||||
"page_images": body.page_images,
|
||||
"picture_images": body.picture_images,
|
||||
"force_full_page_ocr": body.force_full_page_ocr,
|
||||
"formats": body.formats,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/capabilities")
|
||||
def docling_capabilities() -> dict[str, Any]:
|
||||
return {"ok": True, **capabilities()}
|
||||
|
||||
|
||||
@router.post("/convert")
|
||||
def docling_convert(body: DoclingConvertBody) -> dict[str, Any]:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
caps = capabilities()
|
||||
if not caps.get("installed"):
|
||||
raise HTTPException(503, "Docling not installed on doc-ingest service")
|
||||
out = convert_nas_file(NAS_ROOT, body.path.lstrip("/"), _options(body))
|
||||
if not out.get("ok"):
|
||||
raise HTTPException(502, out.get("error") or "Conversion failed")
|
||||
return out
|
||||
|
||||
|
||||
@router.post("/batch")
|
||||
def docling_batch(body: DoclingBatchBody) -> dict[str, Any]:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
caps = capabilities()
|
||||
if not caps.get("installed"):
|
||||
raise HTTPException(503, "Docling not installed on doc-ingest service")
|
||||
paths = list(body.paths or [])
|
||||
if not paths:
|
||||
from app.ingest import scan_share_files
|
||||
|
||||
files = scan_share_files(NAS_ROOT)
|
||||
if body.ext:
|
||||
want = body.ext if body.ext.startswith(".") else f".{body.ext}"
|
||||
files = [f for f in files if f.suffix.lower() == want.lower()]
|
||||
paths = [str(f.relative_to(NAS_ROOT)) for f in files[: body.limit]]
|
||||
if not paths:
|
||||
return {"ok": True, "converted": 0, "total": 0, "results": []}
|
||||
return convert_nas_batch(NAS_ROOT, paths, _options(body))
|
||||
|
||||
|
||||
@router.get("/history")
|
||||
def docling_history(limit: int = Query(default=30, ge=1, le=100)) -> dict[str, Any]:
|
||||
return {"ok": True, "items": list_history(limit)}
|
||||
|
||||
|
||||
@router.get("/result")
|
||||
def docling_result(
|
||||
path: str = Query(..., min_length=1),
|
||||
file_sig: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
row = get_result(path.lstrip("/"), file_sig)
|
||||
if not row:
|
||||
raise HTTPException(404, "No Docling result for this file")
|
||||
return {"ok": True, "result": row}
|
||||
|
||||
|
||||
class NasWriteBody(BaseModel):
|
||||
path: str = Field(..., min_length=1, description="Bronbestand (voor bestandsnaam)")
|
||||
content: str = ""
|
||||
subdir: str = Field(default="Telegram/Exports", max_length=256)
|
||||
|
||||
|
||||
@router.get("/nas/diagnostics")
|
||||
def docling_nas_diagnostics() -> dict[str, Any]:
|
||||
if not NAS_ROOT.is_dir():
|
||||
return {"ok": False, "nas_mounted": False, "error": f"NAS not at {NAS_ROOT}"}
|
||||
from app.ingest import scan_share_files, SKIP_DIR_NAMES
|
||||
|
||||
all_files = scan_share_files(NAS_ROOT)
|
||||
folder_counts: dict[str, int] = {}
|
||||
for p in all_files:
|
||||
top = p.relative_to(NAS_ROOT).parts[0] if p.relative_to(NAS_ROOT).parts else "."
|
||||
folder_counts[top] = folder_counts.get(top, 0) + 1
|
||||
empty_tops = []
|
||||
for child in NAS_ROOT.iterdir():
|
||||
if not child.is_dir() or child.name.startswith(("#", ".")) or child.name in SKIP_DIR_NAMES:
|
||||
continue
|
||||
if child.name not in folder_counts:
|
||||
empty_tops.append(child.name)
|
||||
return {
|
||||
"ok": True,
|
||||
"nas_mounted": True,
|
||||
"total_files": len(all_files),
|
||||
"by_top_folder": folder_counts,
|
||||
"empty_folder_trees": empty_tops,
|
||||
"hint": (
|
||||
"Mappen zonder zichtbare bestanden = Synology ACL probleem. "
|
||||
"Geef SMB-gebruiker aissa Read/Write op share + submappen in DSM."
|
||||
) if empty_tops else "",
|
||||
}
|
||||
|
||||
|
||||
TEXT_SUFFIXES = {
|
||||
".txt", ".md", ".csv", ".json", ".xml", ".html", ".htm", ".yaml", ".yml", ".log", ".rtf",
|
||||
}
|
||||
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff", ".tif"}
|
||||
PREVIEW_SUFFIXES = TEXT_SUFFIXES | IMAGE_SUFFIXES | {".pdf"}
|
||||
|
||||
|
||||
@router.get("/nas/read")
|
||||
def docling_nas_read(path: str = Query(..., min_length=1)) -> dict[str, Any]:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
rel = path.lstrip("/")
|
||||
full = (NAS_ROOT / rel).resolve()
|
||||
if not str(full).startswith(str(NAS_ROOT.resolve())) or not full.is_file():
|
||||
raise HTTPException(404, "File not found")
|
||||
ext = full.suffix.lower()
|
||||
if ext not in TEXT_SUFFIXES and ext not in {".csv", ".rtf"}:
|
||||
raise HTTPException(400, f"Not a plain-text file: {ext}")
|
||||
try:
|
||||
content = full.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
content = full.read_text(encoding="latin-1", errors="replace")
|
||||
return {
|
||||
"ok": True,
|
||||
"path": rel,
|
||||
"filename": full.name,
|
||||
"extension": ext,
|
||||
"content": content,
|
||||
"bytes": full.stat().st_size,
|
||||
"editable": True,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/nas/file")
|
||||
def docling_nas_file(path: str = Query(..., min_length=1)):
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
rel = path.lstrip("/")
|
||||
full = (NAS_ROOT / rel).resolve()
|
||||
if not str(full).startswith(str(NAS_ROOT.resolve())) or not full.is_file():
|
||||
raise HTTPException(404, "File not found")
|
||||
media = "application/octet-stream"
|
||||
ext = full.suffix.lower()
|
||||
if ext == ".pdf":
|
||||
media = "application/pdf"
|
||||
elif ext in IMAGE_SUFFIXES:
|
||||
media = f"image/{ext.lstrip('.')}"
|
||||
if ext == ".jpg":
|
||||
media = "image/jpeg"
|
||||
elif ext == ".html":
|
||||
media = "text/html"
|
||||
return FileResponse(full, media_type=media, filename=full.name)
|
||||
|
||||
|
||||
@router.post("/nas/write")
|
||||
def docling_nas_write(body: NasWriteBody) -> dict[str, Any]:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
subdir = body.subdir.strip().strip("/")
|
||||
out_dir = (NAS_ROOT / subdir).resolve()
|
||||
if not str(out_dir).startswith(str(NAS_ROOT.resolve())):
|
||||
raise HTTPException(400, "Invalid subdir")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = Path(body.path.lstrip("/")).stem
|
||||
fname = f"{stem}-edited.md"
|
||||
full = out_dir / fname
|
||||
full.write_text(body.content or "", encoding="utf-8")
|
||||
rel = str(full.relative_to(NAS_ROOT))
|
||||
return {"ok": True, "path": rel, "filename": fname, "bytes": len(body.content or "")}
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Docling document conversion for NAS files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
|
||||
log = logging.getLogger("doc-ingest.docling")
|
||||
|
||||
_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
DB_HOST = os.getenv("DB_HOST", "10.4.7.18")
|
||||
DB_USER = os.getenv("DB_USER", "aissa")
|
||||
DB_PASSWORD = os.getenv("DB_PASSWORD", "Foodlinkk#2026")
|
||||
DB_NAME = os.getenv("DB_NAME", "foodlinkk")
|
||||
|
||||
SUPPORTED_SUFFIXES = {
|
||||
".pdf", ".docx", ".pptx", ".xlsx", ".html", ".htm", ".md", ".txt",
|
||||
".png", ".jpg", ".jpeg", ".webp", ".tiff", ".tif", ".bmp", ".gif",
|
||||
}
|
||||
|
||||
EXPORT_FORMATS = ("markdown", "html", "json", "doctags", "text", "yaml")
|
||||
|
||||
|
||||
def _db():
|
||||
return psycopg2.connect(host=DB_HOST, user=DB_USER, password=DB_PASSWORD, dbname=DB_NAME)
|
||||
|
||||
|
||||
def file_signature(path: Path) -> str:
|
||||
st = path.stat()
|
||||
return f"{int(st.st_mtime)}_{st.st_size}"
|
||||
|
||||
|
||||
def capabilities() -> dict[str, Any]:
|
||||
try:
|
||||
import docling # noqa: F401
|
||||
installed = True
|
||||
version = getattr(docling, "__version__", "unknown")
|
||||
except Exception:
|
||||
installed = False
|
||||
version = None
|
||||
return {
|
||||
"installed": installed,
|
||||
"version": version,
|
||||
"supported_suffixes": sorted(SUPPORTED_SUFFIXES),
|
||||
"export_formats": list(EXPORT_FORMATS),
|
||||
"pipeline_options": [
|
||||
{"id": "ocr", "label": "OCR", "default": True},
|
||||
{"id": "tables", "label": "Tabelstructuur", "default": True},
|
||||
{"id": "page_images", "label": "Pagina-afbeeldingen", "default": False},
|
||||
{"id": "picture_images", "label": "Figuur-afbeeldingen", "default": True},
|
||||
{"id": "force_full_page_ocr", "label": "Volledige pagina OCR", "default": False},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_converter(options: dict[str, Any]):
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
||||
|
||||
pipeline_options = PdfPipelineOptions()
|
||||
pipeline_options.do_ocr = bool(options.get("ocr", True))
|
||||
pipeline_options.do_table_structure = bool(options.get("tables", True))
|
||||
pipeline_options.generate_page_images = bool(options.get("page_images", False))
|
||||
pipeline_options.generate_picture_images = bool(options.get("picture_images", True))
|
||||
if hasattr(pipeline_options, "force_full_page_ocr"):
|
||||
pipeline_options.force_full_page_ocr = bool(options.get("force_full_page_ocr", False))
|
||||
|
||||
return DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _export_document(doc: Any, fmt: str) -> str | dict | list:
|
||||
if fmt == "markdown":
|
||||
return doc.export_to_markdown()
|
||||
if fmt == "html":
|
||||
return doc.export_to_html()
|
||||
if fmt == "json":
|
||||
if hasattr(doc, "export_to_dict"):
|
||||
return doc.export_to_dict()
|
||||
if hasattr(doc, "model_dump"):
|
||||
return doc.model_dump()
|
||||
return json.loads(doc.model_dump_json()) if hasattr(doc, "model_dump_json") else {}
|
||||
if fmt == "doctags":
|
||||
if hasattr(doc, "export_to_doctags"):
|
||||
return doc.export_to_doctags()
|
||||
if hasattr(doc, "export_to_document_tokens"):
|
||||
return doc.export_to_document_tokens()
|
||||
return ""
|
||||
if fmt == "text":
|
||||
if hasattr(doc, "export_to_text"):
|
||||
return doc.export_to_text()
|
||||
return doc.export_to_markdown()
|
||||
if fmt == "yaml":
|
||||
if hasattr(doc, "export_to_yaml"):
|
||||
return doc.export_to_yaml()
|
||||
return ""
|
||||
raise ValueError(f"Unknown export format: {fmt}")
|
||||
|
||||
|
||||
def _extract_tables(doc: Any) -> list[dict[str, Any]]:
|
||||
tables: list[dict[str, Any]] = []
|
||||
for idx, table in enumerate(getattr(doc, "tables", []) or []):
|
||||
row: dict[str, Any] = {"index": idx}
|
||||
try:
|
||||
if hasattr(table, "export_to_markdown"):
|
||||
row["markdown"] = table.export_to_markdown()
|
||||
if hasattr(table, "export_to_dataframe"):
|
||||
df = table.export_to_dataframe()
|
||||
row["csv"] = df.to_csv(index=False)
|
||||
row["rows"] = df.values.tolist()
|
||||
row["columns"] = list(df.columns)
|
||||
except Exception as exc:
|
||||
row["error"] = str(exc)[:200]
|
||||
tables.append(row)
|
||||
return tables
|
||||
|
||||
|
||||
def _extract_pictures(doc: Any) -> list[dict[str, Any]]:
|
||||
pictures: list[dict[str, Any]] = []
|
||||
for idx, pic in enumerate(getattr(doc, "pictures", []) or []):
|
||||
item: dict[str, Any] = {"index": idx}
|
||||
prov = getattr(pic, "prov", None) or []
|
||||
if prov:
|
||||
item["page"] = getattr(prov[0], "page_no", None)
|
||||
if hasattr(pic, "caption_text"):
|
||||
item["caption"] = pic.caption_text()
|
||||
pictures.append(item)
|
||||
return pictures
|
||||
|
||||
|
||||
def convert_path(path: Path, options: dict[str, Any]) -> dict[str, Any]:
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(str(path))
|
||||
if path.suffix.lower() not in SUPPORTED_SUFFIXES:
|
||||
raise ValueError(f"Unsupported type: {path.suffix}")
|
||||
|
||||
formats = options.get("formats") or ["markdown", "json"]
|
||||
formats = [f for f in formats if f in EXPORT_FORMATS]
|
||||
if not formats:
|
||||
formats = ["markdown"]
|
||||
|
||||
converter = _build_converter(options)
|
||||
result = converter.convert(str(path))
|
||||
doc = result.document
|
||||
|
||||
exports: dict[str, Any] = {}
|
||||
for fmt in formats:
|
||||
try:
|
||||
exports[fmt] = _export_document(doc, fmt)
|
||||
except Exception as exc:
|
||||
exports[fmt] = {"error": str(exc)[:300]}
|
||||
|
||||
meta = {}
|
||||
if hasattr(doc, "name"):
|
||||
meta["name"] = doc.name
|
||||
if hasattr(result, "input") and result.input:
|
||||
meta["input"] = str(result.input.file) if hasattr(result.input, "file") else str(result.input)
|
||||
|
||||
return {
|
||||
"exports": exports,
|
||||
"tables": _extract_tables(doc),
|
||||
"pictures": _extract_pictures(doc),
|
||||
"page_count": len(getattr(doc, "pages", []) or []),
|
||||
"metadata": meta,
|
||||
}
|
||||
|
||||
|
||||
def _upsert_result(
|
||||
storage_path: str,
|
||||
file_sig: str,
|
||||
status: str,
|
||||
options: dict,
|
||||
payload: dict | None = None,
|
||||
error: str | None = None,
|
||||
) -> int:
|
||||
payload = payload or {}
|
||||
conn = _db()
|
||||
conn.autocommit = True
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO docling_results
|
||||
(storage_path, file_sig, status, options, exports, tables_data, pictures,
|
||||
page_count, metadata, error_text, completed_at)
|
||||
VALUES (%s, %s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s::jsonb, %s, %s::jsonb, %s, %s)
|
||||
ON CONFLICT (storage_path, file_sig) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
options = EXCLUDED.options,
|
||||
exports = EXCLUDED.exports,
|
||||
tables_data = EXCLUDED.tables_data,
|
||||
pictures = EXCLUDED.pictures,
|
||||
page_count = EXCLUDED.page_count,
|
||||
metadata = EXCLUDED.metadata,
|
||||
error_text = EXCLUDED.error_text,
|
||||
completed_at = EXCLUDED.completed_at
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
storage_path,
|
||||
file_sig,
|
||||
status,
|
||||
json.dumps(options),
|
||||
json.dumps(payload.get("exports", {})),
|
||||
json.dumps(payload.get("tables", [])),
|
||||
json.dumps(payload.get("pictures", [])),
|
||||
payload.get("page_count", 0),
|
||||
json.dumps(payload.get("metadata", {})),
|
||||
error,
|
||||
datetime.now(timezone.utc) if status in ("done", "failed") else None,
|
||||
),
|
||||
)
|
||||
row_id = cur.fetchone()[0]
|
||||
conn.close()
|
||||
return row_id
|
||||
|
||||
|
||||
def get_result(storage_path: str, file_sig: str | None = None) -> dict | None:
|
||||
conn = _db()
|
||||
with conn.cursor() as cur:
|
||||
if file_sig:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, storage_path, file_sig, status, options, exports, tables_data, pictures,
|
||||
page_count, metadata, error_text, created_at, completed_at
|
||||
FROM docling_results WHERE storage_path = %s AND file_sig = %s
|
||||
""",
|
||||
(storage_path, file_sig),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, storage_path, file_sig, status, options, exports, tables_data, pictures,
|
||||
page_count, metadata, error_text, created_at, completed_at
|
||||
FROM docling_results WHERE storage_path = %s
|
||||
ORDER BY completed_at DESC NULLS LAST, created_at DESC LIMIT 1
|
||||
""",
|
||||
(storage_path,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
if not row:
|
||||
return None
|
||||
cols = [
|
||||
"id", "storage_path", "file_sig", "status", "options", "exports", "tables_data",
|
||||
"pictures", "page_count", "metadata", "error_text", "created_at", "completed_at",
|
||||
]
|
||||
out = dict(zip(cols, row))
|
||||
for key in ("created_at", "completed_at"):
|
||||
if out.get(key):
|
||||
out[key] = out[key].isoformat()
|
||||
return out
|
||||
|
||||
|
||||
def list_history(limit: int = 40) -> list[dict[str, Any]]:
|
||||
limit = max(1, min(limit, 100))
|
||||
conn = _db()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, storage_path, file_sig, status, page_count,
|
||||
jsonb_array_length(COALESCE(tables_data, '[]'::jsonb)) AS table_count,
|
||||
jsonb_array_length(COALESCE(pictures, '[]'::jsonb)) AS picture_count,
|
||||
completed_at, error_text,
|
||||
LEFT(COALESCE(exports->>'markdown', ''), 120) AS md_preview
|
||||
FROM docling_results
|
||||
ORDER BY completed_at DESC NULLS LAST, created_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
cols = [
|
||||
"id", "storage_path", "file_sig", "status", "page_count", "table_count",
|
||||
"picture_count", "completed_at", "error_text", "md_preview",
|
||||
]
|
||||
out = []
|
||||
for row in rows:
|
||||
item = dict(zip(cols, row))
|
||||
if item.get("completed_at"):
|
||||
item["completed_at"] = item["completed_at"].isoformat()
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def convert_nas_file(nas_root: Path, rel_path: str, options: dict[str, Any]) -> dict[str, Any]:
|
||||
path = (nas_root / rel_path).resolve()
|
||||
if not str(path).startswith(str(nas_root.resolve())):
|
||||
raise ValueError("Path outside NAS root")
|
||||
sig = file_signature(path)
|
||||
_upsert_result(rel_path, sig, "running", options)
|
||||
try:
|
||||
payload = convert_path(path, options)
|
||||
job_id = _upsert_result(rel_path, sig, "done", options, payload)
|
||||
return {"ok": True, "id": job_id, "storage_path": rel_path, "file_sig": sig, **payload}
|
||||
except Exception as exc:
|
||||
job_id = _upsert_result(rel_path, sig, "failed", options, error=str(exc)[:500])
|
||||
return {"ok": False, "id": job_id, "storage_path": rel_path, "error": str(exc)[:500]}
|
||||
|
||||
|
||||
def convert_nas_batch(nas_root: Path, rel_paths: list[str], options: dict[str, Any]) -> dict[str, Any]:
|
||||
results = []
|
||||
ok_count = 0
|
||||
for rel in rel_paths:
|
||||
try:
|
||||
out = convert_nas_file(nas_root, rel, options)
|
||||
if out.get("ok"):
|
||||
ok_count += 1
|
||||
results.append(out)
|
||||
except Exception as exc:
|
||||
results.append({"ok": False, "storage_path": rel, "error": str(exc)[:300]})
|
||||
return {"ok": True, "converted": ok_count, "total": len(rel_paths), "results": results}
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Excel parsing routes for Revenue Cockpit bootstrap."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.excel_service import parse_revenue_sheet
|
||||
|
||||
router = APIRouter(prefix="/excel", tags=["excel"])
|
||||
|
||||
NAS_ROOT = Path(os.getenv("NAS_ROOT", "/nas"))
|
||||
|
||||
DEFAULT_FILE = os.getenv("CEO_EXCEL_PATH", "Succes Sheet .xlsx")
|
||||
DEFAULT_SHEET = os.getenv("CEO_EXCEL_SHEET", "Projects next steps revenue")
|
||||
|
||||
|
||||
class RevenueParseBody(BaseModel):
|
||||
path: str = Field(default=DEFAULT_FILE)
|
||||
sheet: Optional[str] = Field(default=DEFAULT_SHEET)
|
||||
|
||||
|
||||
@router.get("/revenue-sheet/preview")
|
||||
def preview_revenue_sheet(path: str = DEFAULT_FILE, sheet: Optional[str] = DEFAULT_SHEET) -> dict:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
result = parse_revenue_sheet(NAS_ROOT, path, sheet)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(400, result.get("error") or "Parse failed")
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/revenue-sheet/parse")
|
||||
def parse_revenue_sheet_api(body: RevenueParseBody) -> dict:
|
||||
if not NAS_ROOT.is_dir():
|
||||
raise HTTPException(503, f"NAS not mounted at {NAS_ROOT}")
|
||||
result = parse_revenue_sheet(NAS_ROOT, body.path, body.sheet)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(400, result.get("error") or "Parse failed")
|
||||
return result
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Parse Succes Sheet revenue tab for Revenue Cockpit bootstrap."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _num(val: Any) -> float | None:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return float(val)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _txt(val: Any) -> str:
|
||||
if val is None:
|
||||
return ""
|
||||
return str(val).strip()
|
||||
|
||||
|
||||
def parse_revenue_sheet(nas_root: Path, rel_path: str, sheet_name: str | None = None) -> dict[str, Any]:
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except ImportError as exc:
|
||||
return {"ok": False, "error": f"openpyxl not installed: {exc}"}
|
||||
|
||||
full = nas_root / rel_path.lstrip("/")
|
||||
if not full.is_file():
|
||||
return {"ok": False, "error": f"File not found: {rel_path}"}
|
||||
|
||||
st = full.stat()
|
||||
wb = load_workbook(full, read_only=True, data_only=True)
|
||||
names = wb.sheetnames
|
||||
target_name = sheet_name
|
||||
if not target_name:
|
||||
for n in names:
|
||||
if "project" in n.lower() and "revenue" in n.lower():
|
||||
target_name = n
|
||||
break
|
||||
if not target_name and len(names) > 2:
|
||||
target_name = names[2]
|
||||
if not target_name:
|
||||
return {"ok": False, "error": "Sheet not found", "sheetnames": names}
|
||||
|
||||
ws = wb[target_name]
|
||||
rows = list(ws.iter_rows(values_only=True))
|
||||
|
||||
goals: dict[str, str] = {"vision_text": "", "horizon_text": "", "tagline": ""}
|
||||
if rows:
|
||||
r0 = rows[0]
|
||||
goals["vision_text"] = _txt(r0[0]) if len(r0) > 0 else ""
|
||||
goals["horizon_text"] = _txt(r0[1]) if len(r0) > 1 else ""
|
||||
goals["tagline"] = _txt(r0[4]) if len(r0) > 4 else (_txt(r0[2]) if len(r0) > 2 else "")
|
||||
|
||||
projects: list[dict[str, Any]] = []
|
||||
sort_order = 0
|
||||
for idx, row in enumerate(rows):
|
||||
if idx <= 1:
|
||||
continue
|
||||
cells = list(row) if row else []
|
||||
name = _txt(cells[1]) if len(cells) > 1 else ""
|
||||
if not name:
|
||||
continue
|
||||
margin_month = _num(cells[2]) if len(cells) > 2 else None
|
||||
margin_year = _num(cells[3]) if len(cells) > 3 else None
|
||||
next_steps = _txt(cells[4]) if len(cells) > 4 else ""
|
||||
target_extra = _num(cells[5]) if len(cells) > 5 else None
|
||||
category = "deal" if margin_month is not None or margin_year is not None else "initiative"
|
||||
if re.search(r"foodlinkk|linknbit|subsid", name, re.I):
|
||||
category = "strategic"
|
||||
projects.append(
|
||||
{
|
||||
"name": name,
|
||||
"category": category,
|
||||
"margin_month": margin_month,
|
||||
"margin_year": margin_year,
|
||||
"target_revenue": target_extra or margin_year,
|
||||
"next_steps": next_steps,
|
||||
"status": "active",
|
||||
"sort_order": sort_order,
|
||||
"source_row": idx,
|
||||
}
|
||||
)
|
||||
sort_order += 1
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"source_file": rel_path,
|
||||
"sheet_name": target_name,
|
||||
"sheetnames": names,
|
||||
"file_mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
|
||||
"file_size": st.st_size,
|
||||
"goals": goals,
|
||||
"projects": projects,
|
||||
"parsed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"project_count": len(projects),
|
||||
}
|
||||
Reference in New Issue
Block a user