Initial commit — Mek-Tech AI Consultancy Framework v2.0
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --default-timeout=120 --retries 5 -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
ENV LEADGEN_PORT=3003
|
||||
EXPOSE 3003
|
||||
|
||||
CMD ["python", "app.py"]
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Flask API server for Lead Generation & Tech Scoping Engine."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from flask import Flask, request, jsonify
|
||||
|
||||
from models import CompanyTechProfile, LeadGenRequest, LeadGenResponse
|
||||
from engine import LeadScopingEngine
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
logger = logging.getLogger("leadgen")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config file
|
||||
# ---------------------------------------------------------------------------
|
||||
CONFIG_PATH = Path(os.getenv("LEADGEN_CONFIG", "/app/data/leadgen_config.json"))
|
||||
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"api_key": os.getenv("DEEPSEEK_API_KEY", ""),
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"model": "qwen/qwen3-coder:free",
|
||||
"fallback_models": [
|
||||
"meta-llama/llama-3.3-70b-instruct:free",
|
||||
"google/gemma-4-26b-a4b-it:free",
|
||||
"nvidia/nemotron-3-super-120b-a12b:free",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
if CONFIG_PATH.exists():
|
||||
try:
|
||||
return json.loads(CONFIG_PATH.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
cfg = dict(DEFAULT_CONFIG)
|
||||
save_config(cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def save_config(cfg: dict) -> None:
|
||||
CONFIG_PATH.write_text(json.dumps(cfg, indent=2))
|
||||
|
||||
|
||||
config = load_config()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App
|
||||
# ---------------------------------------------------------------------------
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
def get_engine() -> LeadScopingEngine:
|
||||
cfg = load_config()
|
||||
return LeadScopingEngine(
|
||||
api_key=cfg.get("api_key"),
|
||||
api_base=cfg.get("base_url"),
|
||||
)
|
||||
|
||||
|
||||
@app.route("/python-leadgen/", methods=["GET"])
|
||||
def index():
|
||||
return jsonify({
|
||||
"service": "Mek-Tech Lead Generation & Tech Scoping Engine",
|
||||
"version": "2.0",
|
||||
"endpoints": {
|
||||
"GET /python-leadgen/config": "Get current config (model, base_url, key masked)",
|
||||
"POST /python-leadgen/config": "Update config",
|
||||
"GET /python-leadgen/models": "List available free models from configured API",
|
||||
"POST /python-leadgen/analyze": "Analyze a single URL",
|
||||
"POST /python-leadgen/batch": "Analyze multiple URLs",
|
||||
"GET /python-leadgen/health": "Health check",
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@app.route("/python-leadgen/health", methods=["GET"])
|
||||
def health():
|
||||
cfg = load_config()
|
||||
return jsonify({
|
||||
"status": "healthy",
|
||||
"api_configured": bool(cfg.get("api_key")),
|
||||
"model": cfg.get("model"),
|
||||
"base_url": cfg.get("base_url"),
|
||||
})
|
||||
|
||||
|
||||
@app.route("/python-leadgen/config", methods=["GET"])
|
||||
def get_config():
|
||||
cfg = load_config()
|
||||
key = cfg.get("api_key", "")
|
||||
return jsonify({
|
||||
"base_url": cfg.get("base_url"),
|
||||
"model": cfg.get("model"),
|
||||
"fallback_models": cfg.get("fallback_models", []),
|
||||
"api_key_set": bool(key),
|
||||
"api_key_preview": (key[:12] + "..." + key[-4:]) if len(key) > 16 else ("***" if key else ""),
|
||||
})
|
||||
|
||||
|
||||
@app.route("/python-leadgen/config", methods=["POST"])
|
||||
def set_config():
|
||||
try:
|
||||
body = request.get_json(silent=True) or {}
|
||||
except Exception:
|
||||
return jsonify({"success": False, "error": "Invalid JSON"}), 400
|
||||
|
||||
cfg = load_config()
|
||||
if "api_key" in body and body["api_key"]:
|
||||
cfg["api_key"] = body["api_key"]
|
||||
if "base_url" in body:
|
||||
cfg["base_url"] = body["base_url"]
|
||||
if "model" in body:
|
||||
cfg["model"] = body["model"]
|
||||
if "fallback_models" in body:
|
||||
cfg["fallback_models"] = body["fallback_models"]
|
||||
|
||||
save_config(cfg)
|
||||
logger.info("Config updated: model=%s base_url=%s", cfg.get("model"), cfg.get("base_url"))
|
||||
return jsonify({"success": True, "model": cfg["model"]})
|
||||
|
||||
|
||||
@app.route("/python-leadgen/models", methods=["GET"])
|
||||
async def list_models():
|
||||
"""Fetch available free models from the configured API endpoint."""
|
||||
cfg = load_config()
|
||||
base = cfg.get("base_url", "https://openrouter.ai/api/v1")
|
||||
key = cfg.get("api_key", "")
|
||||
|
||||
models_url = base.rstrip("/") + "/models"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.get(
|
||||
models_url,
|
||||
headers={"Authorization": f"Bearer {key}"} if key else {},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": f"API returned {resp.status_code}",
|
||||
"models": _fallback_model_list(),
|
||||
})
|
||||
|
||||
data = resp.json()
|
||||
all_models = data.get("data", [])
|
||||
|
||||
free_models = [
|
||||
{
|
||||
"id": m["id"],
|
||||
"name": m.get("name", m["id"]),
|
||||
"context_length": m.get("context_length", 0),
|
||||
"pricing": str(m.get("pricing", {})),
|
||||
}
|
||||
for m in all_models
|
||||
if ":free" in m.get("id", "")
|
||||
]
|
||||
free_models.sort(key=lambda m: -m["context_length"])
|
||||
|
||||
return jsonify({"success": True, "total": len(free_models), "models": free_models})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"models": _fallback_model_list(),
|
||||
})
|
||||
|
||||
|
||||
def _fallback_model_list():
|
||||
return [
|
||||
{"id": "qwen/qwen3-coder:free", "name": "Qwen 3 Coder (Free)", "context_length": 1048576},
|
||||
{"id": "google/gemma-4-31b-it:free", "name": "Gemma 4 31B (Free)", "context_length": 262144},
|
||||
{"id": "google/gemma-4-26b-a4b-it:free", "name": "Gemma 4 26B (Free)", "context_length": 262144},
|
||||
{"id": "meta-llama/llama-3.3-70b-instruct:free", "name": "Llama 3.3 70B (Free)", "context_length": 131072},
|
||||
{"id": "nvidia/nemotron-3-super-120b-a12b:free", "name": "Nemotron Super 120B (Free)", "context_length": 1000000},
|
||||
]
|
||||
|
||||
|
||||
@app.route("/python-leadgen/analyze", methods=["POST"])
|
||||
async def analyze():
|
||||
try:
|
||||
body = request.get_json(silent=True) or {}
|
||||
except Exception:
|
||||
return jsonify({"success": False, "error": "Invalid JSON body"}), 400
|
||||
|
||||
url = body.get("url", "").strip()
|
||||
if not url:
|
||||
return jsonify({"success": False, "error": "Missing required field: url"}), 400
|
||||
|
||||
engine = get_engine()
|
||||
try:
|
||||
result: LeadGenResponse = await engine.analyze_url(url)
|
||||
except Exception as exc:
|
||||
logger.exception("Unexpected error analyzing %s", url)
|
||||
return jsonify({"success": False, "error": str(exc)}), 500
|
||||
|
||||
if result.success and result.profile:
|
||||
return jsonify(result.model_dump())
|
||||
else:
|
||||
return jsonify(result.model_dump()), 422
|
||||
|
||||
|
||||
@app.route("/python-leadgen/batch", methods=["POST"])
|
||||
async def batch():
|
||||
try:
|
||||
body = request.get_json(silent=True) or {}
|
||||
except Exception:
|
||||
return jsonify({"success": False, "error": "Invalid JSON body"}), 400
|
||||
|
||||
urls = body.get("urls", [])
|
||||
if not urls or not isinstance(urls, list):
|
||||
return jsonify({"success": False, "error": "Missing required field: urls (list)"}), 400
|
||||
|
||||
if len(urls) > 10:
|
||||
return jsonify({"success": False, "error": "Maximum 10 URLs per batch"}), 400
|
||||
|
||||
engine = get_engine()
|
||||
try:
|
||||
results = await engine.batch_analyze(urls)
|
||||
except Exception as exc:
|
||||
logger.exception("Batch analysis failed")
|
||||
return jsonify({"success": False, "error": str(exc)}), 500
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"total": len(urls),
|
||||
"results": [r.model_dump() for r in results],
|
||||
})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
port = int(os.getenv("LEADGEN_PORT", "3003"))
|
||||
debug = os.getenv("LEADGEN_DEBUG", "0") == "1"
|
||||
app.run(host="0.0.0.0", port=port, debug=debug)
|
||||
@@ -0,0 +1,133 @@
|
||||
"""LeadScopingEngine – orchestrates scraping + AI extraction for lead generation."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from scraper import AsyncScraper
|
||||
from extractor import DeepSeekExtractor
|
||||
from models import CompanyTechProfile, ScrapeResult, LeadGenResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONFIG_PATH = Path(os.getenv("LEADGEN_CONFIG", "/app/data/leadgen_config.json"))
|
||||
|
||||
CAREER_PATHS = ["/careers", "/jobs", "/about/careers", "/careers/openings", "/company/careers", "/en/careers"]
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if CONFIG_PATH.exists():
|
||||
try:
|
||||
return json.loads(CONFIG_PATH.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
class LeadScopingEngine:
|
||||
"""Orchestrates async scraping and AI-powered tech signal extraction."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
scrape_timeout_ms: int = 30_000,
|
||||
):
|
||||
self.api_key = api_key
|
||||
self.api_base = api_base
|
||||
self.scraper = AsyncScraper(timeout_ms=scrape_timeout_ms)
|
||||
self._extractor: Optional[DeepSeekExtractor] = None
|
||||
|
||||
@property
|
||||
def extractor(self) -> DeepSeekExtractor:
|
||||
if self._extractor is None:
|
||||
cfg = _load_config()
|
||||
key = self.api_key or cfg.get("api_key", "")
|
||||
base = self.api_base or cfg.get("base_url", "https://openrouter.ai/api/v1")
|
||||
self._extractor = DeepSeekExtractor(api_key=key, base_url=base)
|
||||
return self._extractor
|
||||
|
||||
async def analyze_url(self, url: str) -> LeadGenResponse:
|
||||
"""Analyze a company URL: scrape main page + careers → extract → return structured profile."""
|
||||
t0 = time.monotonic()
|
||||
|
||||
# Step 1: Scrape main page
|
||||
logger.info("Scraping %s ...", url)
|
||||
main_result: ScrapeResult = await self.scraper.scrape(url)
|
||||
|
||||
if main_result.error or not main_result.text_content:
|
||||
logger.warning("Scrape failed for %s: %s", url, main_result.error)
|
||||
return LeadGenResponse(
|
||||
success=False,
|
||||
error=main_result.error or "No content extracted",
|
||||
elapsed_ms=round((time.monotonic() - t0) * 1000, 1),
|
||||
)
|
||||
|
||||
# Step 2: Also scrape careers page for hiring data
|
||||
combined_text = main_result.text_content
|
||||
career_sections = []
|
||||
base = url.rstrip("/")
|
||||
|
||||
for path in CAREER_PATHS:
|
||||
career_url = base + path
|
||||
try:
|
||||
cr = await self.scraper.scrape(career_url)
|
||||
if cr.text_content and len(cr.text_content) > 100:
|
||||
logger.info("Careers page found: %s (%d chars)", career_url, len(cr.text_content))
|
||||
career_sections.append(f"\n\n=== JOB LISTINGS FROM {career_url} ===\n\n" + cr.text_content[:20000])
|
||||
break # Stop after first match
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if career_sections:
|
||||
combined_text += "\n".join(career_sections)
|
||||
|
||||
logger.info(
|
||||
"Scraped %s — main: %d chars, careers: %d chars — %.1fs",
|
||||
url, len(main_result.text_content),
|
||||
sum(len(c) for c in career_sections),
|
||||
time.monotonic() - t0,
|
||||
)
|
||||
|
||||
# Step 3: AI Extraction
|
||||
try:
|
||||
cfg = _load_config()
|
||||
model = cfg.get("model", "qwen/qwen3-coder:free")
|
||||
fallbacks = cfg.get("fallback_models", [])
|
||||
profile: CompanyTechProfile = await self.extractor.extract(
|
||||
combined_text, url, model=model, fallback_models=fallbacks
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("AI extraction failed for %s", url)
|
||||
return LeadGenResponse(
|
||||
success=False,
|
||||
error=f"AI extraction error: {str(exc)}",
|
||||
elapsed_ms=round((time.monotonic() - t0) * 1000, 1),
|
||||
)
|
||||
|
||||
elapsed = round((time.monotonic() - t0) * 1000, 1)
|
||||
logger.info(
|
||||
"Analysis complete — %s | Tech: %d | Pains: %d | Roles: %d | Skills: %d | Maturity: %s | %.0fms",
|
||||
profile.company_name,
|
||||
len(profile.detected_technologies),
|
||||
len(profile.current_pain_points),
|
||||
len(profile.hiring_roles),
|
||||
len(profile.desired_skills),
|
||||
profile.estimated_data_maturity,
|
||||
elapsed,
|
||||
)
|
||||
|
||||
return LeadGenResponse(
|
||||
success=True,
|
||||
profile=profile,
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
async def batch_analyze(self, urls: list[str]) -> list[LeadGenResponse]:
|
||||
"""Analyze multiple URLs concurrently."""
|
||||
import asyncio
|
||||
tasks = [self.analyze_url(url) for url in urls]
|
||||
return await asyncio.gather(*tasks, return_exceptions=False)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""DeepSeek AI extractor for structured tech profiling from scraped text."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from models import CompanyTechProfile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEEPSEEK_BASE_URL = "https://openrouter.ai/api/v1"
|
||||
DEEPSEEK_MODEL = "qwen/qwen3-coder:free"
|
||||
FALLBACK_MODELS = [
|
||||
"meta-llama/llama-3.3-70b-instruct:free",
|
||||
"google/gemma-4-26b-a4b-it:free",
|
||||
"nvidia/nemotron-3-super-120b-a12b:free",
|
||||
]
|
||||
MAX_INPUT_CHARS = 50_000 # Truncate input to stay within context limits
|
||||
|
||||
EXTRACTION_SYSTEM_PROMPT = """You are a Principal AI Architect and technical due-diligence expert.
|
||||
Analyze the company website text and extract structured technical intelligence.
|
||||
|
||||
CRITICAL RULES:
|
||||
- detected_technologies MUST contain at least 3 items. Look for ANY technology mention: programming languages, databases, cloud providers, DevOps tools, frameworks, AI/ML tools, data platforms, monitoring, CI/CD, etc. If the page mentions nothing technical, list technologies commonly associated with their industry.
|
||||
- current_pain_points MUST contain at least 2 items. Infer from their industry, product description, or company stage.
|
||||
- hiring_roles and desired_skills: If there is a careers/jobs section in the text, list actual job titles and required skills. Look for words like "hiring", "we are looking for", "join our team", "job", "position", "engineer", "developer", "scientist", "architect".
|
||||
- outreach_hook MUST be at least 30 characters. Never leave it empty.
|
||||
|
||||
Return ONLY valid JSON matching this exact schema:
|
||||
{
|
||||
"company_name": "string",
|
||||
"detected_technologies": ["at least 3 items, be creative if needed based on industry"],
|
||||
"current_pain_points": ["at least 2 items"],
|
||||
"estimated_data_maturity": "Low | Medium | High",
|
||||
"outreach_hook": "string (minimum 1 sentence, be specific)",
|
||||
"industry": "string or null",
|
||||
"company_size_hint": "Startup | Scale-up | Mid-market | Enterprise or null",
|
||||
"tech_blog_posts": "url or null",
|
||||
"hiring_roles": ["list of job titles from careers section, empty if no jobs section found"],
|
||||
"desired_skills": ["list of skills from job requirements, empty if no jobs section found"],
|
||||
"job_listings_url": "url to careers page or null"
|
||||
}
|
||||
|
||||
No markdown fences, no trailing commas. Every array MUST have content unless truly no information exists."""
|
||||
|
||||
|
||||
class DeepSeekExtractor:
|
||||
"""Extracts structured CompanyTechProfile from text using DeepSeek API."""
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None):
|
||||
self.api_key = api_key or os.getenv("DEEPSEEK_API_KEY", "")
|
||||
self.base_url = base_url or os.getenv("DEEPSEEK_BASE_URL", DEEPSEEK_BASE_URL)
|
||||
self._client: Optional[AsyncOpenAI] = None
|
||||
|
||||
@property
|
||||
def client(self) -> AsyncOpenAI:
|
||||
if self._client is None:
|
||||
if not self.api_key:
|
||||
raise ValueError(
|
||||
"DeepSeek API key not configured. "
|
||||
"Set DEEPSEEK_API_KEY environment variable or pass api_key parameter."
|
||||
)
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=self.api_key,
|
||||
base_url=self.base_url,
|
||||
max_retries=2,
|
||||
timeout=60.0,
|
||||
default_headers={
|
||||
"HTTP-Referer": "http://mek-tech.nl",
|
||||
"X-Title": "Mek-Tech LeadGen Engine",
|
||||
},
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def extract(self, text: str, url: str, model: str = DEEPSEEK_MODEL, fallback_models: list[str] | None = None) -> CompanyTechProfile:
|
||||
"""Extract CompanyTechProfile from scraped page text."""
|
||||
if not text.strip():
|
||||
raise ValueError("Empty text content – cannot extract profile")
|
||||
|
||||
# Truncate text to fit model context
|
||||
truncated = text[:MAX_INPUT_CHARS]
|
||||
if len(text) > MAX_INPUT_CHARS:
|
||||
logger.info("Text truncated from %d to %d chars", len(text), MAX_INPUT_CHARS)
|
||||
|
||||
user_prompt = f"URL: {url}\n\nANALYZE THIS COMPANY WEBSITE TEXT:\n\n{truncated}"
|
||||
|
||||
models_to_try = [model] + (fallback_models or FALLBACK_MODELS)
|
||||
last_error = None
|
||||
|
||||
for m in models_to_try:
|
||||
try:
|
||||
response = await self.client.chat.completions.create(
|
||||
model=m,
|
||||
messages=[
|
||||
{"role": "system", "content": EXTRACTION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=2048,
|
||||
)
|
||||
logger.info("Used model: %s", m)
|
||||
break
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
logger.warning("Model %s failed: %s", m, str(exc)[:200])
|
||||
if m != models_to_try[-1]:
|
||||
import asyncio
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError(f"All models failed. Last error: {str(last_error)}") from last_error
|
||||
|
||||
raw_content = response.choices[0].message.content or "{}"
|
||||
parsed = self._parse_json(raw_content)
|
||||
|
||||
# Enrich with metadata
|
||||
parsed["extracted_at"] = datetime.now(timezone.utc).isoformat()
|
||||
parsed["source_url"] = url
|
||||
|
||||
# Normalize maturity
|
||||
maturity = str(parsed.get("estimated_data_maturity", "")).strip().capitalize()
|
||||
if maturity not in ("Low", "Medium", "High"):
|
||||
maturity = "Medium"
|
||||
parsed["estimated_data_maturity"] = maturity
|
||||
|
||||
# Ensure lists are lists, strings are strings
|
||||
for field in ("detected_technologies", "current_pain_points", "hiring_roles", "desired_skills"):
|
||||
if not isinstance(parsed.get(field), list):
|
||||
parsed[field] = []
|
||||
for field in ("outreach_hook", "industry", "company_size_hint", "tech_blog_posts", "job_listings_url"):
|
||||
if parsed.get(field) is None:
|
||||
parsed[field] = ""
|
||||
if not parsed.get("company_name"):
|
||||
parsed["company_name"] = url.split("//")[-1].split("/")[0]
|
||||
|
||||
try:
|
||||
return CompanyTechProfile(**parsed)
|
||||
except Exception as exc:
|
||||
logger.error("Pydantic validation failed: %s\nRaw data: %s", exc, parsed)
|
||||
raise ValueError(f"Failed to validate extracted profile: {str(exc)}") from exc
|
||||
|
||||
@staticmethod
|
||||
def _parse_json(text: str) -> dict:
|
||||
"""Robust JSON parsing from LLM output."""
|
||||
# Strip markdown code fences if present
|
||||
text = text.strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*", "", text)
|
||||
text = re.sub(r"\s*```$", "", text)
|
||||
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Attempt to extract JSON object from text
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
raise ValueError(f"Unable to parse valid JSON from LLM response: {text[:500]}")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Pydantic v2 data models for AI-driven Lead Generation & Tech Scoping."""
|
||||
from __future__ import annotations
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class CompanyTechProfile(BaseModel):
|
||||
company_name: str = Field(
|
||||
...,
|
||||
description="Official company name as detected from the page content",
|
||||
min_length=1,
|
||||
)
|
||||
detected_technologies: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Detected technologies: Spark, Kafka, AWS, Kubernetes, Snowflake, Airflow, dbt, etc.",
|
||||
)
|
||||
current_pain_points: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Inferred pain points: legacy migration, data quality, scalability, missing AI/ML, etc.",
|
||||
)
|
||||
estimated_data_maturity: Optional[str] = Field(
|
||||
default="Medium",
|
||||
description="Estimated data maturity level: Low, Medium, or High",
|
||||
)
|
||||
outreach_hook: Optional[str] = Field(
|
||||
default="",
|
||||
description="Concrete reason why a Forward Deployed Engineer can help this company right now",
|
||||
)
|
||||
industry: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Detected industry or sector",
|
||||
)
|
||||
company_size_hint: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Estimated company size: Startup, Scale-up, Mid-market, Enterprise",
|
||||
)
|
||||
tech_blog_posts: Optional[str] = Field(
|
||||
default=None,
|
||||
description="URL to engineering/tech blog if detected",
|
||||
)
|
||||
hiring_roles: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Job titles/roles detected from careers page: Data Engineer, ML Engineer, DevOps, etc.",
|
||||
)
|
||||
desired_skills: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Specific skills/technologies they are hiring for: Python, Spark, Terraform, dbt, etc.",
|
||||
)
|
||||
job_listings_url: Optional[str] = Field(
|
||||
default=None,
|
||||
description="URL to careers/jobs page if detected on the site",
|
||||
)
|
||||
extracted_at: Optional[str] = Field(
|
||||
default=None,
|
||||
description="ISO timestamp of extraction",
|
||||
)
|
||||
source_url: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Original URL that was scraped",
|
||||
)
|
||||
|
||||
|
||||
class ScrapeResult(BaseModel):
|
||||
url: str
|
||||
title: str
|
||||
text_content: str
|
||||
text_length: int
|
||||
status_code: Optional[int] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class LeadGenRequest(BaseModel):
|
||||
url: str = Field(..., description="URL of the target company page to analyze")
|
||||
force_refresh: bool = Field(default=False, description="Skip cache and force fresh scrape")
|
||||
|
||||
|
||||
class LeadGenResponse(BaseModel):
|
||||
success: bool
|
||||
profile: Optional[CompanyTechProfile] = None
|
||||
error: Optional[str] = None
|
||||
elapsed_ms: Optional[float] = None
|
||||
@@ -0,0 +1,6 @@
|
||||
flask[async]>=3.0
|
||||
pydantic>=2.0
|
||||
openai>=1.12
|
||||
httpx>=0.27
|
||||
beautifulsoup4>=4.12
|
||||
python-dotenv>=1.0
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Async httpx web scraper with retries, rotating user agents, and fallback."""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from models import ScrapeResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
|
||||
]
|
||||
|
||||
REMOVE_TAGS = {"script", "style", "nav", "footer", "header", "noscript", "iframe", "svg"}
|
||||
|
||||
|
||||
class AsyncScraper:
|
||||
"""Asynchronous web scraper with retries, rotating UAs, and fallback HTTP clients."""
|
||||
|
||||
def __init__(self, timeout_ms: int = 20_000):
|
||||
self.timeout = timeout_ms / 1000.0
|
||||
|
||||
async def scrape(self, url: str) -> ScrapeResult:
|
||||
"""Scrape a URL with retries and fallbacks."""
|
||||
errors = []
|
||||
|
||||
# Strategy 1: Standard httpx client
|
||||
for attempt in range(3):
|
||||
try:
|
||||
result = await self._try_scrape(url, attempt)
|
||||
if result.text_content and len(result.text_content) > 50:
|
||||
return result
|
||||
if result.error:
|
||||
errors.append(f"Attempt {attempt+1}: {result.error}")
|
||||
except Exception as e:
|
||||
errors.append(f"Attempt {attempt+1}: {str(e)}")
|
||||
|
||||
# Strategy 2: Minimal headers (like curl)
|
||||
for attempt in range(2):
|
||||
try:
|
||||
result = await self._try_minimal(url)
|
||||
if result.text_content and len(result.text_content) > 50:
|
||||
return result
|
||||
if result.error:
|
||||
errors.append(f"Minimal {attempt+1}: {result.error}")
|
||||
except Exception as e:
|
||||
errors.append(f"Minimal {attempt+1}: {str(e)}")
|
||||
|
||||
return ScrapeResult(url=url, title="", text_content="", text_length=0,
|
||||
error="; ".join(errors[-3:]) or "All scrape attempts failed")
|
||||
|
||||
async def _try_scrape(self, url: str, attempt: int) -> ScrapeResult:
|
||||
ua = USER_AGENTS[attempt % len(USER_AGENTS)]
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,nl;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Cache-Control": "max-age=0",
|
||||
}
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self.timeout,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
http2=True,
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
return self._parse_response(response, url)
|
||||
|
||||
async def _try_minimal(self, url: str) -> ScrapeResult:
|
||||
headers = {
|
||||
"User-Agent": "curl/8.4.0",
|
||||
"Accept": "*/*",
|
||||
}
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self.timeout,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
http2=False,
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
return self._parse_response(response, url)
|
||||
|
||||
def _parse_response(self, response, url: str) -> ScrapeResult:
|
||||
result = ScrapeResult(url=url, title="", text_content="", text_length=0)
|
||||
result.status_code = response.status_code
|
||||
|
||||
if response.status_code >= 500:
|
||||
result.error = f"HTTP {response.status_code}"
|
||||
return result
|
||||
if response.status_code >= 400 and response.status_code < 500:
|
||||
if response.status_code in (403, 429):
|
||||
result.error = f"HTTP {response.status_code} (blocked/rate-limited)"
|
||||
else:
|
||||
result.error = f"HTTP {response.status_code}"
|
||||
return result
|
||||
|
||||
try:
|
||||
html = response.text
|
||||
except Exception:
|
||||
result.error = "Could not decode response"
|
||||
return result
|
||||
|
||||
if not html or len(html) < 100:
|
||||
result.error = "Empty response"
|
||||
return result
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
result.title = soup.title.string.strip() if soup.title else ""
|
||||
|
||||
# Remove non-content elements
|
||||
for tag in soup(REMOVE_TAGS):
|
||||
tag.decompose()
|
||||
|
||||
body = soup.find("body")
|
||||
text = body.get_text(separator="\n") if body else soup.get_text(separator="\n")
|
||||
|
||||
# Also extract text from JSON-LD structured data
|
||||
for script in soup.find_all("script", type="application/ld+json"):
|
||||
try:
|
||||
import json
|
||||
ld = json.loads(script.string or "")
|
||||
if isinstance(ld, dict):
|
||||
desc = ld.get("description", "")
|
||||
if desc:
|
||||
text += "\n" + desc
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
result.text_content = self._clean_text(text)
|
||||
result.text_length = len(result.text_content)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _clean_text(text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
text = re.sub(r"[ \t]+", " ", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
lines = [ln.strip() for ln in text.split("\n")]
|
||||
lines = [ln for ln in lines if len(ln) > 2]
|
||||
return "\n".join(lines).strip()
|
||||
Reference in New Issue
Block a user