dashboard-nanobot/backend/services/skill_market_service.py

import json
import os
import tempfile
import zipfile
from datetime import datetime
from typing import Any, Dict, List, Optional

from fastapi import HTTPException, UploadFile
from sqlmodel import Session, select

from core.settings import DATA_ROOT
from core.utils import (
    _is_ignored_skill_zip_top_level,
    _is_valid_top_level_skill_name,
    _parse_json_string_list,
    _read_description_from_text,
    _sanitize_skill_market_key,
    _sanitize_zip_filename,
)
from models.skill import BotSkillInstall, SkillMarketItem
from services.platform_settings_service import get_platform_settings_snapshot
from services.skill_service import get_bot_skills_root, install_skill_zip_into_workspace


def _skill_market_root() -> str:
    return os.path.abspath(os.path.join(DATA_ROOT, "skills"))


def _extract_skill_zip_summary(zip_path: str) -> Dict[str, Any]:
    entry_names: List[str] = []
    description = ""
    with zipfile.ZipFile(zip_path) as archive:
        members = archive.infolist()
        file_members = [member for member in members if not member.is_dir()]
        for member in file_members:
            raw_name = str(member.filename or "").replace("\\", "/").lstrip("/")
            if not raw_name:
                continue
            first = raw_name.split("/", 1)[0].strip()
            if _is_ignored_skill_zip_top_level(first):
                continue
            if _is_valid_top_level_skill_name(first) and first not in entry_names:
                entry_names.append(first)

        candidates = sorted(
            [
                str(member.filename or "").replace("\\", "/").lstrip("/")
                for member in file_members
                if str(member.filename or "").replace("\\", "/").rsplit("/", 1)[-1].lower()
                in {"skill.md", "readme.md"}
            ],
            key=lambda value: (value.count("/"), value.lower()),
        )
        for candidate in candidates:
            try:
                with archive.open(candidate, "r") as file:
                    preview = file.read(4096).decode("utf-8", errors="ignore")
                description = _read_description_from_text(preview)
                if description:
                    break
            except Exception:
                continue
    return {
        "entry_names": entry_names,
        "description": description,
    }


def _resolve_unique_skill_market_key(session: Session, preferred_key: str, exclude_id: Optional[int] = None) -> str:
    base_key = _sanitize_skill_market_key(preferred_key) or "skill"
    candidate = base_key
    counter = 2
    while True:
        stmt = select(SkillMarketItem).where(SkillMarketItem.skill_key == candidate)
        rows = session.exec(stmt).all()
        conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None)
        if not conflict:
            return candidate
        candidate = f"{base_key}-{counter}"
        counter += 1


def _resolve_unique_skill_market_zip_filename(
    session: Session,
    filename: str,
    *,
    exclude_filename: Optional[str] = None,
    exclude_id: Optional[int] = None,
) -> str:
    root = _skill_market_root()
    os.makedirs(root, exist_ok=True)
    safe_name = _sanitize_zip_filename(filename)
    if not safe_name.lower().endswith(".zip"):
        raise HTTPException(status_code=400, detail="Only .zip skill package is supported")
    candidate = safe_name
    stem, ext = os.path.splitext(safe_name)
    counter = 2
    while True:
        file_conflict = os.path.exists(os.path.join(root, candidate)) and candidate != str(exclude_filename or "").strip()
        rows = session.exec(select(SkillMarketItem).where(SkillMarketItem.zip_filename == candidate)).all()
        db_conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None)
        if not file_conflict and not db_conflict:
            return candidate
        candidate = f"{stem}-{counter}{ext}"
        counter += 1


async def _store_skill_market_zip_upload(
    session: Session,
    upload: UploadFile,
    *,
    exclude_filename: Optional[str] = None,
    exclude_id: Optional[int] = None,
) -> Dict[str, Any]:
    root = _skill_market_root()
    os.makedirs(root, exist_ok=True)

    incoming_name = _sanitize_zip_filename(upload.filename or "")
    if not incoming_name.lower().endswith(".zip"):
        raise HTTPException(status_code=400, detail="Only .zip skill package is supported")

    target_filename = _resolve_unique_skill_market_zip_filename(
        session,
        incoming_name,
        exclude_filename=exclude_filename,
        exclude_id=exclude_id,
    )
    max_bytes = get_platform_settings_snapshot().upload_max_mb * 1024 * 1024
    total_size = 0
    tmp_path: Optional[str] = None
    try:
        with tempfile.NamedTemporaryFile(prefix=".skill_market_", suffix=".zip", dir=root, delete=False) as tmp_zip:
            tmp_path = tmp_zip.name
            while True:
                chunk = await upload.read(1024 * 1024)
                if not chunk:
                    break
                total_size += len(chunk)
                if total_size > max_bytes:
                    raise HTTPException(
                        status_code=413,
                        detail=f"Zip package too large (max {max_bytes // (1024 * 1024)}MB)",
                    )
                tmp_zip.write(chunk)
        if total_size == 0:
            raise HTTPException(status_code=400, detail="Zip package is empty")
        summary = _extract_skill_zip_summary(tmp_path)
        if not summary["entry_names"]:
            raise HTTPException(status_code=400, detail="Zip package has no valid skill entries")
        final_path = os.path.join(root, target_filename)
        os.replace(tmp_path, final_path)
        tmp_path = None
        return {
            "zip_filename": target_filename,
            "zip_size_bytes": total_size,
            "entry_names": summary["entry_names"],
            "description": summary["description"],
        }
    except zipfile.BadZipFile as exc:
        raise HTTPException(status_code=400, detail="Invalid zip file") from exc
    finally:
        await upload.close()
        if tmp_path and os.path.exists(tmp_path):
            os.remove(tmp_path)


def _serialize_skill_market_item(
    item: SkillMarketItem,
    *,
    install_count: int = 0,
    install_row: Optional[BotSkillInstall] = None,
    workspace_installed: Optional[bool] = None,
    installed_entries: Optional[List[str]] = None,
) -> Dict[str, Any]:
    zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or ""))
    entry_names = _parse_json_string_list(item.entry_names_json)
    payload = {
        "id": item.id,
        "skill_key": item.skill_key,
        "display_name": item.display_name or item.skill_key,
        "description": item.description or "",
        "zip_filename": item.zip_filename,
        "zip_size_bytes": int(item.zip_size_bytes or 0),
        "entry_names": entry_names,
        "entry_count": len(entry_names),
        "zip_exists": os.path.isfile(zip_path),
        "install_count": int(install_count or 0),
        "created_at": item.created_at.isoformat() + "Z" if item.created_at else None,
        "updated_at": item.updated_at.isoformat() + "Z" if item.updated_at else None,
    }
    if install_row is not None:
        resolved_entries = (
            installed_entries
            if installed_entries is not None
            else _parse_json_string_list(install_row.installed_entries_json)
        )
        resolved_installed = workspace_installed if workspace_installed is not None else install_row.status == "INSTALLED"
        payload.update(
            {
                "installed": resolved_installed,
                "install_status": install_row.status,
                "installed_at": install_row.installed_at.isoformat() + "Z" if install_row.installed_at else None,
                "installed_entries": resolved_entries,
                "install_error": install_row.last_error,
            }
        )
    return payload


def _build_install_count_by_skill(session: Session) -> Dict[int, int]:
    installs = session.exec(select(BotSkillInstall)).all()
    install_count_by_skill: Dict[int, int] = {}
    for row in installs:
        skill_id = int(row.skill_market_item_id or 0)
        if skill_id <= 0 or row.status != "INSTALLED":
            continue
        install_count_by_skill[skill_id] = install_count_by_skill.get(skill_id, 0) + 1
    return install_count_by_skill


def list_skill_market_items(session: Session) -> List[Dict[str, Any]]:
    items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all()
    install_count_by_skill = _build_install_count_by_skill(session)
    return [
        _serialize_skill_market_item(item, install_count=install_count_by_skill.get(int(item.id or 0), 0))
        for item in items
    ]


async def create_skill_market_item_record(
    session: Session,
    *,
    skill_key: str,
    display_name: str,
    description: str,
    upload: UploadFile,
) -> Dict[str, Any]:
    upload_meta = await _store_skill_market_zip_upload(session, upload)
    try:
        preferred_key = skill_key or display_name or os.path.splitext(upload_meta["zip_filename"])[0]
        next_key = _resolve_unique_skill_market_key(session, preferred_key)
        item = SkillMarketItem(
            skill_key=next_key,
            display_name=str(display_name or next_key).strip() or next_key,
            description=str(description or upload_meta["description"] or "").strip(),
            zip_filename=upload_meta["zip_filename"],
            zip_size_bytes=int(upload_meta["zip_size_bytes"] or 0),
            entry_names_json=json.dumps(upload_meta["entry_names"], ensure_ascii=False),
        )
        session.add(item)
        session.commit()
        session.refresh(item)
        return _serialize_skill_market_item(item, install_count=0)
    except Exception:
        target_path = os.path.join(_skill_market_root(), upload_meta["zip_filename"])
        if os.path.exists(target_path):
            os.remove(target_path)
        raise


async def update_skill_market_item_record(
    session: Session,
    *,
    skill_id: int,
    skill_key: str,
    display_name: str,
    description: str,
    upload: Optional[UploadFile] = None,
) -> Dict[str, Any]:
    item = session.get(SkillMarketItem, skill_id)
    if not item:
        raise HTTPException(status_code=404, detail="Skill market item not found")

    old_filename = str(item.zip_filename or "").strip()
    upload_meta: Optional[Dict[str, Any]] = None
    if upload is not None:
        upload_meta = await _store_skill_market_zip_upload(
            session,
            upload,
            exclude_filename=old_filename or None,
            exclude_id=item.id,
        )

    next_key = _resolve_unique_skill_market_key(
        session,
        skill_key or item.skill_key or display_name or os.path.splitext(upload_meta["zip_filename"] if upload_meta else old_filename)[0],
        exclude_id=item.id,
    )
    item.skill_key = next_key
    item.display_name = str(display_name or item.display_name or next_key).strip() or next_key
    item.description = str(description or (upload_meta["description"] if upload_meta else item.description) or "").strip()
    item.updated_at = datetime.utcnow()
    if upload_meta:
        item.zip_filename = upload_meta["zip_filename"]
        item.zip_size_bytes = int(upload_meta["zip_size_bytes"] or 0)
        item.entry_names_json = json.dumps(upload_meta["entry_names"], ensure_ascii=False)
    session.add(item)
    session.commit()
    session.refresh(item)

    if upload_meta and old_filename and old_filename != upload_meta["zip_filename"]:
        old_path = os.path.join(_skill_market_root(), old_filename)
        if os.path.exists(old_path):
            os.remove(old_path)

    installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all()
    install_count = sum(1 for row in installs if row.status == "INSTALLED")
    return _serialize_skill_market_item(item, install_count=install_count)


def delete_skill_market_item_record(session: Session, *, skill_id: int) -> Dict[str, Any]:
    item = session.get(SkillMarketItem, skill_id)
    if not item:
        raise HTTPException(status_code=404, detail="Skill market item not found")
    zip_filename = str(item.zip_filename or "").strip()
    installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all()
    for row in installs:
        session.delete(row)
    session.delete(item)
    session.commit()
    if zip_filename:
        zip_path = os.path.join(_skill_market_root(), zip_filename)
        if os.path.exists(zip_path):
            os.remove(zip_path)
    return {"status": "deleted", "id": skill_id}


def list_bot_skill_market_items(session: Session, *, bot_id: str) -> List[Dict[str, Any]]:
    items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all()
    install_rows = session.exec(select(BotSkillInstall).where(BotSkillInstall.bot_id == bot_id)).all()
    install_lookup = {int(row.skill_market_item_id): row for row in install_rows}
    install_count_by_skill = _build_install_count_by_skill(session)
    return [
        _serialize_skill_market_item(
            item,
            install_count=install_count_by_skill.get(int(item.id or 0), 0),
            install_row=install_lookup.get(int(item.id or 0)),
            workspace_installed=(
                None
                if install_lookup.get(int(item.id or 0)) is None
                else (
                    install_lookup[int(item.id or 0)].status == "INSTALLED"
                    and all(
                        os.path.exists(os.path.join(get_bot_skills_root(bot_id), name))
                        for name in _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json)
                    )
                )
            ),
            installed_entries=(
                None
                if install_lookup.get(int(item.id or 0)) is None
                else _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json)
            ),
        )
        for item in items
    ]


def install_skill_market_item_for_bot(
    session: Session,
    *,
    bot_id: str,
    skill_id: int,
) -> Dict[str, Any]:
    item = session.get(SkillMarketItem, skill_id)
    if not item:
        raise HTTPException(status_code=404, detail="Skill market item not found")

    zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or ""))
    if not os.path.isfile(zip_path):
        raise HTTPException(status_code=404, detail="Skill zip package not found")

    install_row = session.exec(
        select(BotSkillInstall).where(
            BotSkillInstall.bot_id == bot_id,
            BotSkillInstall.skill_market_item_id == skill_id,
        )
    ).first()

    try:
        install_result = install_skill_zip_into_workspace(bot_id, zip_path)
        now = datetime.utcnow()
        if not install_row:
            install_row = BotSkillInstall(
                bot_id=bot_id,
                skill_market_item_id=skill_id,
            )
        install_row.installed_entries_json = json.dumps(install_result["installed"], ensure_ascii=False)
        install_row.source_zip_filename = str(item.zip_filename or "")
        install_row.status = "INSTALLED"
        install_row.last_error = None
        install_row.installed_at = now
        install_row.updated_at = now
        session.add(install_row)
        session.commit()
        session.refresh(install_row)
        return {
            "status": "installed",
            "bot_id": bot_id,
            "skill_market_item_id": skill_id,
            "installed": install_result["installed"],
            "skills": install_result["skills"],
            "market_item": _serialize_skill_market_item(item, install_count=0, install_row=install_row),
        }
    except HTTPException as exc:
        now = datetime.utcnow()
        if not install_row:
            install_row = BotSkillInstall(
                bot_id=bot_id,
                skill_market_item_id=skill_id,
                installed_at=now,
            )
        install_row.source_zip_filename = str(item.zip_filename or "")
        install_row.status = "FAILED"
        install_row.last_error = str(exc.detail or "Install failed")
        install_row.updated_at = now
        session.add(install_row)
        session.commit()
        raise
    except Exception as exc:
        now = datetime.utcnow()
        if not install_row:
            install_row = BotSkillInstall(
                bot_id=bot_id,
                skill_market_item_id=skill_id,
                installed_at=now,
            )
        install_row.source_zip_filename = str(item.zip_filename or "")
        install_row.status = "FAILED"
        install_row.last_error = str(exc or "Install failed")[:1000]
        install_row.updated_at = now
        session.add(install_row)
        session.commit()
        raise HTTPException(status_code=500, detail="Skill install failed unexpectedly") from exc