import json import os import tempfile import zipfile from datetime import datetime from typing import Any, Dict, List, Optional from fastapi import HTTPException, UploadFile from sqlmodel import Session, select from core.settings import DATA_ROOT from core.utils import ( _is_ignored_skill_zip_top_level, _is_valid_top_level_skill_name, _parse_json_string_list, _read_description_from_text, _sanitize_skill_market_key, _sanitize_zip_filename, ) from models.skill import BotSkillInstall, SkillMarketItem from services.platform_service import get_platform_settings_snapshot from services.skill_service import get_bot_skills_root, install_skill_zip_into_workspace def _skill_market_root() -> str: return os.path.abspath(os.path.join(DATA_ROOT, "skills")) def _extract_skill_zip_summary(zip_path: str) -> Dict[str, Any]: entry_names: List[str] = [] description = "" with zipfile.ZipFile(zip_path) as archive: members = archive.infolist() file_members = [member for member in members if not member.is_dir()] for member in file_members: raw_name = str(member.filename or "").replace("\\", "/").lstrip("/") if not raw_name: continue first = raw_name.split("/", 1)[0].strip() if _is_ignored_skill_zip_top_level(first): continue if _is_valid_top_level_skill_name(first) and first not in entry_names: entry_names.append(first) candidates = sorted( [ str(member.filename or "").replace("\\", "/").lstrip("/") for member in file_members if str(member.filename or "").replace("\\", "/").rsplit("/", 1)[-1].lower() in {"skill.md", "readme.md"} ], key=lambda value: (value.count("/"), value.lower()), ) for candidate in candidates: try: with archive.open(candidate, "r") as file: preview = file.read(4096).decode("utf-8", errors="ignore") description = _read_description_from_text(preview) if description: break except Exception: continue return { "entry_names": entry_names, "description": description, } def _resolve_unique_skill_market_key(session: Session, preferred_key: str, exclude_id: Optional[int] = None) -> str: base_key = _sanitize_skill_market_key(preferred_key) or "skill" candidate = base_key counter = 2 while True: stmt = select(SkillMarketItem).where(SkillMarketItem.skill_key == candidate) rows = session.exec(stmt).all() conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None) if not conflict: return candidate candidate = f"{base_key}-{counter}" counter += 1 def _resolve_unique_skill_market_zip_filename( session: Session, filename: str, *, exclude_filename: Optional[str] = None, exclude_id: Optional[int] = None, ) -> str: root = _skill_market_root() os.makedirs(root, exist_ok=True) safe_name = _sanitize_zip_filename(filename) if not safe_name.lower().endswith(".zip"): raise HTTPException(status_code=400, detail="Only .zip skill package is supported") candidate = safe_name stem, ext = os.path.splitext(safe_name) counter = 2 while True: file_conflict = os.path.exists(os.path.join(root, candidate)) and candidate != str(exclude_filename or "").strip() rows = session.exec(select(SkillMarketItem).where(SkillMarketItem.zip_filename == candidate)).all() db_conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None) if not file_conflict and not db_conflict: return candidate candidate = f"{stem}-{counter}{ext}" counter += 1 async def _store_skill_market_zip_upload( session: Session, upload: UploadFile, *, exclude_filename: Optional[str] = None, exclude_id: Optional[int] = None, ) -> Dict[str, Any]: root = _skill_market_root() os.makedirs(root, exist_ok=True) incoming_name = _sanitize_zip_filename(upload.filename or "") if not incoming_name.lower().endswith(".zip"): raise HTTPException(status_code=400, detail="Only .zip skill package is supported") target_filename = _resolve_unique_skill_market_zip_filename( session, incoming_name, exclude_filename=exclude_filename, exclude_id=exclude_id, ) max_bytes = get_platform_settings_snapshot().upload_max_mb * 1024 * 1024 total_size = 0 tmp_path: Optional[str] = None try: with tempfile.NamedTemporaryFile(prefix=".skill_market_", suffix=".zip", dir=root, delete=False) as tmp_zip: tmp_path = tmp_zip.name while True: chunk = await upload.read(1024 * 1024) if not chunk: break total_size += len(chunk) if total_size > max_bytes: raise HTTPException( status_code=413, detail=f"Zip package too large (max {max_bytes // (1024 * 1024)}MB)", ) tmp_zip.write(chunk) if total_size == 0: raise HTTPException(status_code=400, detail="Zip package is empty") summary = _extract_skill_zip_summary(tmp_path) if not summary["entry_names"]: raise HTTPException(status_code=400, detail="Zip package has no valid skill entries") final_path = os.path.join(root, target_filename) os.replace(tmp_path, final_path) tmp_path = None return { "zip_filename": target_filename, "zip_size_bytes": total_size, "entry_names": summary["entry_names"], "description": summary["description"], } except zipfile.BadZipFile as exc: raise HTTPException(status_code=400, detail="Invalid zip file") from exc finally: await upload.close() if tmp_path and os.path.exists(tmp_path): os.remove(tmp_path) def _serialize_skill_market_item( item: SkillMarketItem, *, install_count: int = 0, install_row: Optional[BotSkillInstall] = None, workspace_installed: Optional[bool] = None, installed_entries: Optional[List[str]] = None, ) -> Dict[str, Any]: zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or "")) entry_names = _parse_json_string_list(item.entry_names_json) payload = { "id": item.id, "skill_key": item.skill_key, "display_name": item.display_name or item.skill_key, "description": item.description or "", "zip_filename": item.zip_filename, "zip_size_bytes": int(item.zip_size_bytes or 0), "entry_names": entry_names, "entry_count": len(entry_names), "zip_exists": os.path.isfile(zip_path), "install_count": int(install_count or 0), "created_at": item.created_at.isoformat() + "Z" if item.created_at else None, "updated_at": item.updated_at.isoformat() + "Z" if item.updated_at else None, } if install_row is not None: resolved_entries = ( installed_entries if installed_entries is not None else _parse_json_string_list(install_row.installed_entries_json) ) resolved_installed = workspace_installed if workspace_installed is not None else install_row.status == "INSTALLED" payload.update( { "installed": resolved_installed, "install_status": install_row.status, "installed_at": install_row.installed_at.isoformat() + "Z" if install_row.installed_at else None, "installed_entries": resolved_entries, "install_error": install_row.last_error, } ) return payload def _build_install_count_by_skill(session: Session) -> Dict[int, int]: installs = session.exec(select(BotSkillInstall)).all() install_count_by_skill: Dict[int, int] = {} for row in installs: skill_id = int(row.skill_market_item_id or 0) if skill_id <= 0 or row.status != "INSTALLED": continue install_count_by_skill[skill_id] = install_count_by_skill.get(skill_id, 0) + 1 return install_count_by_skill def list_skill_market_items(session: Session) -> List[Dict[str, Any]]: items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all() install_count_by_skill = _build_install_count_by_skill(session) return [ _serialize_skill_market_item(item, install_count=install_count_by_skill.get(int(item.id or 0), 0)) for item in items ] async def create_skill_market_item_record( session: Session, *, skill_key: str, display_name: str, description: str, upload: UploadFile, ) -> Dict[str, Any]: upload_meta = await _store_skill_market_zip_upload(session, upload) try: preferred_key = skill_key or display_name or os.path.splitext(upload_meta["zip_filename"])[0] next_key = _resolve_unique_skill_market_key(session, preferred_key) item = SkillMarketItem( skill_key=next_key, display_name=str(display_name or next_key).strip() or next_key, description=str(description or upload_meta["description"] or "").strip(), zip_filename=upload_meta["zip_filename"], zip_size_bytes=int(upload_meta["zip_size_bytes"] or 0), entry_names_json=json.dumps(upload_meta["entry_names"], ensure_ascii=False), ) session.add(item) session.commit() session.refresh(item) return _serialize_skill_market_item(item, install_count=0) except Exception: target_path = os.path.join(_skill_market_root(), upload_meta["zip_filename"]) if os.path.exists(target_path): os.remove(target_path) raise async def update_skill_market_item_record( session: Session, *, skill_id: int, skill_key: str, display_name: str, description: str, upload: Optional[UploadFile] = None, ) -> Dict[str, Any]: item = session.get(SkillMarketItem, skill_id) if not item: raise HTTPException(status_code=404, detail="Skill market item not found") old_filename = str(item.zip_filename or "").strip() upload_meta: Optional[Dict[str, Any]] = None if upload is not None: upload_meta = await _store_skill_market_zip_upload( session, upload, exclude_filename=old_filename or None, exclude_id=item.id, ) next_key = _resolve_unique_skill_market_key( session, skill_key or item.skill_key or display_name or os.path.splitext(upload_meta["zip_filename"] if upload_meta else old_filename)[0], exclude_id=item.id, ) item.skill_key = next_key item.display_name = str(display_name or item.display_name or next_key).strip() or next_key item.description = str(description or (upload_meta["description"] if upload_meta else item.description) or "").strip() item.updated_at = datetime.utcnow() if upload_meta: item.zip_filename = upload_meta["zip_filename"] item.zip_size_bytes = int(upload_meta["zip_size_bytes"] or 0) item.entry_names_json = json.dumps(upload_meta["entry_names"], ensure_ascii=False) session.add(item) session.commit() session.refresh(item) if upload_meta and old_filename and old_filename != upload_meta["zip_filename"]: old_path = os.path.join(_skill_market_root(), old_filename) if os.path.exists(old_path): os.remove(old_path) installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all() install_count = sum(1 for row in installs if row.status == "INSTALLED") return _serialize_skill_market_item(item, install_count=install_count) def delete_skill_market_item_record(session: Session, *, skill_id: int) -> Dict[str, Any]: item = session.get(SkillMarketItem, skill_id) if not item: raise HTTPException(status_code=404, detail="Skill market item not found") zip_filename = str(item.zip_filename or "").strip() installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all() for row in installs: session.delete(row) session.delete(item) session.commit() if zip_filename: zip_path = os.path.join(_skill_market_root(), zip_filename) if os.path.exists(zip_path): os.remove(zip_path) return {"status": "deleted", "id": skill_id} def list_bot_skill_market_items(session: Session, *, bot_id: str) -> List[Dict[str, Any]]: items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all() install_rows = session.exec(select(BotSkillInstall).where(BotSkillInstall.bot_id == bot_id)).all() install_lookup = {int(row.skill_market_item_id): row for row in install_rows} install_count_by_skill = _build_install_count_by_skill(session) return [ _serialize_skill_market_item( item, install_count=install_count_by_skill.get(int(item.id or 0), 0), install_row=install_lookup.get(int(item.id or 0)), workspace_installed=( None if install_lookup.get(int(item.id or 0)) is None else ( install_lookup[int(item.id or 0)].status == "INSTALLED" and all( os.path.exists(os.path.join(get_bot_skills_root(bot_id), name)) for name in _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json) ) ) ), installed_entries=( None if install_lookup.get(int(item.id or 0)) is None else _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json) ), ) for item in items ] def install_skill_market_item_for_bot( session: Session, *, bot_id: str, skill_id: int, ) -> Dict[str, Any]: item = session.get(SkillMarketItem, skill_id) if not item: raise HTTPException(status_code=404, detail="Skill market item not found") zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or "")) if not os.path.isfile(zip_path): raise HTTPException(status_code=404, detail="Skill zip package not found") install_row = session.exec( select(BotSkillInstall).where( BotSkillInstall.bot_id == bot_id, BotSkillInstall.skill_market_item_id == skill_id, ) ).first() try: install_result = install_skill_zip_into_workspace(bot_id, zip_path) now = datetime.utcnow() if not install_row: install_row = BotSkillInstall( bot_id=bot_id, skill_market_item_id=skill_id, ) install_row.installed_entries_json = json.dumps(install_result["installed"], ensure_ascii=False) install_row.source_zip_filename = str(item.zip_filename or "") install_row.status = "INSTALLED" install_row.last_error = None install_row.installed_at = now install_row.updated_at = now session.add(install_row) session.commit() session.refresh(install_row) return { "status": "installed", "bot_id": bot_id, "skill_market_item_id": skill_id, "installed": install_result["installed"], "skills": install_result["skills"], "market_item": _serialize_skill_market_item(item, install_count=0, install_row=install_row), } except HTTPException as exc: now = datetime.utcnow() if not install_row: install_row = BotSkillInstall( bot_id=bot_id, skill_market_item_id=skill_id, installed_at=now, ) install_row.source_zip_filename = str(item.zip_filename or "") install_row.status = "FAILED" install_row.last_error = str(exc.detail or "Install failed") install_row.updated_at = now session.add(install_row) session.commit() raise except Exception as exc: now = datetime.utcnow() if not install_row: install_row = BotSkillInstall( bot_id=bot_id, skill_market_item_id=skill_id, installed_at=now, ) install_row.source_zip_filename = str(item.zip_filename or "") install_row.status = "FAILED" install_row.last_error = str(exc or "Install failed")[:1000] install_row.updated_at = now session.add(install_row) session.commit() raise HTTPException(status_code=500, detail="Skill install failed unexpectedly") from exc