435 lines
17 KiB
Python
435 lines
17 KiB
Python
import json
|
|
import os
|
|
import tempfile
|
|
import zipfile
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from fastapi import HTTPException, UploadFile
|
|
from sqlmodel import Session, select
|
|
|
|
from core.settings import DATA_ROOT
|
|
from core.utils import (
|
|
_is_ignored_skill_zip_top_level,
|
|
_is_valid_top_level_skill_name,
|
|
_parse_json_string_list,
|
|
_read_description_from_text,
|
|
_sanitize_skill_market_key,
|
|
_sanitize_zip_filename,
|
|
)
|
|
from models.skill import BotSkillInstall, SkillMarketItem
|
|
from services.platform_service import get_platform_settings_snapshot
|
|
from services.skill_service import _install_skill_zip_into_workspace, _skills_root
|
|
|
|
|
|
def _skill_market_root() -> str:
|
|
return os.path.abspath(os.path.join(DATA_ROOT, "skills"))
|
|
|
|
|
|
def _extract_skill_zip_summary(zip_path: str) -> Dict[str, Any]:
|
|
entry_names: List[str] = []
|
|
description = ""
|
|
with zipfile.ZipFile(zip_path) as archive:
|
|
members = archive.infolist()
|
|
file_members = [member for member in members if not member.is_dir()]
|
|
for member in file_members:
|
|
raw_name = str(member.filename or "").replace("\\", "/").lstrip("/")
|
|
if not raw_name:
|
|
continue
|
|
first = raw_name.split("/", 1)[0].strip()
|
|
if _is_ignored_skill_zip_top_level(first):
|
|
continue
|
|
if _is_valid_top_level_skill_name(first) and first not in entry_names:
|
|
entry_names.append(first)
|
|
|
|
candidates = sorted(
|
|
[
|
|
str(member.filename or "").replace("\\", "/").lstrip("/")
|
|
for member in file_members
|
|
if str(member.filename or "").replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
in {"skill.md", "readme.md"}
|
|
],
|
|
key=lambda value: (value.count("/"), value.lower()),
|
|
)
|
|
for candidate in candidates:
|
|
try:
|
|
with archive.open(candidate, "r") as file:
|
|
preview = file.read(4096).decode("utf-8", errors="ignore")
|
|
description = _read_description_from_text(preview)
|
|
if description:
|
|
break
|
|
except Exception:
|
|
continue
|
|
return {
|
|
"entry_names": entry_names,
|
|
"description": description,
|
|
}
|
|
|
|
|
|
def _resolve_unique_skill_market_key(session: Session, preferred_key: str, exclude_id: Optional[int] = None) -> str:
|
|
base_key = _sanitize_skill_market_key(preferred_key) or "skill"
|
|
candidate = base_key
|
|
counter = 2
|
|
while True:
|
|
stmt = select(SkillMarketItem).where(SkillMarketItem.skill_key == candidate)
|
|
rows = session.exec(stmt).all()
|
|
conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None)
|
|
if not conflict:
|
|
return candidate
|
|
candidate = f"{base_key}-{counter}"
|
|
counter += 1
|
|
|
|
|
|
def _resolve_unique_skill_market_zip_filename(
|
|
session: Session,
|
|
filename: str,
|
|
*,
|
|
exclude_filename: Optional[str] = None,
|
|
exclude_id: Optional[int] = None,
|
|
) -> str:
|
|
root = _skill_market_root()
|
|
os.makedirs(root, exist_ok=True)
|
|
safe_name = _sanitize_zip_filename(filename)
|
|
if not safe_name.lower().endswith(".zip"):
|
|
raise HTTPException(status_code=400, detail="Only .zip skill package is supported")
|
|
candidate = safe_name
|
|
stem, ext = os.path.splitext(safe_name)
|
|
counter = 2
|
|
while True:
|
|
file_conflict = os.path.exists(os.path.join(root, candidate)) and candidate != str(exclude_filename or "").strip()
|
|
rows = session.exec(select(SkillMarketItem).where(SkillMarketItem.zip_filename == candidate)).all()
|
|
db_conflict = next((row for row in rows if exclude_id is None or row.id != exclude_id), None)
|
|
if not file_conflict and not db_conflict:
|
|
return candidate
|
|
candidate = f"{stem}-{counter}{ext}"
|
|
counter += 1
|
|
|
|
|
|
async def _store_skill_market_zip_upload(
|
|
session: Session,
|
|
upload: UploadFile,
|
|
*,
|
|
exclude_filename: Optional[str] = None,
|
|
exclude_id: Optional[int] = None,
|
|
) -> Dict[str, Any]:
|
|
root = _skill_market_root()
|
|
os.makedirs(root, exist_ok=True)
|
|
|
|
incoming_name = _sanitize_zip_filename(upload.filename or "")
|
|
if not incoming_name.lower().endswith(".zip"):
|
|
raise HTTPException(status_code=400, detail="Only .zip skill package is supported")
|
|
|
|
target_filename = _resolve_unique_skill_market_zip_filename(
|
|
session,
|
|
incoming_name,
|
|
exclude_filename=exclude_filename,
|
|
exclude_id=exclude_id,
|
|
)
|
|
max_bytes = get_platform_settings_snapshot().upload_max_mb * 1024 * 1024
|
|
total_size = 0
|
|
tmp_path: Optional[str] = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(prefix=".skill_market_", suffix=".zip", dir=root, delete=False) as tmp_zip:
|
|
tmp_path = tmp_zip.name
|
|
while True:
|
|
chunk = await upload.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
total_size += len(chunk)
|
|
if total_size > max_bytes:
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=f"Zip package too large (max {max_bytes // (1024 * 1024)}MB)",
|
|
)
|
|
tmp_zip.write(chunk)
|
|
if total_size == 0:
|
|
raise HTTPException(status_code=400, detail="Zip package is empty")
|
|
summary = _extract_skill_zip_summary(tmp_path)
|
|
if not summary["entry_names"]:
|
|
raise HTTPException(status_code=400, detail="Zip package has no valid skill entries")
|
|
final_path = os.path.join(root, target_filename)
|
|
os.replace(tmp_path, final_path)
|
|
tmp_path = None
|
|
return {
|
|
"zip_filename": target_filename,
|
|
"zip_size_bytes": total_size,
|
|
"entry_names": summary["entry_names"],
|
|
"description": summary["description"],
|
|
}
|
|
except zipfile.BadZipFile as exc:
|
|
raise HTTPException(status_code=400, detail="Invalid zip file") from exc
|
|
finally:
|
|
await upload.close()
|
|
if tmp_path and os.path.exists(tmp_path):
|
|
os.remove(tmp_path)
|
|
|
|
|
|
def _serialize_skill_market_item(
|
|
item: SkillMarketItem,
|
|
*,
|
|
install_count: int = 0,
|
|
install_row: Optional[BotSkillInstall] = None,
|
|
workspace_installed: Optional[bool] = None,
|
|
installed_entries: Optional[List[str]] = None,
|
|
) -> Dict[str, Any]:
|
|
zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or ""))
|
|
entry_names = _parse_json_string_list(item.entry_names_json)
|
|
payload = {
|
|
"id": item.id,
|
|
"skill_key": item.skill_key,
|
|
"display_name": item.display_name or item.skill_key,
|
|
"description": item.description or "",
|
|
"zip_filename": item.zip_filename,
|
|
"zip_size_bytes": int(item.zip_size_bytes or 0),
|
|
"entry_names": entry_names,
|
|
"entry_count": len(entry_names),
|
|
"zip_exists": os.path.isfile(zip_path),
|
|
"install_count": int(install_count or 0),
|
|
"created_at": item.created_at.isoformat() + "Z" if item.created_at else None,
|
|
"updated_at": item.updated_at.isoformat() + "Z" if item.updated_at else None,
|
|
}
|
|
if install_row is not None:
|
|
resolved_entries = (
|
|
installed_entries
|
|
if installed_entries is not None
|
|
else _parse_json_string_list(install_row.installed_entries_json)
|
|
)
|
|
resolved_installed = workspace_installed if workspace_installed is not None else install_row.status == "INSTALLED"
|
|
payload.update(
|
|
{
|
|
"installed": resolved_installed,
|
|
"install_status": install_row.status,
|
|
"installed_at": install_row.installed_at.isoformat() + "Z" if install_row.installed_at else None,
|
|
"installed_entries": resolved_entries,
|
|
"install_error": install_row.last_error,
|
|
}
|
|
)
|
|
return payload
|
|
|
|
|
|
def _build_install_count_by_skill(session: Session) -> Dict[int, int]:
|
|
installs = session.exec(select(BotSkillInstall)).all()
|
|
install_count_by_skill: Dict[int, int] = {}
|
|
for row in installs:
|
|
skill_id = int(row.skill_market_item_id or 0)
|
|
if skill_id <= 0 or row.status != "INSTALLED":
|
|
continue
|
|
install_count_by_skill[skill_id] = install_count_by_skill.get(skill_id, 0) + 1
|
|
return install_count_by_skill
|
|
|
|
|
|
def list_skill_market_items(session: Session) -> List[Dict[str, Any]]:
|
|
items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all()
|
|
install_count_by_skill = _build_install_count_by_skill(session)
|
|
return [
|
|
_serialize_skill_market_item(item, install_count=install_count_by_skill.get(int(item.id or 0), 0))
|
|
for item in items
|
|
]
|
|
|
|
|
|
async def create_skill_market_item_record(
|
|
session: Session,
|
|
*,
|
|
skill_key: str,
|
|
display_name: str,
|
|
description: str,
|
|
upload: UploadFile,
|
|
) -> Dict[str, Any]:
|
|
upload_meta = await _store_skill_market_zip_upload(session, upload)
|
|
try:
|
|
preferred_key = skill_key or display_name or os.path.splitext(upload_meta["zip_filename"])[0]
|
|
next_key = _resolve_unique_skill_market_key(session, preferred_key)
|
|
item = SkillMarketItem(
|
|
skill_key=next_key,
|
|
display_name=str(display_name or next_key).strip() or next_key,
|
|
description=str(description or upload_meta["description"] or "").strip(),
|
|
zip_filename=upload_meta["zip_filename"],
|
|
zip_size_bytes=int(upload_meta["zip_size_bytes"] or 0),
|
|
entry_names_json=json.dumps(upload_meta["entry_names"], ensure_ascii=False),
|
|
)
|
|
session.add(item)
|
|
session.commit()
|
|
session.refresh(item)
|
|
return _serialize_skill_market_item(item, install_count=0)
|
|
except Exception:
|
|
target_path = os.path.join(_skill_market_root(), upload_meta["zip_filename"])
|
|
if os.path.exists(target_path):
|
|
os.remove(target_path)
|
|
raise
|
|
|
|
|
|
async def update_skill_market_item_record(
|
|
session: Session,
|
|
*,
|
|
skill_id: int,
|
|
skill_key: str,
|
|
display_name: str,
|
|
description: str,
|
|
upload: Optional[UploadFile] = None,
|
|
) -> Dict[str, Any]:
|
|
item = session.get(SkillMarketItem, skill_id)
|
|
if not item:
|
|
raise HTTPException(status_code=404, detail="Skill market item not found")
|
|
|
|
old_filename = str(item.zip_filename or "").strip()
|
|
upload_meta: Optional[Dict[str, Any]] = None
|
|
if upload is not None:
|
|
upload_meta = await _store_skill_market_zip_upload(
|
|
session,
|
|
upload,
|
|
exclude_filename=old_filename or None,
|
|
exclude_id=item.id,
|
|
)
|
|
|
|
next_key = _resolve_unique_skill_market_key(
|
|
session,
|
|
skill_key or item.skill_key or display_name or os.path.splitext(upload_meta["zip_filename"] if upload_meta else old_filename)[0],
|
|
exclude_id=item.id,
|
|
)
|
|
item.skill_key = next_key
|
|
item.display_name = str(display_name or item.display_name or next_key).strip() or next_key
|
|
item.description = str(description or (upload_meta["description"] if upload_meta else item.description) or "").strip()
|
|
item.updated_at = datetime.utcnow()
|
|
if upload_meta:
|
|
item.zip_filename = upload_meta["zip_filename"]
|
|
item.zip_size_bytes = int(upload_meta["zip_size_bytes"] or 0)
|
|
item.entry_names_json = json.dumps(upload_meta["entry_names"], ensure_ascii=False)
|
|
session.add(item)
|
|
session.commit()
|
|
session.refresh(item)
|
|
|
|
if upload_meta and old_filename and old_filename != upload_meta["zip_filename"]:
|
|
old_path = os.path.join(_skill_market_root(), old_filename)
|
|
if os.path.exists(old_path):
|
|
os.remove(old_path)
|
|
|
|
installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all()
|
|
install_count = sum(1 for row in installs if row.status == "INSTALLED")
|
|
return _serialize_skill_market_item(item, install_count=install_count)
|
|
|
|
|
|
def delete_skill_market_item_record(session: Session, *, skill_id: int) -> Dict[str, Any]:
|
|
item = session.get(SkillMarketItem, skill_id)
|
|
if not item:
|
|
raise HTTPException(status_code=404, detail="Skill market item not found")
|
|
zip_filename = str(item.zip_filename or "").strip()
|
|
installs = session.exec(select(BotSkillInstall).where(BotSkillInstall.skill_market_item_id == skill_id)).all()
|
|
for row in installs:
|
|
session.delete(row)
|
|
session.delete(item)
|
|
session.commit()
|
|
if zip_filename:
|
|
zip_path = os.path.join(_skill_market_root(), zip_filename)
|
|
if os.path.exists(zip_path):
|
|
os.remove(zip_path)
|
|
return {"status": "deleted", "id": skill_id}
|
|
|
|
|
|
def list_bot_skill_market_items(session: Session, *, bot_id: str) -> List[Dict[str, Any]]:
|
|
items = session.exec(select(SkillMarketItem).order_by(SkillMarketItem.display_name, SkillMarketItem.id)).all()
|
|
install_rows = session.exec(select(BotSkillInstall).where(BotSkillInstall.bot_id == bot_id)).all()
|
|
install_lookup = {int(row.skill_market_item_id): row for row in install_rows}
|
|
install_count_by_skill = _build_install_count_by_skill(session)
|
|
return [
|
|
_serialize_skill_market_item(
|
|
item,
|
|
install_count=install_count_by_skill.get(int(item.id or 0), 0),
|
|
install_row=install_lookup.get(int(item.id or 0)),
|
|
workspace_installed=(
|
|
None
|
|
if install_lookup.get(int(item.id or 0)) is None
|
|
else (
|
|
install_lookup[int(item.id or 0)].status == "INSTALLED"
|
|
and all(
|
|
os.path.exists(os.path.join(_skills_root(bot_id), name))
|
|
for name in _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json)
|
|
)
|
|
)
|
|
),
|
|
installed_entries=(
|
|
None
|
|
if install_lookup.get(int(item.id or 0)) is None
|
|
else _parse_json_string_list(install_lookup[int(item.id or 0)].installed_entries_json)
|
|
),
|
|
)
|
|
for item in items
|
|
]
|
|
|
|
|
|
def install_skill_market_item_for_bot(
|
|
session: Session,
|
|
*,
|
|
bot_id: str,
|
|
skill_id: int,
|
|
) -> Dict[str, Any]:
|
|
item = session.get(SkillMarketItem, skill_id)
|
|
if not item:
|
|
raise HTTPException(status_code=404, detail="Skill market item not found")
|
|
|
|
zip_path = os.path.join(_skill_market_root(), str(item.zip_filename or ""))
|
|
if not os.path.isfile(zip_path):
|
|
raise HTTPException(status_code=404, detail="Skill zip package not found")
|
|
|
|
install_row = session.exec(
|
|
select(BotSkillInstall).where(
|
|
BotSkillInstall.bot_id == bot_id,
|
|
BotSkillInstall.skill_market_item_id == skill_id,
|
|
)
|
|
).first()
|
|
|
|
try:
|
|
install_result = _install_skill_zip_into_workspace(bot_id, zip_path)
|
|
now = datetime.utcnow()
|
|
if not install_row:
|
|
install_row = BotSkillInstall(
|
|
bot_id=bot_id,
|
|
skill_market_item_id=skill_id,
|
|
)
|
|
install_row.installed_entries_json = json.dumps(install_result["installed"], ensure_ascii=False)
|
|
install_row.source_zip_filename = str(item.zip_filename or "")
|
|
install_row.status = "INSTALLED"
|
|
install_row.last_error = None
|
|
install_row.installed_at = now
|
|
install_row.updated_at = now
|
|
session.add(install_row)
|
|
session.commit()
|
|
session.refresh(install_row)
|
|
return {
|
|
"status": "installed",
|
|
"bot_id": bot_id,
|
|
"skill_market_item_id": skill_id,
|
|
"installed": install_result["installed"],
|
|
"skills": install_result["skills"],
|
|
"market_item": _serialize_skill_market_item(item, install_count=0, install_row=install_row),
|
|
}
|
|
except HTTPException as exc:
|
|
now = datetime.utcnow()
|
|
if not install_row:
|
|
install_row = BotSkillInstall(
|
|
bot_id=bot_id,
|
|
skill_market_item_id=skill_id,
|
|
installed_at=now,
|
|
)
|
|
install_row.source_zip_filename = str(item.zip_filename or "")
|
|
install_row.status = "FAILED"
|
|
install_row.last_error = str(exc.detail or "Install failed")
|
|
install_row.updated_at = now
|
|
session.add(install_row)
|
|
session.commit()
|
|
raise
|
|
except Exception as exc:
|
|
now = datetime.utcnow()
|
|
if not install_row:
|
|
install_row = BotSkillInstall(
|
|
bot_id=bot_id,
|
|
skill_market_item_id=skill_id,
|
|
installed_at=now,
|
|
)
|
|
install_row.source_zip_filename = str(item.zip_filename or "")
|
|
install_row.status = "FAILED"
|
|
install_row.last_error = str(exc or "Install failed")[:1000]
|
|
install_row.updated_at = now
|
|
session.add(install_row)
|
|
session.commit()
|
|
raise HTTPException(status_code=500, detail="Skill install failed unexpectedly") from exc
|