mikhail
1644ff82e0
- Группировка 26K HTML-страниц по COM-интерфейсному префиксу (one MD per interface) - Пропуск KsAPI (ksapi_*.html) — используется только COM API5/API7 - YAML-фронтматтер: type, api, domain, tags, sources - Изображения копируются файлами в resources/ (не base64), collision-safe именование - Классификация: interface / enum / struct / guide по имени и заголовку - Теги API-версии (api7/api5) и домена (3d, 2d, sketch, assembly, ...) - Генерация docs/Kompas3D_SDK/index.md с TOC по 4 категориям - Добавлен docs/Kompas3D_SDK/ в .gitignore (генерируемый артефакт) Результат: 3080 статей, 860 интерфейсов, 392 enum, 428 структур, 1400 руководств, 25 изображений Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
472 lines
17 KiB
Python
472 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
build_kompas3d_sdk.py — Преобразование KOMPAS SDK HTML-справки в
|
||
структурированную базу знаний docs/Kompas3D_SDK/.
|
||
|
||
Стратегия:
|
||
- Группировка: одна MD-статья на COM-интерфейс/тему (по префиксу имени файла).
|
||
- Изображения: копируются файлами в resources/ (не base64).
|
||
- KsAPI (ksapi_*.html): пропускаются.
|
||
- Фронтматтер: type, api, domain, tags, sources.
|
||
|
||
Запуск (из корня репо):
|
||
python tools/build_kompas3d_sdk.py
|
||
"""
|
||
import os
|
||
import sys
|
||
import re
|
||
import shutil
|
||
import html as hmod
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from collections import defaultdict
|
||
from typing import Optional
|
||
|
||
HERE = Path(__file__).resolve().parent.parent
|
||
DOC_DIR = HERE / "docs" / "KOMPAS_SDK_ru-RU"
|
||
INDEX_PATH = HERE / "docs" / "kompas_sdk_index.tsv"
|
||
OUT_DIR = HERE / "docs" / "Kompas3D_SDK"
|
||
RESOURCES_DIR = OUT_DIR / "resources"
|
||
|
||
UI_IMAGE_PREFIXES = [
|
||
"leftright", "toc", "mob_", "TextPlus", "TextMinus", "TextNormal",
|
||
"ZoomIn", "ZoomClose", "spacer.gif", "bg.png", "page-bg.png",
|
||
"blackpaisley.jpeg", "banner_company", "kompas_logo_circle",
|
||
"kompas-logo", "internet.svg", "icon-new-window", "icon-open-file",
|
||
"warn", "note", "closehelp", "cicon_loadindex_ani", "kw-page",
|
||
"social_mail", "_a", "_praezession", "kbd.png", "page-icon",
|
||
"logo-image-desk", "logo-image-mobile", "close_data", "open_data",
|
||
]
|
||
|
||
NOISE_WORDS = {
|
||
"please enable javascript to view this site.",
|
||
"scroll", "предыдущий", "вверх", "следующий",
|
||
"содержание", "содержане", "указатель", "поиск",
|
||
"подразделы:", "(отсутствуют)", "поделиться",
|
||
}
|
||
FOOTER_RE = re.compile(r"©?\s*ООО\s*«?АСКОН")
|
||
|
||
|
||
# ─── Утилиты ────────────────────────────────────────────────────────────────
|
||
|
||
def is_informative_image(src: str) -> bool:
|
||
basename = os.path.basename(src).lower()
|
||
for prefix in UI_IMAGE_PREFIXES:
|
||
if prefix.lower() in basename:
|
||
return False
|
||
img_path = DOC_DIR / src
|
||
if img_path.exists() and img_path.stat().st_size < 500:
|
||
return False
|
||
return True
|
||
|
||
|
||
class TopicExtractor(HTMLParser):
|
||
BLOCK_TAGS = {"p", "div", "br", "tr", "li", "h1", "h2", "h3", "h4",
|
||
"table", "pre", "td", "th"}
|
||
VOID_TAGS = {"img", "input", "link", "meta", "hr", "area", "base",
|
||
"col", "embed", "source", "track", "wbr"}
|
||
|
||
def __init__(self):
|
||
super().__init__(convert_charrefs=True)
|
||
self.depth = 0
|
||
self.capturing = False
|
||
self.skip = 0
|
||
self.out = []
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrd = dict(attrs)
|
||
if not self.capturing and attrd.get("id") == "topicbody":
|
||
self.capturing = True
|
||
self.depth = 1
|
||
return
|
||
if not self.capturing:
|
||
return
|
||
if tag in ("script", "style"):
|
||
self.skip += 1
|
||
if tag in self.BLOCK_TAGS:
|
||
self.out.append("\n")
|
||
if tag not in ("br",) and tag not in self.VOID_TAGS:
|
||
self.depth += 1
|
||
|
||
def handle_endtag(self, tag):
|
||
if not self.capturing:
|
||
return
|
||
if tag in ("script", "style") and self.skip:
|
||
self.skip -= 1
|
||
if tag != "br":
|
||
self.depth -= 1
|
||
if self.depth <= 0:
|
||
self.capturing = False
|
||
|
||
def handle_data(self, data):
|
||
if self.capturing and not self.skip:
|
||
self.out.append(data)
|
||
|
||
|
||
def clean_text(raw: str) -> list[str]:
|
||
cleaned = []
|
||
for line in raw.splitlines():
|
||
s = re.sub(r"[ \t]+", " ", line).strip()
|
||
if not s:
|
||
continue
|
||
if s.lower() in NOISE_WORDS:
|
||
continue
|
||
if FOOTER_RE.search(s):
|
||
break
|
||
cleaned.append(s)
|
||
return cleaned
|
||
|
||
|
||
|
||
def extract_title_and_breadcrumbs(html_content: str) -> tuple[str, list[tuple[str, str]]]:
|
||
title = ""
|
||
for m in re.finditer(r'<title>(.*?)</title>', html_content):
|
||
title = hmod.unescape(m.group(1)).strip()
|
||
break
|
||
if not title:
|
||
for m in re.finditer(r'<p\s+class="topictitle"[^>]*>(.*?)</p>', html_content, re.DOTALL):
|
||
title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
|
||
title = hmod.unescape(title)
|
||
break
|
||
breadcrumbs = []
|
||
bc_match = re.search(
|
||
r'<p\s+id="ptopic_breadcrumbs"[^>]*>(.*?)</p>',
|
||
html_content, re.DOTALL)
|
||
if bc_match:
|
||
bc_html = bc_match.group(1)
|
||
for a_m in re.finditer(r'<a[^>]+href="([^"]*)"[^>]*>(.*?)</a>', bc_html, re.DOTALL):
|
||
href = a_m.group(1)
|
||
text = re.sub(r'<[^>]+>', '', a_m.group(2)).strip()
|
||
text = hmod.unescape(text)
|
||
if text:
|
||
breadcrumbs.append((text, os.path.basename(href)))
|
||
return title, breadcrumbs
|
||
|
||
|
||
def read_index() -> list[dict]:
|
||
pages = []
|
||
with open(INDEX_PATH, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
parts = line.rstrip("\n").split("\t")
|
||
if len(parts) >= 1 and parts[0]:
|
||
pages.append({
|
||
"file": parts[0],
|
||
"title": parts[1] if len(parts) > 1 else "",
|
||
"desc": parts[2] if len(parts) > 2 else "",
|
||
})
|
||
return pages
|
||
|
||
|
||
# ─── Группировка ─────────────────────────────────────────────────────────────
|
||
|
||
def get_group_key(filename: str) -> str:
|
||
"""
|
||
Извлекает ключ группы из имени файла.
|
||
Стратегия: часть до первого '_' (после удаления ведущего '_').
|
||
Примеры:
|
||
'iapplication_visible.html' → 'iapplication'
|
||
'iapplication_visible_ex.html' → 'iapplication'
|
||
'_dkgax_caption.html' → 'dkgax'
|
||
'3d_curves.html' → '3d'
|
||
'zoomtypeenum.html' → 'zoomtypeenum' (standalone)
|
||
"""
|
||
stem = filename.replace(".html", "")
|
||
if stem.startswith("_"):
|
||
stem = stem[1:]
|
||
parts = stem.split("_", 1)
|
||
return parts[0] if len(parts) > 1 else stem
|
||
|
||
|
||
def build_groups(pages: list[dict]) -> dict[str, list[dict]]:
|
||
"""
|
||
Разбивает страницы на группы по ключу.
|
||
Пропускает ksapi_* (KsAPI — Qt/C++ cross-platform, не нужен).
|
||
"""
|
||
raw: dict[str, list[dict]] = defaultdict(list)
|
||
for page in pages:
|
||
fname = page["file"]
|
||
if fname.startswith("ksapi_"):
|
||
continue
|
||
key = get_group_key(fname)
|
||
raw[key].append(page)
|
||
return dict(raw)
|
||
|
||
|
||
# ─── Классификация ───────────────────────────────────────────────────────────
|
||
|
||
def classify_type(group_key: str, representative_title: str) -> str:
|
||
"""Определяет тип: 'interface', 'enum', 'struct', 'guide'."""
|
||
key = group_key.lower()
|
||
title = representative_title.lower()
|
||
|
||
if key.endswith("enum") or "enum" in title or "перечислени" in title:
|
||
return "enum"
|
||
if re.match(r"^i[a-z]", key):
|
||
return "interface"
|
||
if any(s in key for s in ["param", "definition", "struct"]):
|
||
return "struct"
|
||
if any(s in title for s in ["параметры", "структура", "param "]):
|
||
return "struct"
|
||
return "guide"
|
||
|
||
|
||
def extract_api_version(title: str, breadcrumbs: list[tuple[str, str]]) -> list[str]:
|
||
all_text = (title + " " + " ".join(t for t, _ in breadcrumbs)).lower()
|
||
versions = []
|
||
if "версия 7" in all_text or "api7" in all_text or "api 7" in all_text:
|
||
versions.append("api7")
|
||
if "api5" in all_text or "api 5" in all_text:
|
||
versions.append("api5")
|
||
return versions or ["api7"]
|
||
|
||
|
||
def extract_domain_tags(title: str, breadcrumbs: list[tuple[str, str]]) -> list[str]:
|
||
all_text = (title + " " + " ".join(t for t, _ in breadcrumbs)).lower()
|
||
domain_map = {
|
||
"3d": ["3d", "трёхмерн", "3д", "solid", "тело"],
|
||
"2d": ["2d", "двухмерн", "2д", "плоск"],
|
||
"sketch": ["эскиз"],
|
||
"assembly": ["сборк", "компонент", "assembly"],
|
||
"sheetmetal": ["листовой", "сгиб", "гибк"],
|
||
"drawing": ["чертёж", "чертеж", "drawing", "выносн"],
|
||
"feature": ["операци", "feature", "выдавлив", "вращени", "скругл", "фаск"],
|
||
}
|
||
found = []
|
||
for domain, keywords in domain_map.items():
|
||
if any(k in all_text for k in keywords):
|
||
found.append(domain)
|
||
return found
|
||
|
||
|
||
def build_tags(article_type: str, api_versions: list[str],
|
||
domain_tags: list[str], group_key: str) -> list[str]:
|
||
tags = [article_type] + api_versions + domain_tags + [group_key]
|
||
return list(dict.fromkeys(tags))
|
||
|
||
|
||
# ─── Парсинг HTML ────────────────────────────────────────────────────────────
|
||
|
||
def parse_page(fpath: Path) -> Optional[dict]:
|
||
"""Парсит одну HTML-страницу. Возвращает None если страница пустая."""
|
||
try:
|
||
html_content = fpath.read_text(encoding="utf-8-sig")
|
||
except Exception as e:
|
||
print(f" ⚠ Ошибка чтения {fpath.name}: {e}", file=sys.stderr)
|
||
return None
|
||
|
||
title, breadcrumbs = extract_title_and_breadcrumbs(html_content)
|
||
|
||
extractor = TopicExtractor()
|
||
extractor.feed(html_content)
|
||
raw_text = "".join(extractor.out)
|
||
|
||
body_lines = clean_text(raw_text)
|
||
if not body_lines or len(body_lines) <= 2:
|
||
return None
|
||
|
||
image_srcs: list[tuple[Path, str]] = []
|
||
for m in re.finditer(r'<img[^>]+>', html_content):
|
||
tag = m.group()
|
||
src_m = re.search(r'''src=["']([^"']+)["']''', tag)
|
||
alt_m = re.search(r'''alt=["']([^"']*)["']''', tag)
|
||
if not src_m:
|
||
continue
|
||
src = src_m.group(1)
|
||
alt = alt_m.group(1) if alt_m else ""
|
||
if is_informative_image(src):
|
||
img_path = DOC_DIR / src
|
||
if img_path.exists():
|
||
image_srcs.append((img_path, alt))
|
||
|
||
# HTML entities are already decoded by TopicExtractor(convert_charrefs=True)
|
||
# No need to unescape again
|
||
|
||
return {
|
||
"title": title,
|
||
"breadcrumbs": breadcrumbs,
|
||
"body_lines": body_lines,
|
||
"image_srcs": image_srcs,
|
||
}
|
||
|
||
|
||
# ─── Изображения ─────────────────────────────────────────────────────────────
|
||
|
||
def copy_image(img_path: Path) -> str:
|
||
"""Копирует изображение в resources/ и возвращает уникальное имя файла."""
|
||
RESOURCES_DIR.mkdir(parents=True, exist_ok=True)
|
||
# Build collision-safe name from relative path: "images/foo.png" → "images_foo.png"
|
||
try:
|
||
rel = img_path.relative_to(DOC_DIR)
|
||
dest_name = str(rel).replace(os.sep, "_").replace("/", "_")
|
||
except ValueError:
|
||
dest_name = img_path.name # fallback if not under DOC_DIR
|
||
dest = RESOURCES_DIR / dest_name
|
||
if not dest.exists():
|
||
shutil.copy2(img_path, dest)
|
||
return dest_name
|
||
|
||
|
||
# ─── Генерация MD ────────────────────────────────────────────────────────────
|
||
|
||
SUBDIR_FOR_TYPE = {
|
||
"interface": "interfaces",
|
||
"enum": "enums",
|
||
"struct": "structures",
|
||
"guide": "guides",
|
||
}
|
||
|
||
|
||
def render_article(group_key: str, pages_data: list[dict],
|
||
article_type: str, source_files: list[str]) -> str:
|
||
"""Генерирует содержимое MD-файла для группы страниц."""
|
||
rep = pages_data[0]
|
||
title = rep["title"]
|
||
breadcrumbs = rep["breadcrumbs"]
|
||
|
||
api_versions = extract_api_version(title, breadcrumbs)
|
||
domain_tags = extract_domain_tags(title, breadcrumbs)
|
||
tags = build_tags(article_type, api_versions, domain_tags, group_key)
|
||
|
||
fm_lines = [
|
||
"---",
|
||
f'title: "{title}"',
|
||
f"type: {article_type}",
|
||
f"api: [{', '.join(api_versions)}]",
|
||
]
|
||
if domain_tags:
|
||
fm_lines.append(f"domain: [{', '.join(domain_tags)}]")
|
||
fm_lines.append(f"tags: [{', '.join(tags)}]")
|
||
fm_lines.append("sources:")
|
||
for sf in source_files:
|
||
fm_lines.append(f" - {sf}")
|
||
fm_lines.append("---")
|
||
fm_lines.append("")
|
||
|
||
md_parts = ["\n".join(fm_lines)]
|
||
|
||
if breadcrumbs:
|
||
bc_text = " > ".join(t for t, _ in breadcrumbs)
|
||
md_parts.append(f"*{bc_text}*\n")
|
||
|
||
for page_data in pages_data:
|
||
page_title = page_data["title"]
|
||
|
||
if len(pages_data) > 1:
|
||
md_parts.append(f"\n## {page_title}\n")
|
||
|
||
for img_path, alt in page_data["image_srcs"]:
|
||
img_name = copy_image(img_path)
|
||
alt_clean = alt.replace('"', "'")
|
||
md_parts.append(f"\n")
|
||
|
||
md_parts.append("\n".join(page_data["body_lines"]))
|
||
md_parts.append("")
|
||
|
||
return "\n".join(md_parts)
|
||
|
||
|
||
# ─── Главная функция ──────────────────────────────────────────────────────────
|
||
|
||
def main():
|
||
for _s in (sys.stdout, sys.stderr):
|
||
try:
|
||
_s.reconfigure(encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
|
||
print(f"Входная справка: {DOC_DIR}")
|
||
print(f"Выходная база: {OUT_DIR}")
|
||
|
||
for subdir in SUBDIR_FOR_TYPE.values():
|
||
(OUT_DIR / subdir).mkdir(parents=True, exist_ok=True)
|
||
RESOURCES_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
pages = read_index()
|
||
print(f"Страниц в индексе: {len(pages)}")
|
||
|
||
groups = build_groups(pages)
|
||
print(f"Групп после группировки: {len(groups)}")
|
||
|
||
stats = {"processed": 0, "skipped": 0, "errors": 0, "images_copied": 0}
|
||
index_entries: list[tuple[str, str, str]] = []
|
||
|
||
for group_key, group_pages in sorted(groups.items()):
|
||
pages_data: list[dict] = []
|
||
source_files: list[str] = []
|
||
rep_title = ""
|
||
|
||
for page_info in group_pages:
|
||
fname = page_info["file"]
|
||
fpath = DOC_DIR / fname
|
||
if not fpath.exists():
|
||
stats["skipped"] += 1
|
||
continue
|
||
parsed = parse_page(fpath)
|
||
if parsed is None:
|
||
stats["skipped"] += 1
|
||
continue
|
||
pages_data.append(parsed)
|
||
source_files.append(fname)
|
||
if not rep_title:
|
||
rep_title = parsed["title"]
|
||
stats["images_copied"] += len(parsed["image_srcs"])
|
||
|
||
if not pages_data:
|
||
continue
|
||
|
||
article_type = classify_type(group_key, rep_title)
|
||
subdir = SUBDIR_FOR_TYPE.get(article_type, "guides")
|
||
out_name = group_key + ".md"
|
||
out_path = OUT_DIR / subdir / out_name
|
||
|
||
try:
|
||
content = render_article(group_key, pages_data, article_type, source_files)
|
||
out_path.write_text(content, encoding="utf-8")
|
||
stats["processed"] += 1
|
||
index_entries.append((subdir, out_name, rep_title))
|
||
except Exception as e:
|
||
print(f" ⚠ Ошибка записи {out_name}: {e}", file=sys.stderr)
|
||
stats["errors"] += 1
|
||
|
||
_write_index(index_entries)
|
||
|
||
print(f"\n✅ Done!")
|
||
print(f" Статей записано: {stats['processed']}")
|
||
print(f" Страниц пропущено: {stats['skipped']}")
|
||
print(f" Ошибок: {stats['errors']}")
|
||
print(f" Изображений: {stats['images_copied']}")
|
||
|
||
|
||
def _write_index(entries: list[tuple[str, str, str]]):
|
||
"""Генерирует docs/Kompas3D_SDK/index.md с оглавлением по категориям."""
|
||
lines = [
|
||
"# КОМПАС-3D SDK — База знаний\n",
|
||
"> Автоматически сгенерировано из `docs/KOMPAS_SDK_ru-RU/` скриптом `build_kompas3d_sdk.py`.\n",
|
||
"",
|
||
]
|
||
categories = {
|
||
"interfaces": "## Интерфейсы (COM API)",
|
||
"enums": "## Перечисления",
|
||
"structures": "## Структуры и параметры",
|
||
"guides": "## Руководства и концепции",
|
||
}
|
||
by_subdir: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
||
for subdir, fname, title in sorted(entries, key=lambda x: x[2]):
|
||
by_subdir[subdir].append((fname, title))
|
||
|
||
for subdir, header in categories.items():
|
||
if subdir not in by_subdir:
|
||
continue
|
||
lines.append(header)
|
||
for fname, title in by_subdir[subdir]:
|
||
lines.append(f"- [{title}](./{subdir}/{fname})")
|
||
lines.append("")
|
||
|
||
(OUT_DIR / "index.md").write_text("\n".join(lines), encoding="utf-8")
|
||
print(f" index.md: {len(entries)} ссылок")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|