diff --git a/schema/dataset.py b/schema/dataset.py new file mode 100644 index 0000000..c60ffed --- /dev/null +++ b/schema/dataset.py @@ -0,0 +1,28 @@ +from typing import Optional +from pydantic import BaseModel, Field +from datetime import datetime, timezone + +class doc(BaseModel): + id: Optional[int] = Field(default=None, description="文档ID") + name: str = Field(default="", description="文档名称") + path: str = Field(default="", description="文档路径") + markdown_files: list[str] = Field(default_factory=list, description="文档路径列表") + +class Q_A(BaseModel): + question: str = Field(default="", min_length=1,description="问题") + answer: str = Field(default="", min_length=1, description="答案") + +class dataset_item(BaseModel): + id: Optional[int] = Field(default=None, description="数据集项ID") + message: list[Q_A] = Field(description="数据集项内容") + +class dataset(BaseModel): + id: Optional[int] = Field(default=None, description="数据集ID") + name: Optional[str] = Field(default=None, description="数据集名称") + model_id: Optional[list[str]] = Field(default=None, description="数据集使用的模型ID") + description: Optional[str] = Field(default="", description="数据集描述") + created_at: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc), + description="记录创建时间" + ) + dataset_items: list[dataset_item] = Field(default_factory=list, description="数据集项列表") \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..1c1e383 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,2 @@ +from .parse_markdown import parse_markdown +from .scan_doc_dir import * \ No newline at end of file diff --git a/tools/parse_markdown.py b/tools/parse_markdown.py index 6eeba01..92dae9e 100644 --- a/tools/parse_markdown.py +++ b/tools/parse_markdown.py @@ -6,6 +6,27 @@ from pathlib import Path sys.path.append(str(Path(__file__).resolve().parent.parent)) from schema import MarkdownNode +def process_markdown_file(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + root = parse_markdown(content) + results = [] + + def traverse(node, parent_titles): + current_titles = parent_titles.copy() + current_titles.append(node.title) + + if not node.children: # 叶子节点 + if node.content: + full_text = ' -> '.join(current_titles) + '\n' + node.content + results.append(full_text) + else: + for child in node.children: + traverse(child, current_titles) + + traverse(root, []) + return results def add_child(parent, child): parent.children.append(child) @@ -60,10 +81,13 @@ def parse_markdown(markdown): return root if __name__=="__main__": - # 从文件读取 Markdown 内容 - with open("workdir/example.md", "r", encoding="utf-8") as f: - markdown = f.read() + # # 从文件读取 Markdown 内容 + # with open("workdir/example.md", "r", encoding="utf-8") as f: + # markdown = f.read() - # 解析 Markdown 并打印树结构 - root = parse_markdown(markdown) - print_tree(root) + # # 解析 Markdown 并打印树结构 + # root = parse_markdown(markdown) + # print_tree(root) + for i in process_markdown_file("workdir/example.md"): + print("~"*20) + print(i) \ No newline at end of file diff --git a/tools/scan_doc_dir.py b/tools/scan_doc_dir.py new file mode 100644 index 0000000..bd17d1b --- /dev/null +++ b/tools/scan_doc_dir.py @@ -0,0 +1,32 @@ +import sys +import os +from pathlib import Path + +# 添加项目根目录到sys.path +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from schema import doc + +def scan_docs_directory(workdir: str): + docs_dir = os.path.join(workdir, "docs") + + doc_list = os.listdir(docs_dir) + + to_return = [] + + for doc_name in doc_list: + doc_path = os.path.join(docs_dir, doc_name) + if os.path.isdir(doc_path): + markdown_files = [] + for root, dirs, files in os.walk(doc_path): + for file in files: + if file.endswith(".md"): + markdown_files.append(os.path.join(root, file)) + to_return.append(doc(name=doc_name, path=doc_path, markdown_files=markdown_files)) + + return to_return + +# 添加测试代码 +if __name__ == "__main__": + workdir = os.path.join(os.path.dirname(__file__), "..", "workdir") + docs = scan_docs_directory(workdir) + print(docs)