feat(frontend): 完成了 API Provider 删除和添加了编辑功能的函数

- 新增选中行的全局变量 selected_row
- 实现 select_record 函数来保存选中行数据
- 在表格中添加选中行事件监听
- 优化代码结构，提高可读性和可维护性

.gitignore

@@ -11,6 +11,7 @@ env/
 # IDE
 .vscode/
 .idea/
+.roo
 
 # Environment files
 .env
@@ -27,4 +28,7 @@ Thumbs.db
 workdir/
 
 # cache
-unsloth_compiled_cache
+unsloth_compiled_cache
+
+# 测试代码
+test.ipynb

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 C-a-r-r-y
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,15 @@
+# 基于文档驱动的自适应编码大模型微调框架
+## 简介
+本人的毕业设计
+### 项目概述
+
+* 通过深度解析私有库的文档以及其他资源，生成指令型语料，据此对大语言模型进行针对私有库的微调。
+
+### 项目技术
+
+* 使用unsloth框架在GPU上实现大语言模型的qlora微调
+* 使用langchain框架编写工作流实现批量生成微调语料
+* 使用tinydb和sqlite实现数据的持久化
+* 使用gradio框架实现前端展示
+
+**施工中......**

db/__init__.py

@@ -0,0 +1,3 @@
+from .init_db import get_engine, initialize_sqlite_db
+
+__all__ = ['get_engine', 'initialize_sqlite_db']

db/init_db.py

@@ -0,0 +1,79 @@
+from sqlmodel import SQLModel, create_engine, Session
+from sqlmodel import select
+from typing import Optional
+import os
+from pathlib import Path
+import sys
+from dotenv import load_dotenv
+from sqlalchemy.engine import Engine
+
+# 将项目根目录添加到系统路径中，以便能够导入项目中的其他模块
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+from schema.dataset_generation import APIProvider
+
+# 全局变量，用于存储数据库引擎实例
+_engine: Optional[Engine] = None
+
+def get_engine(workdir: str) -> Engine:
+    """
+    获取数据库引擎实例。如果引擎尚未创建，则创建一个新的引擎并返回。
+
+    Args:
+        workdir (str): 工作目录路径，用于确定数据库文件的存储位置。
+
+    Returns:
+        Engine: SQLAlchemy 数据库引擎实例。
+    """
+    global _engine
+    if not _engine:
+        # 创建数据库目录（如果不存在）
+        db_dir = os.path.join(workdir, "db")
+        os.makedirs(db_dir, exist_ok=True)
+        # 定义数据库文件路径
+        db_path = os.path.join(db_dir, "db.sqlite")
+        # 创建数据库URL
+        db_url = f"sqlite:///{db_path}"
+        # 创建数据库引擎
+        _engine = create_engine(db_url)
+    return _engine
+
+def initialize_sqlite_db(engine: Engine) -> None:
+    """
+    初始化数据库，创建所有表结构，并插入初始数据（如果不存在）。
+
+    Args:
+        engine (Engine): SQLAlchemy 数据库引擎实例。
+    """
+    # 创建所有表结构
+    SQLModel.metadata.create_all(engine)
+    # 加载环境变量
+    load_dotenv()
+    # 从环境变量中获取API相关配置
+    api_key = os.getenv("API_KEY")
+    base_url = os.getenv("BASE_URL")
+    model_id = os.getenv("MODEL_ID")
+    
+    # 如果所有必要的环境变量都存在，则插入初始数据
+    if api_key and base_url and model_id:
+        with Session(engine) as session:
+            # 查询是否已存在APIProvider记录
+            statement = select(APIProvider).limit(1)
+            existing_provider = session.exec(statement).first()
+            
+            # 如果不存在，则插入新的APIProvider记录
+            if not existing_provider:
+                provider = APIProvider(
+                    base_url=base_url,
+                    model_id=model_id,
+                    api_key=api_key
+                )
+                session.add(provider)
+                session.commit()
+
+if __name__ == "__main__":
+    # 定义工作目录路径
+    workdir = os.path.join(os.path.dirname(__file__), "..", "workdir")
+    # 获取数据库引擎
+    engine = get_engine(workdir)
+    # 初始化数据库
+    initialize_sqlite_db(engine)

frontend/__init__.py

@@ -0,0 +1,4 @@
+from .chat_page import chat_page
+from .setting_page import setting_page
+from .train_page import train_page
+from .dataset_page import dataset_page

frontend/chat_page.py

@@ -0,0 +1,9 @@
+import gradio as gr
+
+def chat_page():
+    with gr.Blocks() as demo:
+        gr.Markdown("## 聊天")
+        with gr.Row():
+            with gr.Column():
+                pass
+    return demo

frontend/dataset_page.py

@@ -0,0 +1,9 @@
+import gradio as gr
+
+def dataset_page():
+    with gr.Blocks() as demo:
+        gr.Markdown("## 数据集")
+        with gr.Row():
+            with gr.Column():
+                pass
+    return demo

frontend/setting_page.py

@@ -0,0 +1,114 @@
+import gradio as gr
+from typing import List
+from sqlmodel import Session, select
+from db import get_engine
+from schema import APIProvider
+import os
+from global_var import sql_engine
+
+def setting_page():
+    def get_providers() -> List[List[str]]:
+        selected_row = None
+        try:  # 添加异常处理
+            with Session(sql_engine) as session:
+                providers = session.exec(select(APIProvider)).all()
+                return [
+                    [p.id, p.model_id, p.base_url, p.api_key or ""]
+                    for p in providers
+                ]
+        except Exception as e:
+            raise gr.Error(f"获取数据失败: {str(e)}")
+
+    def add_provider(model_id, base_url, api_key):
+        try:
+            with Session(sql_engine) as session:
+                new_provider = APIProvider(
+                    model_id=model_id if model_id else None,
+                    base_url=base_url if base_url else None,
+                    api_key=api_key if api_key else None
+                )
+                session.add(new_provider)
+                session.commit()
+                session.refresh(new_provider)
+                return get_providers()
+        except Exception as e:
+            raise gr.Error(f"添加失败: {str(e)}")
+
+    def edit_provider(model_id, base_url, api_key):
+        global selected_row
+        print(f"edit_provider Selected row: {selected_row}")
+        return get_providers()
+
+    def delete_provider():
+        global selected_row
+        if not selected_row:
+            raise gr.Error("请先选择要删除的行")
+        try:
+            with Session(sql_engine) as session:
+                provider = session.get(APIProvider, selected_row[0])
+                if not provider:
+                    raise gr.Error("找不到选中的记录")
+                session.delete(provider)
+                session.commit()
+                return get_providers()
+        except Exception as e:
+            raise gr.Error(f"删除失败: {str(e)}")
+        
+    selected_row = None  # 保存当前选中行的全局变量
+    
+    def select_record(evt: gr.SelectData):
+        global selected_row
+        selected_row = evt.row_value
+
+    with gr.Blocks() as demo:
+        gr.Markdown("## API Provider 管理")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                model_id_input = gr.Textbox(label="Model ID")
+                base_url_input = gr.Textbox(label="Base URL")
+                api_key_input = gr.Textbox(label="API Key")
+                add_button = gr.Button("添加新API", variant="primary")
+
+            with gr.Column(scale=3):
+                provider_table = gr.DataFrame(
+                    headers=["id", "model id", "base URL", "API Key"],
+                    datatype=["number", "str", "str", "str"],
+                    interactive=True,
+                    value=get_providers(),
+                    wrap=True,
+                    col_count=(4, "auto")
+                )
+
+                with gr.Row():
+                    refresh_button = gr.Button("刷新数据", variant="secondary")
+                    edit_button = gr.Button("编辑选中行", variant="primary")
+                    delete_button = gr.Button("删除选中行", variant="stop")
+
+        refresh_button.click(
+            fn=get_providers,
+            outputs=[provider_table],
+            queue=False  # 立即刷新不需要排队
+        )
+
+        add_button.click(
+            fn=add_provider,
+            inputs=[model_id_input, base_url_input, api_key_input],
+            outputs=[provider_table]
+        )
+
+        provider_table.select(select_record, [], [], show_progress="hidden")
+
+        edit_button.click(
+            fn=edit_provider,
+            inputs=[],
+            outputs=[provider_table]
+        )
+
+        delete_button.click(
+            fn=delete_provider,
+            inputs=[],
+            outputs=[provider_table]
+        )
+
+        return demo

frontend/train_page.py

@@ -0,0 +1,9 @@
+import gradio as gr
+
+def train_page():
+    with gr.Blocks() as demo:
+        gr.Markdown("## 微调")
+        with gr.Row():
+            with gr.Column():
+                pass
+    return demo

global_var.py

@@ -0,0 +1,3 @@
+from db import get_engine
+
+sql_engine = get_engine("workdir")

main.py

@@ -0,0 +1,22 @@
+import gradio as gr
+from frontend.setting_page import setting_page
+from frontend import chat_page,setting_page,train_page,dataset_page
+from db import initialize_sqlite_db 
+from global_var import sql_engine
+
+if __name__ == "__main__":
+    initialize_sqlite_db(sql_engine)
+
+    with gr.Blocks() as app:
+        gr.Markdown("# 基于文档驱动的自适应编码大模型微调框架")
+        with gr.Tabs():
+            with gr.TabItem("聊天"):
+                chat_page()
+            with gr.TabItem("微调"):
+                train_page()
+            with gr.TabItem("数据集"):
+                dataset_page()
+            with gr.TabItem("设置"):
+                setting_page()
+    
+    app.launch()

requirements.txt

@@ -1,2 +1,5 @@
 openai>=1.0.0
-python-dotenv>=1.0.0
+python-dotenv>=1.0.0
+pydantic>=2.0.0
+gradio>=5.0.0
+langchain>=0.3

schema/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import *
+from .dataset_generation import APIProvider, LLMResponse, LLMRequest
+from .md_doc import MarkdownNode
+from .prompt import promptTempleta

schema/dataset_generation.py

@@ -0,0 +1,51 @@
+from datetime import datetime, timezone
+from typing import Optional
+from sqlmodel import SQLModel, Relationship, Field
+
+class APIProvider(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True,allow_mutation=False)
+    base_url: str = Field(...,min_length=1,description="API的基础URL，不能为空")
+    model_id: str = Field(...,min_length=1,description="API使用的模型ID，不能为空")
+    api_key: Optional[str] = Field(default=None, description="用于身份验证的API密钥")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="记录创建时间"
+    )
+
+class LLMResponse(SQLModel):
+    timestamp: datetime  = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="响应的时间戳"
+    )
+    response_id: str = Field(..., description="响应的唯一ID")
+    tokens_usage: dict = Field(default_factory=lambda: {
+        "prompt_tokens": 0,
+        "completion_tokens": 0,
+        "prompt_cache_hit_tokens": None,
+        "prompt_cache_miss_tokens": None
+    }, description="token使用信息")
+    response_content: dict = Field(default_factory=dict, description="API响应的内容")
+    total_duration: float = Field(default=0.0, description="请求的总时长，单位为秒")
+    llm_parameters: dict = Field(default_factory=lambda: {
+        "temperature": None,
+        "max_tokens": None,
+        "top_p": None,
+        "frequency_penalty": None,
+        "presence_penalty": None,
+        "seed": None
+    }, description="API的生成参数")
+
+class LLMRequest(SQLModel):
+    prompt: str = Field(..., description="发送给API的提示词") 
+    provider_id: int = Field(foreign_key="apiprovider.id")
+    provider: APIProvider = Relationship()
+    format: Optional[str] = Field(default=None, description="API响应的格式")
+    response: list[LLMResponse] = Field(default_factory=list, description="API响应列表")
+    error: Optional[list[str]] = Field(default=None, description="API请求过程中发生的错误信息")
+    total_duration: float = Field(default=0.0, description="请求的总时长，单位为秒")
+    total_tokens_usage: dict = Field(default_factory=lambda: {
+        "prompt_tokens": 0,
+        "completion_tokens": 0,
+        "prompt_cache_hit_tokens": None,
+        "prompt_cache_miss_tokens": None
+    }, description="token使用信息")

schema/md_doc.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional
+
+class MarkdownNode(BaseModel):
+    level: int = Field(default=0, description="节点层级")
+    title: str = Field(default="Root", description="节点标题")
+    content: Optional[str] = Field(default=None, description="节点内容")
+    children: List['MarkdownNode'] = Field(default_factory=list, description="子节点列表")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+MarkdownNode.model_rebuild()

tools/parse_markdown.py

@@ -1,28 +1,24 @@
 import re
+import sys
+from pathlib import Path
 
-class MarkdownNode:
+# 添加项目根目录到sys.path
-    def __init__(self, level=0, title="Root"):
+sys.path.append(str(Path(__file__).resolve().parent.parent))
-        self.level = level
+from schema import MarkdownNode
-        self.title = title
-        self.content = ""  # 使用字符串存储合并后的内容
-        self.children = []
 
-    def __repr__(self):
+def add_child(parent, child):
-        return f"({self.level}) {self.title}"
+    parent.children.append(child)
 
-    def add_child(self, child):
+def print_tree(node, indent=0):
-        self.children.append(child)
+    prefix = "│  " * (indent - 1) + "└─ " if indent > 0 else ""
-
+    print(f"{prefix}{node.title}")
-    def print_tree(self, indent=0):
+    if node.content:
-        prefix = "│  " * (indent - 1) + "└─ " if indent > 0 else ""
+        content_prefix = "│  " * indent + "├─ [内容]"
-        print(f"{prefix}{self.title}")
+        print(content_prefix)
-        if self.content:
+        for line in node.content.split('\n'):
-            content_prefix = "│  " * indent + "├─ [内容]"
+            print("│  " * indent + "│  " + line)
-            print(content_prefix)
+    for child in node.children:
-            for line in self.content.split('\n'):
+        print_tree(child, indent + 1)
-                print("│  " * indent + "│  " + line)
-        for child in self.children:
-            child.print_tree(indent + 1)
 
 def parse_markdown(markdown):
     lines = markdown.split('\n')
@@ -51,10 +47,10 @@ def parse_markdown(markdown):
         if match:
             level = len(match.group(1))
             title = match.group(2)
-            node = MarkdownNode(level, title)
+            node = MarkdownNode(level=level, title=title, content="", children=[])
             while stack[-1].level >= level:
                 stack.pop()
-            stack[-1].add_child(node)
+            add_child(stack[-1], node)
             stack.append(node)
         else:
             if stack[-1].content:
@@ -65,9 +61,9 @@ def parse_markdown(markdown):
 
 if __name__=="__main__":
     # 从文件读取 Markdown 内容
-    with open("example.md", "r", encoding="utf-8") as f:
+    with open("workdir/example.md", "r", encoding="utf-8") as f:
         markdown = f.read()
 
     # 解析 Markdown 并打印树结构
     root = parse_markdown(markdown)
-    root.print_tree()
+    print_tree(root)