// 从 Git 仓库下载模型文件// HuggingFacegit lfs installgit clone https://huggingface.co/THUDM/chatglm3-6b
// ModelScopegit lfs installgit clone https://www.modelscope.cn/ZhipuAI/chatglm3-6b.git

cd /Users/yaolu/AGI/github/chatglm.cpp
# 先初始化 git 仓库git submodule update --init --recursive
# 构建可执行文件cmake -B buildcmake --build build -j
# 安装 Python 依赖pip install .

python ./chatglm_cpp/convert.py -i /Users/yaolu/AGI/huggingface/THUDM/chatglm3-6b -t q8_0 -o chatglm3-ggml-q8.bin

./build/bin/main -m chatglm3-ggml-q8.bin -i

import asyncioimport loggingimport timefrom typing import List, Literal, Optional, Union
import chatglm_cppfrom fastapi import FastAPI, HTTPException, status, Dependsfrom fastapi.middleware.cors import CORSMiddlewarefrom pydantic import BaseModel, Field#, computed_field#from pydantic_settings import BaseSettingsfrom sse_starlette.sse import EventSourceResponse
from sentence_transformers import SentenceTransformerfrom sklearn.preprocessing import PolynomialFeaturesimport numpy as npimport tiktoken
logging.basicConfig(level=logging.INFO, format=r"%(asctime)s - %(module)s - %(levelname)s - %(message)s")

class Settings(object):    model: str = "/Users/yaolu/AGI/github/chatglm.cpp/chatglm3-ggml-q8.bin"    num_threads: int = 0

class ChatMessage(BaseModel):    role: Literal["system", "user", "assistant"]    content: str

class DeltaMessage(BaseModel):    role: Optional[Literal["system", "user", "assistant"]] = None    content: Optional[str] = None

class ChatCompletionRequest(BaseModel):    model: str = "default-model"    messages: List[ChatMessage]    temperature: float = Field(default=0.95, ge=0.0, le=2.0)    top_p: float = Field(default=0.7, ge=0.0, le=1.0)    stream: bool = False    max_tokens: int = Field(default=2048, ge=0)
    model_config = {        "json_schema_extra": {"examples": [{"model": "default-model", "messages": [{"role": "user", "content": "你好"}]}]}    }

class ChatCompletionResponseChoice(BaseModel):    index: int = 0    message: ChatMessage    finish_reason: Literal["stop", "length"] = "stop"

class ChatCompletionResponseStreamChoice(BaseModel):    index: int = 0    delta: DeltaMessage    finish_reason: Optional[Literal["stop", "length"]] = None

class ChatCompletionUsage(BaseModel):    prompt_tokens: int    completion_tokens: int
    #@computed_field    @property    def total_tokens(self) -> int:        return self.prompt_tokens + self.completion_tokens

class ChatCompletionResponse(BaseModel):    id: str = "chatcmpl"    model: str = "default-model"    object: Literal["chat.completion", "chat.completion.chunk"]    created: int = Field(default_factory=lambda: int(time.time()))    choices: Union[List[ChatCompletionResponseChoice], List[ChatCompletionResponseStreamChoice]]    usage: Optional[ChatCompletionUsage] = None
    model_config = {        "json_schema_extra": {            "examples": [                {                    "id": "chatcmpl",                    "model": "default-model",                    "object": "chat.completion",                    "created": 1691166146,                    "choices": [                        {                            "index": 0,                            "message": {"role": "assistant", "content": "你好👋！我是人工智能助手 ChatGLM2-6B，很高兴见到你，欢迎问我任何问题。"},                            "finish_reason": "stop",                        }                    ],                    "usage": {"prompt_tokens": 17, "completion_tokens": 29, "total_tokens": 46},                }            ]        }    }

settings = Settings()app = FastAPI()app.add_middleware(    CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])pipeline = chatglm_cpp.Pipeline(settings.model)lock = asyncio.Lock()
embeddings_model = SentenceTransformer('/Users/yaolu/AGI/huggingface/moka-ai/m3e-base', device='cpu')

def stream_chat(history, body):    yield ChatCompletionResponse(        object="chat.completion.chunk",        choices=[ChatCompletionResponseStreamChoice(delta=DeltaMessage(role="assistant"))],    )
    for piece in pipeline.chat(        history,        max_length=body.max_tokens,        do_sample=body.temperature > 0,        top_p=body.top_p,        temperature=body.temperature,        num_threads=settings.num_threads,        stream=True,    ):        yield ChatCompletionResponse(            object="chat.completion.chunk",            choices=[ChatCompletionResponseStreamChoice(delta=DeltaMessage(content=piece))],        )
    yield ChatCompletionResponse(        object="chat.completion.chunk",        choices=[ChatCompletionResponseStreamChoice(delta=DeltaMessage(), finish_reason="stop")],    )

async def stream_chat_event_publisher(history, body):    output = ""    try:        async with lock:            for chunk in stream_chat(history, body):                await asyncio.sleep(0)  # yield control back to event loop for cancellation check                output += chunk.choices[0].delta.content or ""                yield chunk.model_dump_json(exclude_unset=True)        logging.info(f'prompt: "{history[-1]}", stream response: "{output}"')    except asyncio.CancelledError as e:        logging.info(f'prompt: "{history[-1]}", stream response (partial): "{output}"')        raise e

@app.post("/v1/chat/completions")async def create_chat_completion(body: ChatCompletionRequest) -> ChatCompletionResponse:    # ignore system messages    history = [msg.content for msg in body.messages if msg.role != "system"]    if len(history) % 2 != 1:        raise HTTPException(status.HTTP_400_BAD_REQUEST, "invalid history size")
    if body.stream:        generator = stream_chat_event_publisher(history, body)        return EventSourceResponse(generator)
    max_context_length = 512    output = pipeline.chat(        history=history,        max_length=body.max_tokens,        max_context_length=max_context_length,        do_sample=body.temperature > 0,        top_p=body.top_p,        temperature=body.temperature,    )    logging.info(f'prompt: "{history[-1]}", sync response: "{output}"')    prompt_tokens = len(pipeline.tokenizer.encode_history(history, max_context_length))    completion_tokens = len(pipeline.tokenizer.encode(output, body.max_tokens))
    return ChatCompletionResponse(        object="chat.completion",        choices=[ChatCompletionResponseChoice(message=ChatMessage(role="assistant", content=output))],        usage=ChatCompletionUsage(prompt_tokens=prompt_tokens, completion_tokens=completion_tokens),    )

class EmbeddingRequest(BaseModel):    input: List[str]    model: str
class EmbeddingResponse(BaseModel):    data: list    model: str    object: str    usage: dict
def num_tokens_from_string(string: str) -> int:    """Returns the number of tokens in a text string."""    encoding = tiktoken.get_encoding('cl100k_base')    num_tokens = len(encoding.encode(string))    return num_tokens
def expand_features(embedding, target_length):    poly = PolynomialFeatures(degree=2)    expanded_embedding = poly.fit_transform(embedding.reshape(1, -1))    expanded_embedding = expanded_embedding.flatten()    if len(expanded_embedding) > target_length:        # 如果扩展后的特征超过目标长度，可以通过截断或其他方法来减少维度        expanded_embedding = expanded_embedding[:target_length]    elif len(expanded_embedding) < target_length:        # 如果扩展后的特征少于目标长度，可以通过填充或其他方法来增加维度        expanded_embedding = np.pad(expanded_embedding, (0, target_length - len(expanded_embedding)))    return expanded_embedding

@app.post("/v1/embeddings", response_model=EmbeddingResponse)async def get_embeddings(request: EmbeddingRequest):

    # 计算嵌入向量和tokens数量     embeddings = [embeddings_model.encode(text) for text in request.input]
    # 如果嵌入向量的维度不为1536，则使用插值法扩展至1536维度     embeddings = [expand_features(embedding, 1536) if len(embedding) < 1536 else embedding for embedding in embeddings]
    # Min-Max normalization    embeddings = [embedding / np.linalg.norm(embedding) for embedding in embeddings]
    # 将numpy数组转换为列表    embeddings = [embedding.tolist() for embedding in embeddings]    prompt_tokens = sum(len(text.split()) for text in request.input)    total_tokens = sum(num_tokens_from_string(text) for text in request.input)

    response = {        "data": [            {                "embedding": embedding,                "index": index,                "object": "embedding"            } for index, embedding in enumerate(embeddings)        ],        "model": request.model,        "object": "list",        "usage": {            "prompt_tokens": prompt_tokens,            "total_tokens": total_tokens,        }    }
    return response


class ModelCard(BaseModel):    id: str    object: Literal["model"] = "model"    owned_by: str = "owner"    permission: List = []

class ModelList(BaseModel):    object: Literal["list"] = "list"    data: List[ModelCard] = []
    model_config = {        "json_schema_extra": {            "examples": [                {                    "object": "list",                    "data": [{"id": "gpt-3.5-turbo", "object": "model", "owned_by": "owner", "permission": []}],                }            ]        }    }

@app.get("/v1/models")async def list_models() -> ModelList:    return ModelList(data=[ModelCard(id="gpt-3.5-turbo")])

uvicorn chatglm_cpp.openai_api:app --host 127.0.0.1 --port 8000

accelerate==0.24.1aiofiles==23.2.1aiohttp==3.8.6aiosignal==1.3.1altair==5.1.2annotated-types==0.6.0anyio==3.7.1async-timeout==4.0.3attrs==23.1.0blinker==1.7.0cachetools==5.3.2certifi==2023.7.22charset-normalizer==3.3.2click==8.1.7contourpy==1.2.0cpm-kernels==1.0.11cycler==0.12.1fastapi==0.103.2ffmpy==0.3.1filelock==3.13.1fonttools==4.44.0frozenlist==1.4.0fsspec==2023.10.0gitdb==4.0.11GitPython==3.1.40gradio==3.50.2gradio_client==0.6.1h11==0.14.0httpcore==1.0.2httpx==0.25.1huggingface-hub==0.19.1idna==3.4importlib-metadata==6.8.0importlib-resources==6.1.1Jinja2==3.1.2joblib==1.3.2jsonschema==4.19.2jsonschema-specifications==2023.7.1kiwisolver==1.4.5latex2mathml==3.76.0linkify-it-py==2.0.2Markdown==3.5.1markdown-it-py==2.2.0MarkupSafe==2.1.3matplotlib==3.8.1mdit-py-plugins==0.3.3mdtex2html==1.2.0mdurl==0.1.2mpmath==1.3.0multidict==6.0.4networkx==3.2.1nltk==3.8.1numpy==1.26.2orjson==3.9.10packaging==23.2pandas==2.1.3Pillow==10.1.0protobuf==4.25.0psutil==5.9.6pyarrow==14.0.1pydantic==2.1.1pydantic_core==2.4.0pydeck==0.8.1b0pydub==0.25.1Pygments==2.16.1pyparsing==3.1.1python-dateutil==2.8.2python-multipart==0.0.6pytz==2023.3.post1PyYAML==6.0.1referencing==0.30.2regex==2023.10.3requests==2.31.0rich==13.6.0rpds-py==0.12.0safetensors==0.4.0scikit-learn==1.3.2scipy==1.11.3semantic-version==2.10.0sentence-transformers==2.2.2sentencepiece==0.1.99six==1.16.0smmap==5.0.1sniffio==1.3.0sse-starlette==1.6.5starlette==0.27.0streamlit==1.28.2sympy==1.12tabulate==0.9.0tenacity==8.2.3threadpoolctl==3.2.0tiktoken==0.5.1tokenizers==0.13.3toml==0.10.2toolz==0.12.0torch==2.1.0torchvision==0.16.0tornado==6.3.3tqdm==4.66.1transformers==4.30.2typing_extensions==4.6.1tzdata==2023.3tzlocal==5.2uc-micro-py==1.0.2urllib3==2.1.0uvicorn==0.24.0.post1validators==0.22.0websockets==11.0.3yarl==1.9.2zipp==3.17.0

git clone https://github.com/songquanpeng/one-api.git
# 构建前端cd one-api/webnpm installnpm run build
# 构建后端cd ..go mod downloadgo build -ldflags "-s -w" -o one-api

go build -ldflags "-s -w" -o one-apiexport SQL_DSN=oneapi:oneapi@tcp(localhost:3306)/oneapi && ./one-api --port 3001 --log-dir ./log

brew install mongodb-communitybrew services start mongodb-community

// 安装 pgvector 前指定 PostgreSQL 位置export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
// 如果 pgvector 认错了 MacOS SDK 的位置，还得帮他软链一个sudo ln -s /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk /Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk// 或者用这个命令export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
// 源码编译安装 makemake install # may need sudo
// 确保插件已安装到 PostgreSQL 目录下cd /Library/PostgreSQL/16/share/postgresql/extension/ls | grep vector

CREATE EXTENSION vector;
SELECT * FROM pg_extension WHERE extname = 'vector';

# 代码根目录下执行，会安装根 package、projects 和 packages 内所有依赖pnpm i# 切换到应用目录cd projects/app# 开发模式运行pnpm dev