Performance Suggestion / Benchmarks

[su77ungr]

Performance Suggestion / Benchmarks

[alxspiker]

Script used for benchmarking:
Requires llama-cpp-python==0.1.49

import json
import argparse

from llama_cpp import Llama

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", type=str, default="./newggjt.bin")
args = parser.parse_args()

llm = Llama(model_path=args.model, n_threads=6)

stream = llm(
    "Question: What are the names of the planets in the solar system? Answer: ",
    max_tokens=48,
    stop=["Q:", "\n"],
    stream=True,
)

for output in stream:
    print(output["choices"][0]["text"], end="")
    #print(json.dumps(output, indent=2))