Allow setting an OPENAI_ENDPOINT

[plandex-ai]

Allow setting an OPENAI_ENDPOINT

[richard-schwab]

I currently am using GPT4ALL and Jan which both support api access. I'd love to try this with those.

[danenania]

Thanks @michaellatman -- I agree. Adding more flexibility here is a priority.

[richard-schwab]

Here is my code to send a prompt to Jan in case that is useful for you to see:

import requests
import json
from datetime import datetime
from tqdm import tqdm

class JanPrompt:
    def __init__(self, ip_address, port, agent_message, command_message):
        self.ip_address = ip_address
        self.port = port
        self.agent_message = agent_message
        self.command_message = command_message
        self.url = f"http://{self.ip_address}:{self.port}/v1/chat/completions"

    def compose_payload(self):
        payload = {
            "messages": [
                {"content": self.agent_message, "role": "system"},
                {"content": self.command_message, "role": "user"}
            ],
            "model": "mistral-ins-7b-q4",
            "stream": True,
            "max_tokens": 2048,
            "stop": ["hello"],
            "frequency_penalty": 0,
            "presence_penalty": 0,
            "temperature": 0.7,
            "top_p": 0.95
        }
        return payload

    def send_prompt(self):
        headers = {'Content-Type': 'application/json'}
        payload = self.compose_payload()
        response = requests.post(self.url, headers=headers, data=json.dumps(payload), stream=True)
        return self.handle_stream(response)

    def handle_stream(self, response):
        final_data = ""
        content_pieces = []
        progress_bar = tqdm(unit="chunk", desc="Receiving data")

        try:
            for line in response.iter_lines():
                if line:
                    decoded_line = line.decode('utf-8')
                    progress_bar.update(1)
                    if decoded_line == "data: [DONE]":
                        break
                    data_json = json.loads(decoded_line.replace("data: ", ""))
                    if "choices" in data_json:
                        for choice in data_json["choices"]:
                            if "delta" in choice and "content" in choice["delta"]:
                                content_pieces.append(choice["delta"]["content"])
        except requests.exceptions.RequestException as e:
            print(f"Request failed: {e}")
        finally:
            progress_bar.close()

        final_data = "".join(content_pieces)
        return final_data

    def save_to_file(self, response_data):
        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
        filename = f"prompt_result_{timestamp}.md"
        with open(filename, 'w') as file:
            file.write(f"## Agent Message\n\n{self.agent_message}\n\n## Command Message\n\n{self.command_message}\n\n## Response\n\n{response_data}")
        print(f"Saved to {filename}")

    def run(self):
        response_data = self.send_prompt()
        self.save_to_file(response_data)

if __name__ == "__main__":
    ip_address = "localhost"  
    port = 1337  # Replace with the actual port number
    agent_message = "You are a helpful assistant."  
    command_message = "Please give me a recipe for steamed hams."  
    jan_prompt = JanPrompt(ip_address, port, agent_message, command_message)
    jan_prompt.run()

[danenania]

Thanks @richard-schwab. Unfortunately as with ollama it looks like Jan doesn't support the tools parameter for function-calling, which Plandex relies on.

[mudler]

👍 I was immediately looking into trying this with https://github.com/go-skynet/LocalAI - would be great if we can set up an OPENAI_ENDPOINT

[danenania]

Fixed by #46 and just released in server/0.8.3 and cli/0.8.2