backstory/llm-proxy.md

Environment Configuration Examples

==== Development (Ollama only) ====

export OLLAMA_HOST="http://localhost:11434" export DEFAULT_LLM_PROVIDER="ollama"

==== Production with OpenAI ====

export OPENAI_API_KEY="sk-your-openai-key-here" export DEFAULT_LLM_PROVIDER="openai"

==== Production with Anthropic ====

export ANTHROPIC_API_KEY="sk-ant-your-anthropic-key-here" export DEFAULT_LLM_PROVIDER="anthropic"

==== Production with multiple providers ====

export OPENAI_API_KEY="sk-your-openai-key-here" export ANTHROPIC_API_KEY="sk-ant-your-anthropic-key-here" export GEMINI_API_KEY="your-gemini-key-here" export OLLAMA_HOST="http://ollama-server:11434" export DEFAULT_LLM_PROVIDER="openai"

==== Docker Compose Example ====

docker-compose.yml

version: '3.8' services: api: build: . ports: - "8000:8000" environment: - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - DEFAULT_LLM_PROVIDER=openai depends_on: - ollama

ollama: image: ollama/ollama ports: - "11434:11434" volumes: - ollama_data:/root/.ollama environment: - OLLAMA_HOST=0.0.0.0

volumes: ollama_data:

==== Kubernetes ConfigMap ====

apiVersion: v1 kind: ConfigMap metadata: name: llm-config data: DEFAULT_LLM_PROVIDER: "openai" OLLAMA_HOST: "http://ollama-service:11434"

---

apiVersion: v1 kind: Secret metadata: name: llm-secrets type: Opaque stringData: OPENAI_API_KEY: "sk-your-key-here" ANTHROPIC_API_KEY: "sk-ant-your-key-here"

==== Python configuration example ====

config.py

import os from llm_proxy import get_llm, LLMProvider

def configure_llm_for_environment(): """Configure LLM based on deployment environment""" llm = get_llm()

# Development: Use Ollama
if os.getenv('ENVIRONMENT') == 'development':
    llm.configure_provider(LLMProvider.OLLAMA, host='http://localhost:11434')
    llm.set_default_provider(LLMProvider.OLLAMA)

# Staging: Use OpenAI with rate limiting
elif os.getenv('ENVIRONMENT') == 'staging':
    llm.configure_provider(LLMProvider.OPENAI, 
                          api_key=os.getenv('OPENAI_API_KEY'),
                          max_retries=3,
                          timeout=30)
    llm.set_default_provider(LLMProvider.OPENAI)

# Production: Use multiple providers with fallback
elif os.getenv('ENVIRONMENT') == 'production':
    # Primary: Anthropic
    llm.configure_provider(LLMProvider.ANTHROPIC,
                          api_key=os.getenv('ANTHROPIC_API_KEY'))
    
    # Fallback: OpenAI
    llm.configure_provider(LLMProvider.OPENAI,
                          api_key=os.getenv('OPENAI_API_KEY'))
    
    # Set primary provider
    llm.set_default_provider(LLMProvider.ANTHROPIC)

==== Usage Examples ====

Example 1: Basic usage with default provider

async def basic_example(): llm = get_llm()

response = await llm.chat(
    model="gpt-4",
    messages=[{"role": "user", "content": "Hello!"}]
)
print(response.content)

Example 2: Specify provider explicitly

async def provider_specific_example(): llm = get_llm()

# Use OpenAI specifically
response = await llm.chat(
    model="gpt-4",
    messages=[{"role": "user", "content": "Hello!"}],
    provider=LLMProvider.OPENAI
)

# Use Anthropic specifically
response2 = await llm.chat(
    model="claude-3-sonnet-20240229",
    messages=[{"role": "user", "content": "Hello!"}],
    provider=LLMProvider.ANTHROPIC
)

Example 3: Streaming with provider fallback

async def streaming_with_fallback(): llm = get_llm()

try:
    async for chunk in llm.chat(
        model="claude-3-sonnet-20240229",
        messages=[{"role": "user", "content": "Write a story"}],
        provider=LLMProvider.ANTHROPIC,
        stream=True
    ):
        print(chunk.content, end='', flush=True)
except Exception as e:
    print(f"Primary provider failed: {e}")
    # Fallback to OpenAI
    async for chunk in llm.chat(
        model="gpt-4",
        messages=[{"role": "user", "content": "Write a story"}],
        provider=LLMProvider.OPENAI,
        stream=True
    ):
        print(chunk.content, end='', flush=True)

Example 4: Load balancing between providers

import random

async def load_balanced_request(): llm = get_llm() available_providers = [LLMProvider.OPENAI, LLMProvider.ANTHROPIC]

# Simple random load balancing
provider = random.choice(available_providers)

# Adjust model based on provider
model_mapping = {
    LLMProvider.OPENAI: "gpt-4",
    LLMProvider.ANTHROPIC: "claude-3-sonnet-20240229"
}

response = await llm.chat(
    model=model_mapping[provider],
    messages=[{"role": "user", "content": "Hello!"}],
    provider=provider
)

print(f"Response from {provider.value}: {response.content}")

==== FastAPI Startup Configuration ====

from fastapi import FastAPI

app = FastAPI()

@app.on_event("startup") async def startup_event(): """Configure LLM providers on application startup""" configure_llm_for_environment()

# Verify configuration
llm = get_llm()
print(f"Configured providers: {[p.value for p in llm._initialized_providers]}")
print(f"Default provider: {llm.default_provider.value}")