Multi-modal with image generation

Dockerfile

@@ -267,7 +267,12 @@ WORKDIR /opt/ollama
 #ENV OLLAMA_VERSION=https://github.com/intel/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250415-ubuntu.tgz
 
 # NOTE: NO longer at github.com/intel -- now at ipex-llm
+
+# This version does not work:
+# ENV OLLAMA_VERSION=https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250429-ubuntu.tgz
+
 ENV OLLAMA_VERSION=https://github.com/ipex-llm/ipex-llm/releases/download/v2.2.0/ollama-ipex-llm-2.2.0-ubuntu.tgz
+
 #ENV OLLAMA_VERSION=https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250429-ubuntu.tgz
 RUN wget -qO - ${OLLAMA_VERSION} | \
     tar --strip-components=1 -C . -xzv 

frontend/src/Components/Message.tsx

@@ -21,7 +21,7 @@ import { SxProps, Theme } from '@mui/material';
 import JsonView from '@uiw/react-json-view';
 
 import { ChatBubble } from './ChatBubble';
-import { StyledMarkdown } from './StyledMarkdown';
+import { StyledMarkdown } from '../NewApp/Components/StyledMarkdown';
 
 import { VectorVisualizer } from './VectorVisualizer';
 import { SetSnackType } from './Snack';
@@ -175,6 +175,7 @@ const MessageMeta = (props: MessageMetaProps) => {
                   <Box sx={{ fontSize: "0.75rem", display: "flex", flexDirection: "column", mt: 1, mb: 1, fontWeight: "bold" }}>
                     {tool.name}
                   </Box>
+                  {tool.content !== "null" &&
                     <JsonView
                       displayDataTypes={false}
                       objectSortKeys={true}
@@ -187,6 +188,8 @@ const MessageMeta = (props: MessageMetaProps) => {
                         }}
                       />
                     </JsonView>
+                  }
+                  {tool.content === "null" && "No response from tool call"}
                 </Box>)
             }
         </AccordionDetails>

frontend/src/NewApp/Components/BackstoryRoutes.tsx

@@ -18,6 +18,7 @@ import { CandidateListingPage } from '../Pages/CandidateListingPage';
 import { JobAnalysisPage } from '../Pages/JobAnalysisPage';
 import { DemoComponent } from "NewApp/Pages/DemoComponent";
 import { GenerateCandidate } from "NewApp/Pages/GenerateCandidate";
+import { ControlsPage } from '../../Pages/ControlsPage';
 
 const DashboardPage = () => (<BetaPage><Typography variant="h4">Dashboard</Typography></BetaPage>);
 const ProfilePage = () => (<BetaPage><Typography variant="h4">Profile</Typography></BetaPage>);
@@ -49,6 +50,7 @@ const getBackstoryDynamicRoutes = (props : BackstoryDynamicRoutesProps, user?: U
     <Route key={`${index++}`} path="/find-a-candidate" element={<CandidateListingPage {...{sessionId, setSnack, submitQuery}} />} />,
     <Route key={`${index++}`} path="/job-analysis" element={<JobAnalysisPage />} />,
     <Route key={`${index++}`} path="/generate-candidate" element={<GenerateCandidate {...{ sessionId, setSnack, submitQuery }} />} />,
+    <Route key={`${index++}`} path="/settings" element={<ControlsPage {...{ sessionId, setSnack, submitQuery }} />} />,
   ];
 
   if (user === undefined || user === null) {

frontend/src/NewApp/Components/CandidateInfo.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Box, Link, Typography, Avatar,Grid, Chip, SxProps } from '@mui/material';
+import { Box, Link, Typography, Avatar, Grid, Chip, SxProps, CardHeader } from '@mui/material';
 import { 
   Card,
   CardContent,
@@ -53,11 +53,7 @@ const CandidateInfo: React.FC<CandidateInfoProps> = (props: CandidateInfoProps)
         ...sx
       }}
     >
-    <CardActionArea 
+      <CardContent sx={{ flexGrow: 1, p: 3, height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'stretch' }}>
-        //onClick={() => setSelectedCandidate(candidate)}
-        sx={{ height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'stretch' }}
-      >
-                  <CardContent sx={{ flexGrow: 1, p: 3 }}>
                     
         <Grid container spacing={2}>
         <Grid 
@@ -115,7 +111,8 @@ const CandidateInfo: React.FC<CandidateInfoProps> = (props: CandidateInfoProps)
               </Box>
             </Box>
 
-            {candidate.rag_content_size !== undefined && candidate.rag_content_size > 0 && <Chip
+              {candidate.rag_content_size !== undefined && candidate.rag_content_size > 0 &&
+                <Chip
               onClick={(event: React.MouseEvent<HTMLDivElement>) => { navigate('/knowledge-explorer'); event.stopPropagation() }}
               label={formatRagSize(candidate.rag_content_size)}
               color="primary"
@@ -143,7 +140,6 @@ const CandidateInfo: React.FC<CandidateInfoProps> = (props: CandidateInfoProps)
 
         </Grid>
       </CardContent>
-                </CardActionArea>
     </Card>
   );
 };

frontend/src/NewApp/Components/Document.tsx

@@ -4,7 +4,7 @@ import { Box, Typography } from '@mui/material';
 import { Message } from '../../Components/Message';
 import { ChatBubble } from '../../Components/ChatBubble';
 import { BackstoryElementProps } from '../../Components/BackstoryTab';
-import { StyledMarkdown } from '../../Components/StyledMarkdown';
+import { StyledMarkdown } from './StyledMarkdown';
 
 interface DocumentProps extends BackstoryElementProps {
   filepath?: string;

frontend/src/NewApp/Components/GenerateImage.tsx

@@ -0,0 +1,150 @@
+import React, { useEffect, useState, useRef, useCallback } from 'react';
+import Avatar from '@mui/material/Avatar';
+import Box from '@mui/material/Box';
+import Tooltip from '@mui/material/Tooltip';
+import Button from '@mui/material/Button';
+import Paper from '@mui/material/Paper';
+import IconButton from '@mui/material/IconButton';
+import CancelIcon from '@mui/icons-material/Cancel';
+import SendIcon from '@mui/icons-material/Send';
+import PropagateLoader from 'react-spinners/PropagateLoader';
+import { CandidateInfo } from '../Components/CandidateInfo';
+import { Query } from '../../Components/ChatQuery'
+import { Quote } from 'NewApp/Components/Quote';
+import { streamQueryResponse, StreamQueryController } from '../Components/streamQueryResponse';
+import { connectionBase } from 'Global';
+import { UserInfo } from '../Components/UserContext';
+import { BackstoryElementProps } from 'Components/BackstoryTab';
+import { BackstoryTextField, BackstoryTextFieldRef } from 'Components/BackstoryTextField';
+import { jsonrepair } from 'jsonrepair';
+import { StyledMarkdown } from 'NewApp/Components/StyledMarkdown';
+import { Scrollable } from 'Components/Scrollable';
+import { Pulse } from 'NewApp/Components/Pulse';
+import { useUser } from '../Components/UserContext';
+
+interface GenerateImageProps extends BackstoryElementProps {
+    prompt: string
+};
+
+const GenerateImage = (props: GenerateImageProps) => {
+    const { user } = useUser();
+    const {sessionId, setSnack, prompt} = props;
+    const [processing, setProcessing] = useState<boolean>(false);
+    const [status, setStatus] = useState<string>('');
+    const [timestamp, setTimestamp] = useState<number>(0);
+    const [image, setImage] = useState<string>('');
+
+    // Only keep refs that are truly necessary
+    const controllerRef = useRef<StreamQueryController>(null);
+
+    // Effect to trigger profile generation when user data is ready
+    useEffect(() => {
+        if (controllerRef.current) {
+            console.log("Controller already active, skipping profile generation");
+            return;
+        }
+        if (!prompt) {
+            return;
+        }
+        setStatus('Starting image generation...');
+        setProcessing(true);
+        const start = Date.now();
+
+        controllerRef.current = streamQueryResponse({
+            query: {
+                prompt: prompt,
+                agent_options: {
+                    username: user?.username,
+                }
+            },
+            type: "image",
+            sessionId,
+            connectionBase,
+            onComplete: (msg) => {
+                switch (msg.status) {
+                    case "partial":
+                    case "done":
+                        if (msg.status === "done") {
+                            if (!msg.response) {
+                                setSnack("Image generation failed", "error");
+                            } else {
+                                setImage(msg.response);
+                            }
+                            setProcessing(false);
+                            controllerRef.current = null;
+                        }
+                        break;
+                    case "error":
+                        console.log(`Error generating profile: ${msg.response} after ${Date.now() - start}`);
+                        setSnack(msg.response || "", "error");
+                        setProcessing(false);
+                        controllerRef.current = null;
+                        break;
+                    default:
+                        const data = JSON.parse(msg.response || '');
+                        if (msg.status !== "heartbeat") {
+                            console.log(data);
+                        }
+                        if (data.timestamp) {
+                            setTimestamp(data.timestamp);
+                        } else {
+                            setTimestamp(Date.now())
+                        }
+                        if (data.message) {
+                            setStatus(data.message);
+                        }
+                        break;
+                }
+            }
+        });
+    }, [user, prompt, sessionId, setSnack]);
+
+    if (!sessionId) {
+        return <></>;
+    }
+
+    return (
+      <Box className="GenerateImage" sx={{ 
+        display: "flex", 
+        flexDirection: "column", 
+        flexGrow: 1, 
+        gap: 1,
+        maxWidth: { xs: '100%', md: '700px', lg: '1024px' },
+        minHeight: "max-content",
+        }}>
+        {image !== '' && <img src={image}/>}
+        { prompt && 
+          <Quote size={processing ? "normal" : "small"} quote={prompt} sx={{ "& *": { color: "#2E2E2E !important" }}}/>
+        }
+        {processing && 
+          <Box sx={{
+              display: "flex",
+              flexDirection: "column",
+              alignItems: "center",
+              justifyContent: "center",
+              m: 0,
+              gap: 1,
+              minHeight: "min-content",
+              mb: 2
+          }}>
+              { status && 
+              <Box sx={{ display: "flex", flexDirection: "column"}}>
+                <Box sx={{ fontSize: "0.5rem"}}>Generation status</Box>
+                <Box sx={{ fontWeight: "bold"}}>{status}</Box>
+              </Box>
+              }
+              <PropagateLoader
+                  size="10px"
+                  loading={processing}
+                  color="white"
+                  aria-label="Loading Spinner"
+                  data-testid="loader"
+              />
+          </Box>
+        }
+        </Box>);
+};
+
+export {
+  GenerateImage
+};

frontend/src/NewApp/Components/Quote.tsx

@@ -1,34 +1,38 @@
-  
-  
-  
-  
 import React from 'react';
-import { Box, Typography, Paper } from '@mui/material';
+import { Box, Typography, Paper, SxProps } from '@mui/material';
 import { styled } from '@mui/material/styles';
 
-const QuoteContainer = styled(Paper)(({ theme }) => ({
+interface QuoteContainerProps {
+  size?: 'normal' | 'small';
+}
+
+const QuoteContainer = styled(Paper, {
+  shouldForwardProp: (prop) => prop !== 'size',
+})<QuoteContainerProps>(({ theme, size = 'normal' }) => ({
   position: 'relative',
-  padding: theme.spacing(4),
+  padding: size === 'small' ? theme.spacing(1) : theme.spacing(4),
-  margin: theme.spacing(2),
+  margin: size === 'small' ? theme.spacing(0.5) : theme.spacing(2),
-  background: 'linear-gradient(135deg, #FFFFFF 0%, #D3CDBF 100%)', // White to Warm Gray
+  background: 'linear-gradient(135deg, #FFFFFF 0%, #D3CDBF 100%)',
-  borderRadius: theme.spacing(2),
+  borderRadius: size === 'small' ? theme.spacing(1) : theme.spacing(2),
-  boxShadow: '0 8px 32px rgba(26, 37, 54, 0.15)', // Midnight Blue shadow
+  boxShadow: '0 8px 32px rgba(26, 37, 54, 0.15)',
   overflow: 'hidden',
-  border: '1px solid rgba(74, 122, 125, 0.2)', // Subtle Dusty Teal border
+  border: '1px solid rgba(74, 122, 125, 0.2)',
   '&::before': {
     content: '""',
     position: 'absolute',
     top: 0,
     left: 0,
     right: 0,
-    height: '4px',
+    height: size === 'small' ? '2px' : '4px',
-    background: 'linear-gradient(90deg, #4A7A7D 0%, #D4A017 100%)', // Dusty Teal to Golden Ochre
+    background: 'linear-gradient(90deg, #4A7A7D 0%, #D4A017 100%)',
-  }
+  },
 }));
 
-const QuoteText = styled(Typography)(({ theme }) => ({
+const QuoteText = styled(Typography, {
-  fontSize: '1.2rem',
+  shouldForwardProp: (prop) => prop !== 'size',
-  lineHeight: 1.6,
+})<QuoteContainerProps>(({ theme, size = 'normal' }) => ({
+  fontSize: size === 'small' ? '0.9rem' : '1.2rem',
+  lineHeight: size === 'small' ? 1.4 : 1.6,
   fontStyle: 'italic',
   color: '#2E2E2E', // Charcoal Black
   position: 'relative',
@@ -38,8 +42,10 @@ const QuoteText = styled(Typography)(({ theme }) => ({
   fontWeight: 400,
 }));
 
-const QuoteMark = styled(Typography)(({ theme }) => ({
+const QuoteMark = styled(Typography, {
-  fontSize: '4rem',
+  shouldForwardProp: (prop) => prop !== 'size',
+})<QuoteContainerProps>(({ theme, size = 'normal' }) => ({
+  fontSize: size === 'small' ? '2.5rem' : '4rem',
   fontFamily: '"Georgia", "Times New Roman", serif',
   fontWeight: 'bold',
   opacity: 0.15,
@@ -49,60 +55,65 @@ const QuoteMark = styled(Typography)(({ theme }) => ({
   userSelect: 'none',
 }));
 
-const OpeningQuote = styled(QuoteMark)({
+const OpeningQuote = styled(QuoteMark)(({ size = 'normal' }: QuoteContainerProps) => ({
-  top: '10px',
+  top: size === 'small' ? '5px' : '10px',
-  left: '15px',
+  left: size === 'small' ? '8px' : '15px',
-});
+}));
 
-const ClosingQuote = styled(QuoteMark)({
+const ClosingQuote = styled(QuoteMark)(({ size = 'normal' }: QuoteContainerProps) => ({
-  bottom: '10px',
+  bottom: size === 'small' ? '5px' : '10px',
-  right: '15px',
+  right: size === 'small' ? '8px' : '15px',
   transform: 'rotate(180deg)',
-});
+}));
 
-const AuthorText = styled(Typography)(({ theme }) => ({
+const AuthorText = styled(Typography, {
-  marginTop: theme.spacing(2),
+  shouldForwardProp: (prop) => prop !== 'size',
+})<QuoteContainerProps>(({ theme, size = 'normal' }) => ({
+  marginTop: size === 'small' ? theme.spacing(1) : theme.spacing(2),
   textAlign: 'right',
   fontStyle: 'normal',
   fontWeight: 500,
   color: '#1A2536', // Midnight Blue
-  fontSize: '0.95rem',
+  fontSize: size === 'small' ? '0.8rem' : '0.95rem',
   '&::before': {
     content: '"— "',
     color: '#D4A017', // Golden Ochre dash
-  }
+  },
 }));
 
-const AccentLine = styled(Box)({
+const AccentLine = styled(Box, {
-  width: '60px',
+  shouldForwardProp: (prop) => prop !== 'size',
-  height: '2px',
+})<QuoteContainerProps>(({ theme, size = 'normal' }) => ({
+  width: size === 'small' ? '40px' : '60px',
+  height: size === 'small' ? '1px' : '2px',
   background: 'linear-gradient(90deg, #D4A017 0%, #4A7A7D 100%)', // Golden Ochre to Dusty Teal
-  margin: '1rem auto',
+  margin: size === 'small' ? '0.5rem auto' : '1rem auto',
   borderRadius: '1px',
-});
+}));
-
 
 interface QuoteProps {
-  quote?: string,
+  quote?: string;
-  author?: string
+  author?: string;
-};
+  size?: 'small' | 'normal';
+  sx?: SxProps;
+}
 
-const Quote = (props : QuoteProps) => {
+const Quote = (props: QuoteProps) => {
-  const { quote, author } = props;
+  const { quote, author, size = 'normal', sx } = props;
   return (
-    <QuoteContainer elevation={0}>
+    <QuoteContainer size={size} elevation={0} sx={sx}>
-      <OpeningQuote>"</OpeningQuote>
+      <OpeningQuote size={size}>"</OpeningQuote>
-      <ClosingQuote>"</ClosingQuote>
+      <ClosingQuote size={size}>"</ClosingQuote>
       
       <Box sx={{ position: 'relative', zIndex: 2 }}>
-        <QuoteText variant="body1">
+        <QuoteText size={size} variant="body1">
           {quote}
         </QuoteText>
         
-        <AccentLine />
+        <AccentLine size={size} />
         
         {author && (
-          <AuthorText variant="body2">
+          <AuthorText size={size} variant="body2">
             {author}
           </AuthorText>
         )}
@@ -111,6 +122,4 @@ const Quote = (props : QuoteProps) => {
   );
 };
 
-export {
+export { Quote };
-  Quote
-};

frontend/src/NewApp/Components/StyledMarkdown.tsx

@@ -2,16 +2,17 @@ import React from 'react';
 import { MuiMarkdown } from 'mui-markdown';
 import { SxProps, useTheme } from '@mui/material/styles';
 import { Link } from '@mui/material';
-import { ChatQuery } from './ChatQuery';
+import { ChatQuery } from '../../Components/ChatQuery';
 import Box from '@mui/material/Box';
 import JsonView from '@uiw/react-json-view';
 import { vscodeTheme } from '@uiw/react-json-view/vscode';
-import { Mermaid } from './Mermaid';
+import { Mermaid } from '../../Components/Mermaid';
-import { Scrollable } from './Scrollable';
+import { Scrollable } from '../../Components/Scrollable';
 import { jsonrepair } from 'jsonrepair';
+import { GenerateImage } from './GenerateImage';
 
 import './StyledMarkdown.css';
-import { BackstoryElementProps } from './BackstoryTab';
+import { BackstoryElementProps } from '../../Components/BackstoryTab';
 
 interface StyledMarkdownProps extends BackstoryElementProps {
   className?: string,
@@ -20,7 +21,7 @@ interface StyledMarkdownProps extends BackstoryElementProps {
 };
 
 const StyledMarkdown: React.FC<StyledMarkdownProps> = (props: StyledMarkdownProps) => {
-  const { className, sessionId, content, submitQuery, sx, streaming } = props;
+  const { className, sessionId, content, submitQuery, sx, streaming, setSnack } = props;
   const theme = useTheme();
 
   const overrides: any = {
@@ -107,8 +108,19 @@ const StyledMarkdown: React.FC<StyledMarkdownProps> = (props: StyledMarkdownProp
           return props.query;
         }
       },
+    },
+      GenerateImage: {
+      component: (props: { prompt: string }) => {
+        const prompt = props.prompt.replace(/(\w+):/g, '"$1":');
+        try {
+          return <GenerateImage prompt={prompt} {...{sessionId, submitQuery, setSnack}}/>
+        } catch (e) {
+          console.log("StyledMarkdown error:", prompt, e);
+          return props.prompt;
         }
-  };
+      },
+    },
+};
 
   return <Box
     className={`MuiMarkdown ${className || ""}`}

frontend/src/NewApp/Components/streamQueryResponse.tsx

@@ -111,7 +111,7 @@ const streamQueryResponse = (options: StreamQueryOptions) => {
             await processLine(line);
           } catch (e) {
             console.error('Error processing line:', e);
-            console.log(line);
+            console.error(line);
           }
         }
       }

frontend/src/NewApp/Pages/GenerateCandidate.tsx

@@ -17,7 +17,7 @@ import { UserInfo } from '../Components/UserContext';
 import { BackstoryElementProps } from 'Components/BackstoryTab';
 import { BackstoryTextField, BackstoryTextFieldRef } from 'Components/BackstoryTextField';
 import { jsonrepair } from 'jsonrepair';
-import { StyledMarkdown } from 'Components/StyledMarkdown';
+import { StyledMarkdown } from 'NewApp/Components/StyledMarkdown';
 import { Scrollable } from 'Components/Scrollable';
 import { Pulse } from 'NewApp/Components/Pulse';
 

src/server.py

@@ -279,6 +279,14 @@ class WebServer:
 
         self.setup_routes()
 
+    def sanitize_input(self, input: str):
+        # Validate input: allow only alphanumeric, underscores, and hyphens
+        if not re.match(r'^[a-zA-Z0-9._-]+$', input): # alphanumeric, _, -, and . are valid
+            raise ValueError("Invalid input format.")
+        if re.match(r'\.\.', input): # two ticks in a row is invalid
+            raise ValueError("Invalid input format.")
+
+
     def setup_routes(self):
         # @self.app.get("/")
         # async def root():
@@ -778,6 +786,30 @@ class WebServer:
                 logger.error(f"get_users error: {str(e)}")
                 return JSONResponse({ "error": "Unable to parse users"}, 500)
 
+        @self.app.get("/api/u/{username}/images/{image_id}/{context_id}")
+        async def get_user_image(username: str, image_id: str, context_id: str, request: Request):
+            logger.info(f"{request.method} {request.url.path}")
+            try:
+                self.sanitize_input(context_id)
+                self.sanitize_input(username)
+                self.sanitize_input(image_id)
+
+                if not User.exists(username):
+                    return JSONResponse({"error": f"User {username} not found."}, status_code=404)
+                context = await self.load_context(context_id)
+                if not context:
+                    return JSONResponse({"error": f"Context {context_id} not found."}, status_code=404)
+                image_path = os.path.join(defines.user_dir, username, "images", image_id)
+                if not os.path.exists(image_path):
+                    return JSONResponse({ "error": "User {username} does not image {image_id}"}, status_code=404)
+                return FileResponse(image_path)
+            except ValueError as e:
+                return JSONResponse({ "error": f"Invalid input: {image_id}" }, 400)
+            except Exception as e:
+                logger.error(traceback.format_exc())
+                logger.error(e)
+                return JSONResponse({ "error": f"Unable to get image {username} {image_id}"}, 500)
+
         @self.app.get("/api/u/{username}/profile/{context_id}")
         async def get_user_profile(username: str, context_id: str, request: Request):
             logger.info(f"{request.method} {request.url.path}")
@@ -792,7 +824,7 @@ class WebServer:
                     return JSONResponse({ "error": "User {username} does not have a profile picture"}, status_code=404)
                 return FileResponse(profile_path)
             except Exception as e:
-                return JSONResponse({ "error": "Unable to load user {username}"}, 500)
+                return JSONResponse({ "error": f"Unable to load user {username}"}, 500)
 
         @self.app.post("/api/u/{username}/{context_id}")
         async def post_user(username: str, context_id: str, request: Request):
@@ -843,7 +875,7 @@ class WebServer:
                 await self.save_context(context_id)
                 return JSONResponse(user_data)
             except Exception as e:
-                return JSONResponse({ "error": "Unable to load user {username}"}, 500)
+                return JSONResponse({ "error": f"Unable to load user {username}"}, 500)
 
         @self.app.post("/api/context/u/{username}")
         async def create_user_context(username: str, request: Request):

src/utils/agents/base.py

@@ -33,7 +33,7 @@ from .types import agent_registry
 from .. import defines
 from ..message import Message, Tunables
 from ..metrics import Metrics
-from ..tools import TickerValue, WeatherForecast, AnalyzeSite, DateTime, llm_tools  # type: ignore -- dynamically added to __all__
+from ..tools import TickerValue, WeatherForecast, AnalyzeSite, GenerateImage, DateTime, llm_tools  # type: ignore -- dynamically added to __all__
 from ..conversation import Conversation
 
 class LLMMessage(BaseModel):
@@ -240,6 +240,22 @@ class Agent(BaseModel, ABC):
                             llm=llm, model=model, url=url, question=question
                         )
 
+                    case "GenerateImage":
+                        prompt = arguments.get("prompt", None)
+                        if not prompt:
+                            logger.info("No prompt supplied to GenerateImage")
+                            ret = { "error": "No prompt supplied to GenerateImage" }
+
+                        # Additional status update for long-running operations
+                        message.response = (
+                            f"Generating image for {prompt}..."
+                        )
+                        yield message
+                        ret = await GenerateImage(
+                            llm=llm, model=model, prompt=prompt
+                        )
+                        logger.info("GenerateImage returning", ret)
+
                     case "DateTime":
                         tz = arguments.get("timezone")
                         ret = DateTime(tz)
@@ -255,6 +271,7 @@ class Agent(BaseModel, ABC):
                         ret = WeatherForecast(city, state)
 
                     case _:
+                        logger.error(f"Requested tool {tool} does not exist")
                         ret = None
 
                 # Build response for this tool

src/utils/agents/chat.py

@@ -24,6 +24,26 @@ When answering queries, follow these steps:
 - If there is information in the <|context|> or <|resume|> sections to enhance the answer, incorporate it seamlessly and refer to it as 'the latest information' or 'recent data' instead of mentioning '<|context|>' (etc.) or quoting it directly.
 - Avoid phrases like 'According to the <|context|>' or similar references to the <|context|> or <|resume|>.
 
+CRITICAL INSTRUCTIONS FOR IMAGE GENERATION:
+
+1. When the user requests to generate an image, inject the following into the response: <GenerateImage prompt="USER-PROMPT"/>. Do this when users request images, drawings, or visual content.
+3. MANDATORY: You must respond with EXACTLY this format: <GenerateImage prompt="{{USER-PROMPT}}"/>
+4. FORBIDDEN: DO NOT use markdown image syntax ![](url) 
+5. FORBIDDEN: DO NOT create fake URLs or file paths
+6. FORBIDDEN: DO NOT use any other image embedding format
+
+CORRECT EXAMPLE:
+User: "Draw a cat"
+Your response: "<GenerateImage prompt='Draw a cat'/>"
+
+WRONG EXAMPLES (DO NOT DO THIS):
+- ![](https://example.com/...)
+- ![Cat image](any_url)
+- <img src="...">
+
+The <GenerateImage prompt="{{USER-PROMPT}}"/> format is the ONLY way to display images in this system. 
+DO NOT make up a URL for an image or provide markdown syntax for embedding an image. Only use <GenerateImage prompt="{{USER-PROMPT}}".
+
 Always use tools, <|resume|>, and <|context|> when possible. Be concise, and never make up information. If you do not know the answer, say so.
 """
 

src/utils/agents/image_generator.py

@@ -21,6 +21,7 @@ import time
 import asyncio
 import time
 import os
+import hashlib
 
 from . base import Agent, agent_registry, LLMMessage
 from .. message import Message
@@ -42,7 +43,7 @@ class ImageGenerator(Agent):
 
     system_prompt: str = "" # No system prompt is used
     username: str
-    filename: str
+    filename: str = ""
 
     llm: Any = Field(default=None, exclude=True)
     model: str = Field(default=None, exclude=True)
@@ -64,45 +65,84 @@ class ImageGenerator(Agent):
         self.llm = llm
         self.model = model
 
-        spinner: List[str] = ["\\", "|", "/", "-"]
+        prompt = message.prompt
+        user_info = os.path.join(defines.user_dir, self.username, defines.user_info_file)
+        with open(user_info, "r") as f:
+            self.user = json.loads(f.read())
+        images = self.user.get("images", None)
+        if not images:
+            self.user["images"] = {}
+            images = self.user["images"]
+            with open(user_info, "w") as f:
+                f.write(json.dumps(self.user))
+
+        if self.filename != "profile.png":
+            # Convert the prompt to a hash using MD5
+            hash_object = hashlib.md5(prompt.encode('utf-8'))
+            # Get the hexadecimal representation of the hash
+            hash_string = hash_object.hexdigest()
+            # Return the filename with the specified extension
+            self.filename = f"{hash_string}.png"
+            file_path = os.path.join(defines.user_dir, self.username, "images")
+            os.makedirs(file_path, exist_ok=True)
+            file_path=os.path.join(file_path, self.filename)
+        else:
+            file_path = os.path.join(defines.user_dir, self.username, self.filename)
+        
+        images[self.filename] = { "status": "thinking", "response": "Generating image" }
+
+        if self.context.processing:
             tick: int = 0
             while self.context.processing:
-            logger.info(
+                with open(user_info, "r") as f:
-                "TODO: Implement delay queing; busy for same agent, otherwise return queue size and estimated wait time"
+                    self.user = json.loads(f.read())
-            )
+                image = self.user["images"].get(self.filename, { "status": "waiting", "response": f"Waiting for image generation slot.{'.' * tick}"})
-            message.status = "waiting"
+                message.status = image["status"]
-            message.response = (
+                message.response = image["response"]
-                f"Busy processing another request. Please wait. {spinner[tick]}"
+                tick = (tick + 1) % 5
-            )
-            tick = (tick + 1) % len(spinner)
                 yield message
                 await asyncio.sleep(1)  # Allow the event loop to process the write
 
+            # Processing of active image is complete. Check if it was this image, and return it if so
+            with open(user_info, "r") as f:
+                self.user = json.loads(f.read())
+            image = self.user["images"].get(self.filename, None)
+            if image:
+                message.status = image["status"]
+                message.response = image["response"]
+                yield message
+                return
+
         self.context.processing = True
 
         try:
             #
             # Generate the profile picture
             #
-            prompt = message.prompt
             message.status = "thinking"
             message.response = f"Generating: {prompt}"
             yield message
 
-            user_info = os.path.join(defines.user_dir, self.username, defines.user_info_file)
-            with open(user_info, "r") as f:
-                self.user = json.loads(f.read())
-
             logger.info("Beginning image generation...", self.user)
-            logger.info("TODO: Add safety checks for filename... actually figure out an entirely different way to figure out where to store them.")
+            if os.path.exists(file_path) and self.filename != "profile.png":               
-            self.filename = "profile.png"
+                logger.info(f"Image already exists: {file_path}")
-            file_path = os.path.join(defines.user_dir, self.user["username"], self.filename)
+                message.status = "done"
+                message.response = f"/api/u/{self.username}/images/{self.filename}"
+                self.user["images"][self.filename] = { "status": message.status, "response": message.response}
+                with open(user_info, "w") as f:
+                    f.write(json.dumps(self.user))
+                yield message
+                return
+            
             logger.info(f"Image generation: {file_path} <- {prompt}")
             request = ImageRequest(filepath=file_path, prompt=prompt)
             async for message in generate_image(
                 message=message,
                 request=request 
             ):
+                self.user["images"][self.filename] = { "status": message.status, "response": message.response}
+                with open(user_info, "w") as f:
+                    f.write(json.dumps(self.user))
                 if message.status != "done":
                     yield message
             logger.info("Image generation done...")
@@ -110,12 +150,13 @@ class ImageGenerator(Agent):
                 logger.info(f"Generated image does not exist: {file_path}")
                 logger.error(f"{message.status} {message.response}")
             else:
-                images = self.user.get("images", [])
-                if self.filename not in images:
-                    images.append(self.filename)
                 if self.filename == "profile.png":
                     self.user["has_profile"] = True
 
+            self.user["images"][self.filename] = { "status": message.status, "response": message.response}
+            with open(user_info, "w") as f:
+                f.write(json.dumps(self.user))
+
             #
             # Write out the completed user information
             #
@@ -124,7 +165,10 @@ class ImageGenerator(Agent):
 
             # Image generated
             message.status = "done"
-            message.response = json.dumps(self.user)
+            if self.filename != "profile.png":
+                message.response = f"/apu/u/{self.username}/images/{self.filename}"
+            else:
+                message.response = f"/api/u/{self.username}/profile"
 
         except Exception as e:
             message.status = "error"
@@ -132,10 +176,16 @@ class ImageGenerator(Agent):
             logger.error(message.response)
             message.response = f"Error in image generation: {str(e)}"
             logger.error(message.response)
+            self.user["images"][self.filename] = { "status": message.status, "response": message.response}
+            with open(user_info, "w") as f:
+                f.write(json.dumps(self.user))
             yield message
             return
 
         # Done processing, add message to conversation
+        self.user["images"][self.filename] = { "status": message.status, "response": message.response}
+        with open(user_info, "w") as f:
+            f.write(json.dumps(self.user))
         self.context.processing = False
         # Return the final message
         yield message

src/utils/profile_image.py

@@ -76,7 +76,7 @@ def flux_worker(pipe: Any, params: ImageRequest, status_queue: queue.Queue, task
             
             status_queue.put({
                 "status": "running",
-                "message": f"Processing step {step+1}/{params.iterations} ({progress}%)",
+                "message": f"Processing step {step+1}/{params.iterations} ({progress}%) complete.",
                 "progress": progress
             })
             return callback_kwargs

src/utils/tools/basetools.py

@@ -279,6 +279,8 @@ def DateTime(timezone="America/Los_Angeles"):
     except Exception as e:
         return {"error": f"Invalid timezone {timezone}: {str(e)}"}
 
+async def GenerateImage(llm, model: str, prompt: str):
+    return { "image_id": "image-a830a83-bd831" }
 
 async def AnalyzeSite(llm, model: str, url: str, question: str):
     """
@@ -346,6 +348,7 @@ async def AnalyzeSite(llm, model: str, url: str, question: str):
         return f"Error processing the website content: {str(e)}"
 
 
+
 # %%
 class Function(BaseModel):
     name: str
@@ -358,6 +361,53 @@ class Tool(BaseModel):
     function: Function
 
 tools : List[Tool] = [
+#     Tool.model_validate({
+#     "type": "function",
+#     "function": {
+#         "name": "GenerateImage", 
+#         "description": """\
+# CRITICAL INSTRUCTIONS FOR IMAGE GENERATION:
+
+# 1. Call this tool when users request images, drawings, or visual content
+# 2. This tool returns an image_id (e.g., "img_abc123")
+# 3. MANDATORY: You must respond with EXACTLY this format: <GenerateImage id={image_id}/>
+# 4. FORBIDDEN: DO NOT use markdown image syntax ![](url) 
+# 5. FORBIDDEN: DO NOT create fake URLs or file paths
+# 6. FORBIDDEN: DO NOT use any other image embedding format
+
+# CORRECT EXAMPLE:
+# User: "Draw a cat"
+# Tool returns: {"image_id": "img_xyz789"}
+# Your response: "Here's your cat image: <GenerateImage id=img_xyz789/>"
+
+# WRONG EXAMPLES (DO NOT DO THIS):
+# - ![](https://example.com/...)
+# - ![Cat image](any_url)
+# - <img src="...">
+
+# The <GenerateImage id={image_id}/> format is the ONLY way to display images in this system.
+# """,
+#         "parameters": {
+#             "type": "object",
+#             "properties": {
+#                 "prompt": {
+#                     "type": "string", 
+#                     "description": "Detailed image description including style, colors, subject, composition"
+#                 }
+#             },
+#             "required": ["prompt"]
+#         },
+#         "returns": {
+#             "type": "object",
+#             "properties": {
+#                 "image_id": {
+#                     "type": "string",
+#                     "description": "Unique identifier for the generated image. Use this EXACTLY in <GenerateImage id={this_value}/>"
+#                 }
+#             }
+#         }
+#     }
+# }),
     Tool.model_validate({
         "type": "function",
         "function": {
@@ -471,6 +521,6 @@ def all_tools() -> List[ToolEntry]:
 def enabled_tools(tools: List[ToolEntry]) -> List[ToolEntry]:
     return [ToolEntry(tool=entry.tool) for entry in tools if entry.enabled == True]
 
-tool_functions = ["DateTime", "WeatherForecast", "TickerValue", "AnalyzeSite"]
+tool_functions = ["DateTime", "WeatherForecast", "TickerValue", "AnalyzeSite", "GenerateImage"]
 __all__ = ["ToolEntry", "all_tools", "llm_tools", "enabled_tools", "tool_functions"]
 # __all__.extend(__tool_functions__) # type: ignore