Tool and RAG is working and displaying results

src/ketr-chat/package-lock.json

@@ -21,6 +21,7 @@
         "@types/node": "^16.18.126",
         "@types/react": "^19.0.12",
         "@types/react-dom": "^19.0.4",
+        "mui-markdown": "^1.2.6",
         "react": "^19.0.0",
         "react-dom": "^19.0.0",
         "react-markdown": "^10.1.0",
@@ -4626,6 +4627,12 @@
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.3.tgz",
       "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA=="
     },
+    "node_modules/@types/prismjs": {
+      "version": "1.26.5",
+      "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.5.tgz",
+      "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==",
+      "optional": true
+    },
     "node_modules/@types/prop-types": {
       "version": "15.7.14",
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz",
@@ -13453,6 +13460,18 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/markdown-to-jsx": {
+      "version": "7.7.4",
+      "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.7.4.tgz",
+      "integrity": "sha512-1bSfXyBKi+EYS3YY+e0Csuxf8oZ3decdfhOav/Z7Wrk89tjudyL5FOmwZQUoy0/qVXGUl+6Q3s2SWtpDEWITfQ==",
+      "peer": true,
+      "engines": {
+        "node": ">= 10"
+      },
+      "peerDependencies": {
+        "react": ">= 0.14.0"
+      }
+    },
     "node_modules/math-intrinsics": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@ -14517,6 +14536,22 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
+    "node_modules/mui-markdown": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/mui-markdown/-/mui-markdown-1.2.6.tgz",
+      "integrity": "sha512-GR3+CVLDS4eSri8d4QFkcrBtdBNUCRbWeyXuf3v/t0qZoWDta1U5Wm/MHsYsWSR4HBy6BVoyxlZ0fMNGOCScyQ==",
+      "optionalDependencies": {
+        "prism-react-renderer": "^2.0.3"
+      },
+      "peerDependencies": {
+        "@emotion/react": "^11.10.8",
+        "@emotion/styled": "^11.10.8",
+        "@mui/material": ">= 5.12.2",
+        "markdown-to-jsx": "^7.3.0",
+        "react": ">= 17.0.2",
+        "react-dom": ">= 17.0.2"
+      }
+    },
     "node_modules/multicast-dns": {
       "version": "7.2.5",
       "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz",
@@ -16752,6 +16787,19 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
+    "node_modules/prism-react-renderer": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz",
+      "integrity": "sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig==",
+      "optional": true,
+      "dependencies": {
+        "@types/prismjs": "^1.26.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.0.0"
+      }
+    },
     "node_modules/probe-image-size": {
       "version": "7.2.3",
       "resolved": "https://registry.npmjs.org/probe-image-size/-/probe-image-size-7.2.3.tgz",

src/ketr-chat/package.json

@@ -16,6 +16,7 @@
     "@types/node": "^16.18.126",
     "@types/react": "^19.0.12",
     "@types/react-dom": "^19.0.4",
+    "mui-markdown": "^1.2.6",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "react-markdown": "^10.1.0",

src/ketr-chat/src/App.css

@@ -77,11 +77,10 @@ div {
   height: 100%;
 }
 
-.user-message {
+.user-message.MuiCard-root {
   background-color: #DCF8C6;
   border: 1px solid #B2E0A7;
   color: #333333;
-  padding: 0.5rem;
   margin-bottom: 0.75rem;
   margin-left: 1rem;
   border-radius: 0.25rem;
@@ -95,23 +94,40 @@ div {
   flex-direction: column;
   align-items: self-end;
   align-self: end;
+  flex-grow: 0;
 }
 
-.assistant-message {
+.assistant-message.MuiCard-root {
-  background-color: #FFFFFF;
   border: 1px solid #E0E0E0;
+  background-color: #FFFFFF;
   color: #333333;
-  padding: 0.5rem;
   margin-bottom: 0.75rem;
   margin-right: 1rem;
   min-width: 70%;
   border-radius: 0.25rem;
   justify-self: left;
   display: flex;
-  /* white-space: pre-wrap; */
+  white-space: pre-wrap;
   overflow-wrap: break-word;
   word-break: break-word;
   flex-direction: column;
+  flex-grow: 0;
+  padding: 16px 0;
+  font-size: 0.9rem;
+}
+
+.assistant-message .MuiCardContent-root {
+  padding: 0 16px !important;
+  font-size: 0.9rem;
+}
+
+.assistant-message span {
+  font-size: 0.9rem;
+}
+
+.user-message .MuiCardContent-root:last-child,
+.assistant-message .MuiCardContent-root:last-child {
+  padding: 16px;
 }
 
 .users > div {
@@ -129,42 +145,50 @@ div {
 }
 
 /* Reduce general whitespace in markdown content */
-.markdown-content p {
+.assistant-message p.MuiTypography-root {
   margin-top: 0.5rem;
   margin-bottom: 0.5rem;
+  font-size: 0.9rem;
 }
 
 /* Reduce space between headings and content */
-.markdown-content h1,
+.assistant-message h1.MuiTypography-root,
-.markdown-content h2,
+.assistant-message h2.MuiTypography-root,
-.markdown-content h3,
+.assistant-message h3.MuiTypography-root,
-.markdown-content h4,
+.assistant-message h4.MuiTypography-root,
-.markdown-content h5,
+.assistant-message h5.MuiTypography-root,
-.markdown-content h6 {
+.assistant-message h6.MuiTypography-root {
   margin-top: 1rem;
   margin-bottom: 0.5rem;
+  font-size: 1rem;
 }
 
 /* Reduce space in lists */
-.markdown-content ul,
+.assistant-message ul.MuiTypography-root,
-.markdown-content ol {
+.assistant-message ol.MuiTypography-root {
   margin-top: 0.5rem;
   margin-bottom: 0.5rem;
-  padding-left: 1.5rem;
+  font-size: 0.9rem;
 }
 
-.markdown-content li {
+.assistant-message li.MuiTypography-root {
   margin-bottom: 0.25rem;
+  font-size: 0.9rem;
 }
 
-/* Reduce space between list items */
+.assistant-message .MuiTypography-root li {
-.markdown-content li p {
   margin-top: 0;
   margin-bottom: 0;
+  padding: 0;
+  font-size: 0.9rem;
 }
 
 /* Reduce space around code blocks */
-.markdown-content pre {
+.assistant-message .MuiTypography-root pre {
-  margin-top: 0.5rem;
+  border: 1px solid #F5F5F5;
-  margin-bottom: 0.5rem;
+  border-radius: 0.5rem;
+  padding: 0.5rem 0.75rem;
+  margin-top: 0;
+  margin-bottom: 0;
+  font-size: 0.9rem;
 }

src/ketr-chat/src/App.tsx

@@ -2,6 +2,7 @@ import React, { useState, useEffect, useRef, useCallback, ReactElement } from 'r
 import FormGroup from '@mui/material/FormGroup';
 import FormControlLabel from '@mui/material/FormControlLabel';
 import { useTheme } from '@mui/material';
+import { styled } from '@mui/material/styles';
 import Switch from '@mui/material/Switch';
 import Divider from '@mui/material/Divider';
 import Tooltip from '@mui/material/Tooltip';
@@ -13,21 +14,29 @@ import AccordionActions from '@mui/material/AccordionActions';
 import AccordionSummary from '@mui/material/AccordionSummary';
 import AccordionDetails from '@mui/material/AccordionDetails';
 import Typography from '@mui/material/Typography';
-import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
 import Button from '@mui/material/Button';
 import AppBar from '@mui/material/AppBar';
 import Drawer from '@mui/material/Drawer';
 import Toolbar from '@mui/material/Toolbar';
 import MenuIcon from '@mui/icons-material/Menu';
 import SettingsIcon from '@mui/icons-material/Settings';
-import IconButton from '@mui/material/IconButton';
+import IconButton, { IconButtonProps } from '@mui/material/IconButton';
 import Box from '@mui/material/Box';
 import CssBaseline from '@mui/material/CssBaseline';
 import AddIcon from '@mui/icons-material/AddCircle';
 import SendIcon from '@mui/icons-material/Send';
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import MoreVertIcon from '@mui/icons-material/MoreVert';
+import Card from '@mui/material/Card';
+import CardHeader from '@mui/material/CardHeader';
+import CardMedia from '@mui/material/CardMedia';
+import CardContent from '@mui/material/CardContent';
+import CardActions from '@mui/material/CardActions';
+import Collapse from '@mui/material/Collapse';
 
 import PropagateLoader  from "react-spinners/PropagateLoader";
-import Markdown from 'react-markdown';
+// import Markdown from 'react-markdown';
+import { MuiMarkdown as Markdown } from "mui-markdown";
 import './App.css';
 import rehypeKatex from 'rehype-katex'
 import remarkMath from 'remark-math'
@@ -38,19 +47,26 @@ import '@fontsource/roboto/400.css';
 import '@fontsource/roboto/500.css';
 import '@fontsource/roboto/700.css';
 
-const welcomeMessage = { "role": "assistant", "content": "Welcome to Ketr-Chat. I have real-time access to a lot of information. Ask things like 'What are the headlines from cnn.com?' or 'What is the weather in Portland, OR?'" };
+const welcomeMarkdown = `
+# Welcome to Ketr-Chat.
+
+This system has real-time access to weather, stocks, the current time, and can answer questions about the contents of a website.
+
+**NOTE**: As of right now, the LLM model being used is refusing to use enabled tools when RAG is enabled to provide context.
+So, in order to use the real-time information, you need to click the Settings ![settings](settings.png) icon, open RAG, and disable JPK: ![disable JPK](disable-jpk.png).
+
+Ask things like:
+  * What are the headlines from CNBC?
+  * What is the weather in Portland, OR?
+  * What is James Ketrenos' work history?
+  * What are the stock value of the most traded companies?
+`;
+
+const welcomeMessage = {
+  "role": "assistant", "content": welcomeMarkdown
+};
 const loadingMessage = { "role": "assistant", "content": "Instancing chat session..." };
 
-//const url: string = "https://ai.ketrenos.com"
-
-const getConnectionBase = (loc: any): string => {
-  if (!loc.host.match(/.*battle-linux.*/)) {
-    return loc.protocol + "//" + loc.host;
-  } else {
-    return loc.protocol + "//battle-linux.ketrenos.com:5000";
-  }
-}
-
 type Tool = {
   type: string,
   function?: {
@@ -88,6 +104,32 @@ type SystemInfo = {
   "CPU": string
 };
 
+type MessageMetadata = {
+  rag: any,
+  tools: any[]
+};
+
+type MessageData = {
+  role: string,
+  content: string,
+  user?: string,
+  type?: string,
+  id?: string,
+  isProcessing?: boolean,
+  metadata?: MessageMetadata
+};
+
+type MessageList = MessageData[];
+
+
+const getConnectionBase = (loc: any): string => {
+  if (!loc.host.match(/.*battle-linux.*/)) {
+    return loc.protocol + "//" + loc.host;
+  } else {
+    return loc.protocol + "//battle-linux.ketrenos.com:5000";
+  }
+}
+
 const SystemInfoComponent: React.FC<{ systemInfo: SystemInfo }> = ({ systemInfo }) => {
   const [systemElements, setSystemElements] = useState<ReactElement[]>([]);
 
@@ -236,6 +278,130 @@ const Controls = ({ tools, rags, systemPrompt, toggleTool, toggleRag, setSystemP
   </div>);
 }
 
+interface ExpandMoreProps extends IconButtonProps {
+  expand: boolean;
+}
+
+const ExpandMore = styled((props: ExpandMoreProps) => {
+  const { expand, ...other } = props;
+  return <IconButton {...other} />;
+})(({ theme }) => ({
+  marginLeft: 'auto',
+  transition: theme.transitions.create('transform', {
+    duration: theme.transitions.duration.shortest,
+  }),
+  variants: [
+    {
+      props: ({ expand }) => !expand,
+      style: {
+        transform: 'rotate(0deg)',
+      },
+    },
+    {
+      props: ({ expand }) => !!expand,
+      style: {
+        transform: 'rotate(180deg)',
+      },
+    },
+  ],
+}));
+
+interface MessageInterface {
+  message: MessageData
+};
+
+interface MessageMetaInterface {
+  metadata: MessageMetadata
+}
+const MessageMeta = ({ metadata }: MessageMetaInterface) => {
+  if (metadata === undefined) {
+    return <></>
+  }
+
+  console.log(JSON.stringify(metadata.tools[0].result, null, 2));
+
+  return (<>
+    {
+      metadata.tools !== undefined &&
+      <Typography sx={{ marginBottom: 2 }}>
+        <p>Tools queried:</p>
+        {metadata.tools.map((tool: any, index: number) => <>
+          <Divider />
+          <Box sx={{ fontSize: "0.75rem", display: "flex", flexDirection: "column", mb: 0.5, mt: 0.5 }} key={index}>
+            <div style={{ display: "flex", flexDirection: "column", paddingRight: "1rem", minWidth: "10rem" }}>
+              <div style={{ whiteSpace: "nowrap" }}>{tool.tool}</div>
+              <div style={{ whiteSpace: "nowrap" }}>Result Len: {JSON.stringify(tool.result).length}</div>
+            </div>
+            <div style={{ display: "flex", padding: "3px", whiteSpace: "pre-wrap", flexGrow: 1, border: "1px solid #E0E0E0", maxHeight: "5rem", overflow: "auto" }}>{JSON.stringify(tool.result, null, 2)}</div>
+          </Box>
+        </>)}
+      </Typography>
+    }
+    {
+      metadata.rag.name !== undefined &&
+      <Typography sx={{ marginBottom: 2 }}>
+        <p>RAG from '{metadata.rag.name}' collection matches against embedding vector of {metadata.rag.query_embedding.length} dimensions:</p>
+        {metadata.rag.ids.map((id: number, index: number) => <>
+          <Divider />
+          <Box sx={{ fontSize: "0.75rem", display: "flex", flexDirection: "row", mb: 0.5, mt: 0.5 }} key={index}>
+            <div style={{ display: "flex", flexDirection: "column", paddingRight: "1rem", minWidth: "10rem" }}>
+              <div style={{ whiteSpace: "nowrap" }}>Doc ID: {metadata.rag.ids[index]}</div>
+              <div style={{ whiteSpace: "nowrap" }}>Similarity: {Math.round(metadata.rag.distances[index] * 100) / 100}</div>
+              <div style={{ whiteSpace: "nowrap" }}>Type: {metadata.rag.metadatas[index].doc_type}</div>
+              <div style={{ whiteSpace: "nowrap" }}>Chunk Len: {metadata.rag.documents[index].length}</div>
+            </div>
+            <div style={{ display: "flex", padding: "3px", flexGrow: 1, border: "1px solid #E0E0E0", maxHeight: "5rem", overflow: "auto" }}>{metadata.rag.documents[index]}</div>
+          </Box>
+        </>
+        )}
+      </Typography >
+    }
+  </>
+  );
+};
+
+const Message = ({ message }: MessageInterface) => {
+  const [expanded, setExpanded] = React.useState(false);
+
+  const handleExpandClick = () => {
+    setExpanded(!expanded);
+  };
+
+  const formattedContent = message.content.trim();
+
+  return (
+    <Card sx={{ flexGrow: 1, pb: message.metadata ? 0 : "8px" }} className={(message.role === 'user' ? 'user-message' : 'assistant-message')}>
+      <CardContent>
+        {message.role === 'assistant' ?
+          <Markdown children={formattedContent} />
+          :
+          <Typography variant="body2" sx={{ color: 'text.secondary' }}>
+            {message.content}
+          </Typography>
+        }
+      </CardContent>
+      {message.metadata && <>
+        <CardActions disableSpacing>
+          <Typography sx={{ color: "darkgrey", p: 1, textAlign: "end", flexGrow: 1 }}>LLM information for this query</Typography>
+          <ExpandMore
+            expand={expanded}
+            onClick={handleExpandClick}
+            aria-expanded={expanded}
+            aria-label="show more"
+          >
+            <ExpandMoreIcon />
+          </ExpandMore>
+        </CardActions>
+        <Collapse in={expanded} timeout="auto" unmountOnExit>
+          <CardContent>
+            <MessageMeta metadata={message.metadata} />
+          </CardContent>
+        </Collapse>
+      </>}
+    </Card>
+  );
+}
+
 const App = () => {
   const [query, setQuery] = useState('');
   const [conversation, setConversation] = useState<MessageList>([]);
@@ -256,8 +422,9 @@ const App = () => {
 
   // Scroll to bottom of conversation when conversation updates
   useEffect(() => {
-    if (conversationRef.current) {
+    const queryElement = document.getElementById('QueryInput');
-      conversationRef.current.scrollTop = conversationRef.current.scrollHeight;
+    if (queryElement) {
+      queryElement.scrollIntoView();
     }
   }, [conversation]);
 
@@ -575,22 +742,6 @@ const App = () => {
     }
   };
 
-  type MessageMetadata = {
-    title: string
-  };
-
-  type Message = {
-    role: string,
-    content: string,
-    user?: string,
-    type?: string,
-    id?: string,
-    isProcessing?: boolean,
-    metadata?: MessageMetadata
-  };
-
-  type MessageList = Message[];
-
   const onNew = async () => {
     reset(["history"], "New chat started.");
   }
@@ -739,11 +890,13 @@ const App = () => {
     setSnackOpen(false);
   };
 
+  const Offset = styled('div')(({ theme }) => theme.mixins.toolbar);
+
   return (
     <Box sx={{ display: 'flex', flexDirection: 'column', height: '100dvh' }}>
       <CssBaseline />
       <AppBar
-        position="static"
+        position="fixed"
         sx={{
           zIndex: (theme) => theme.zIndex.drawer + 1,
         }}
@@ -778,6 +931,8 @@ const App = () => {
         </Toolbar>
       </AppBar>
 
+      <Offset />
+
       <Box sx={{ display: "flex", flexGrow: 1, flexDirection: "column" }}>
         <Box
           component="nav"
@@ -804,45 +959,9 @@ const App = () => {
             {drawer}
           </Drawer>
         </Box>
-        <Box component="main" sx={{ flexGrow: 1, overflow: 'auto' }} className="ChatBox">
+        <Box component="main" sx={{ flexGrow: 1, overflow: 'auto' }} className="ChatBox" ref={conversationRef}>
-          <Box className="Conversation" sx={{ flexGrow: 2, p: 1 }} ref={conversationRef}>
+          <Box className="Conversation" sx={{ flexGrow: 2, p: 1 }}>
-            {conversation.map((message, index) => {
+            {conversation.map((message, index) => <Message key={index} message={message} />)}
-              const formattedContent = message.content.trim();
-
-              return (
-                <div key={index} className={message.role === 'user' ? 'user-message' : 'assistant-message'}>
-                  {message.metadata ? (
-                    <>
-                      <div className="metadata">{message.metadata.title}</div>
-                      {message.user && (
-                        <div>{message.user}</div>
-                      )}
-                      {message.role === 'assistant' ? (
-                        <div className="markdown-content">
-                          <Markdown children={formattedContent} />
-                          {/* <Markdown remarkPlugins={[remarkMath]} rehypePlugins={[rehypeKatex]} children={formattedContent} /> */}
-                        </div>
-                      ) : (
-                        <div>{formattedContent}</div>
-                      )}
-                    </>
-                  ) : (
-                    <>
-                      {message.user && (
-                        <div>{message.user}</div>
-                      )}
-                      {message.role === 'assistant' ? (
-                        <div className="markdown-content">
-                          <Markdown remarkPlugins={[remarkMath]} rehypePlugins={[rehypeKatex]} children={formattedContent} />
-                        </div>
-                      ) : (
-                        <div>{formattedContent}</div>
-                      )}
-                    </>
-                  )}
-                </div>
-              );
-            })}
             <div style={{ justifyContent: "center", display: "flex", paddingBottom: "0.5rem" }}>
               <PropagateLoader
                 size="10px"

src/server.py

@@ -55,10 +55,9 @@ from fastapi.middleware.cors import CORSMiddleware
 from utils import rag as Rag
 
 from tools import (
-    get_tool_alias,
+    DateTime, 
-    get_weather_by_location,
+    WeatherForecast, 
-    get_current_datetime,
+    TickerValue,
-    get_ticker_price,
     tools
 )
 
@@ -147,17 +146,18 @@ WEB_PORT=5000
 # %%
 # Globals
 system_message = f"""
-You are a helpful information agent.
+Launched on {DateTime()}.
-You have real time access to any website or URL the user asks about, to stock prices, the current date and time, and current weather information for locations in the United States. 
+
-You are running { { 'model': MODEL_NAME, 'gpu': 'Intel Arc B580', 'cpu': 'Intel Core i9-14900KS', 'ram': '64G' } }.
+When answering queries, follow these steps:
-You were launched on {get_current_datetime()}.
+
-If you use any real time access, do not mention your knowledge cutoff. 
+1. First analyze the query to determine if real-time information might be helpful
-Give short, courteous answers, no more than 2-3 sentences.
+2. Even when [CONTEXT] is provided, consider whether the tools would provide more current or comprehensive information
-Always be accurate. If you don't know the answer, say so. Do not make up details.
+3. Use the provided tools whenever they would enhance your response, regardless of whether context is also available
-When you receive a response from summarize_site, you must:
+4. When both [CONTEXT] and tool outputs are relevant, synthesize information from both sources to provide the most complete answer
-1. Review the entire content returned by the second LLM
+5. Always prioritize the most up-to-date and relevant information, whether it comes from [CONTEXT] or tools
-2. Provide the URL used to obtain the information.
+6. If [CONTEXT] and tool outputs contain conflicting information, prefer the tool outputs as they likely represent more current data
-3. Incorporate the information into your response as appropriate
+
+Always use tools and [CONTEXT] when possible. Be concise, and never make up information. If you do not know the answer, say so.
 """.strip()
 
 tool_log = []
@@ -282,46 +282,6 @@ def split_paragraph_with_hyphenation(text, line_length=80, language='en_US'):
     
     return result_lines
 
-# %%
-async def handle_tool_calls(message):
-    response = []
-    tools_used = []
-    for tool_call in message['tool_calls']:
-        arguments = tool_call['function']['arguments']
-        tool = tool_call['function']['name']
-        match tool:
-            case 'get_ticker_price':
-                ticker = arguments.get('ticker')
-                if not ticker:
-                    ret = None
-                else:
-                    ret = get_ticker_price(ticker)
-                tools_used.append(f"{get_tool_alias(tool)}({ticker})")
-            case 'summarize_site':
-                url = arguments.get('url');
-                question = arguments.get('question', 'what is the summary of this content?')
-                ret = await summarize_site(url, question)
-                tools_used.append(f"{get_tool_alias(tool)}('{url}', '{question}')")
-            case 'get_current_datetime':
-                tz = arguments.get('timezone')
-                ret = get_current_datetime(tz)
-                tools_used.append(f"{get_tool_alias(tool)}('{tz}')")
-            case 'get_weather_by_location':
-                city = arguments.get('city')
-                state = arguments.get('state')
-                ret = get_weather_by_location(city, state)
-                tools_used.append(f"{get_tool_alias(tool)}('{city}', '{state}')")
-            case _:
-                ret = None
-        response.append({
-            "role": "tool",
-            "content": str(ret),
-            "name": tool_call['function']['name']
-        })
-    if len(response) == 1:
-        return response[0], tools_used
-    else:
-        return response, tools_used
 
 # %%
 def total_json_length(dict_array):
@@ -332,7 +292,7 @@ def total_json_length(dict_array):
         total += len(json_string)
     return total
 
-async def summarize_site(url, question):
+async def AnalyzeSite(url, question):
     """
     Fetches content from a URL, extracts the text, and uses Ollama to summarize it.
     
@@ -386,7 +346,7 @@ async def summarize_site(url, question):
         return {
             'source': 'summarizer-llm',
             'content': response['response'],
-            'metadata': get_current_datetime()
+            'metadata': DateTime()
         }
         
     except requests.exceptions.RequestException as e:
@@ -410,6 +370,69 @@ def default_tools(tools):
 def llm_tools(tools):
     return [tool for tool in tools if tool.get("enabled", False) == True]
 
+# %%
+async def handle_tool_calls(message):
+    """
+    Process tool calls and yield status updates along the way.
+    The last yielded item will be a tuple containing (tool_result, tools_used).
+    """
+    tools_used = []
+    all_responses = []
+
+    for i, tool_call in enumerate(message['tool_calls']):
+        arguments = tool_call['function']['arguments']
+        tool = tool_call['function']['name']
+        
+        # Yield status update before processing each tool
+        yield {"status": "processing", "message": f"Processing tool {i+1}/{len(message['tool_calls'])}: {tool}..."}
+        
+        # Process the tool based on its type
+        match tool:
+            case 'TickerValue':
+                ticker = arguments.get('ticker')
+                if not ticker:
+                    ret = None
+                else:
+                    ret = TickerValue(ticker)
+                tools_used.append({ "tool": f"{tool}({ticker})", "result": ret})
+            
+            case 'AnalyzeSite':
+                url = arguments.get('url')
+                question = arguments.get('question', 'what is the summary of this content?')
+                
+                # Additional status update for long-running operations
+                yield {"status": "processing", "message": f"Retrieving and summarizing content from {url}..."}
+                ret = await AnalyzeSite(url, question)
+                tools_used.append({ "tool": f"{tool}('{url}', '{question}')", "result": ret })
+            
+            case 'DateTime':
+                tz = arguments.get('timezone')
+                ret = DateTime(tz)
+                tools_used.append({ "tool": f"{tool}('{tz}')", "result": ret })
+            
+            case 'WeatherForecast':
+                city = arguments.get('city')
+                state = arguments.get('state')
+                
+                yield {"status": "processing", "message": f"Fetching weather data for {city}, {state}..."}
+                ret = WeatherForecast(city, state)
+                tools_used.append({ "tool": f"{tool}('{city}', '{state}')", "result": ret })
+            
+            case _:
+                ret = None
+        
+        # Build response for this tool
+        tool_response = {
+            "role": "tool",
+            "content": str(ret),
+            "name": tool_call['function']['name']
+        }
+        all_responses.append(tool_response)
+    
+    # Yield the final result as the last item
+    final_result = all_responses[0] if len(all_responses) == 1 else all_responses
+    yield (final_result, tools_used)
+
 # %%
 class WebServer:
     def __init__(self, logging, client, collection, model=MODEL_NAME):
@@ -460,8 +483,9 @@ class WebServer:
                             context["tools"] = default_tools(tools)
                             response["tools"] = context["tools"]
                         case "history":
-                            context["history"] = []
+                            context["llm_history"] = []
-                            response["history"] = context["history"]
+                            context["user_history"] = []
+                            response["history"] = []
                 if not response:
                     return JSONResponse({ "error": "Usage: { reset: rags|tools|history|system-prompt}"})
                 else:
@@ -530,7 +554,7 @@ class WebServer:
         @self.app.get('/api/history/{context_id}')
         async def get_history(context_id: str):
             context = self.upsert_context(context_id)
-            return JSONResponse(context["ragless_history"])
+            return JSONResponse(context["user_history"])
 
         @self.app.get('/api/tools/{context_id}')
         async def get_tools(context_id: str):
@@ -648,8 +672,8 @@ class WebServer:
         context = {
             "id": context_id,
             "system": [{"role": "system", "content": system_message}],
-            "history": [],
+            "llm_history": [],
-            "ragless_history": [],
+            "user_history": [],
             "tools": default_tools(tools),
             "rags": rags.copy()
         }
@@ -679,31 +703,33 @@ class WebServer:
 
         self.processing = True
         
-        history = context["history"]
+        llm_history = context["llm_history"]
-        ragless_history = context["ragless_history"]
+        user_history = context["user_history"]
-
+        metadata = {
-        rag_used = []
+            "rag": {},
+            "tools": []
+        }
         rag_docs = []
         for rag in context["rags"]:
             if rag["enabled"] and rag["name"] == "JPK": # Only support JPK rag right now...
                 yield {"status": "processing", "message": f"Checking RAG context {rag['name']}..."}
-                matches = Rag.find_similar(llm=self.client, collection=self.collection, query=content, top_k=10)
+                chroma_results = Rag.find_similar(llm=self.client, collection=self.collection, query=content, top_k=10)
-                if len(matches):
+                if chroma_results:
-                    rag_used.append(rag['name'])
+                    rag_docs.extend(chroma_results["documents"])
-                    rag_docs.extend(matches)
+                    metadata["rag"] = { "name": rag["name"], **chroma_results }
-
         preamble = ""
         if len(rag_docs):
-            preamble = "Context:\n"
+            preamble = "In addition to real-time tools, use the following context to answer the question:\n[CONTEXT]:\n"
             for doc in rag_docs:
                 preamble += doc
-            preamble += "\nHuman: "
+            preamble += "\n[/CONTEXT]\nHuman: "
 
         # Figure 
-        history.append({"role": "user", "content": preamble + content})
+        llm_history.append({"role": "user", "content": preamble + content})
-        ragless_history.append({"role": "user", "content": content})
+        user_history.append({"role": "user", "content": content})
+
+        messages = context["system"] + llm_history[-1:]
 
-        messages = context["system"] + history[-1:]
 
         try:
             yield {"status": "processing", "message": "Processing request..."}
@@ -718,7 +744,16 @@ class WebServer:
                 yield {"status": "processing", "message": "Processing tool calls..."}
                 
                 message = response['message']
-                tool_result, tools_used = await handle_tool_calls(message)
+                tool_result = None
+                
+                # Process all yielded items from the handler
+                async for item in handle_tool_calls(message):
+                    if isinstance(item, tuple) and len(item) == 2:
+                        # This is the final result tuple (tool_result, tools_used)
+                        tool_result, tools_used = item
+                    else:
+                        # This is a status update, forward it
+                        yield item
                 
                 message_dict = {
                     'role': message.get('role', 'assistant'),
@@ -737,24 +772,23 @@ class WebServer:
                 else:
                     messages.append(tool_result)
 
+                metadata["tools"] = tools_used
+
                 yield {"status": "processing", "message": "Generating final response..."}
                 response = self.client.chat(model=self.model, messages=messages, stream=False)
 
             reply = response['message']['content']
-            if len(tools_used):
+            final_message = {"role": "assistant", "content": reply }
-                final_message = {"role": "assistant", "content": reply, 'metadata': {"title": f"🛠️ Tool(s) used: {','.join(tools_used)}"}}
-            else:
-                final_message = {"role": "assistant", "content": reply}
-            if len(rag_used):
-                if "metadata" in final_message:
-                    final_message["metadata"]["title"] += f"🔍 RAG(s) used: {','.join(rag_used)}"
-                else:
-                    final_message["metadata"] = { "title": f"🔍 RAG(s) used: {','.join(rag_used)}" }
 
-            history.append(final_message)
+            # history is provided to the LLM and should not have additional metadata
-            ragless_history.append(final_message)
+            llm_history.append(final_message)
+            final_message["metadata"] = metadata
 
-            yield {"status": "done", "message": final_message}
+            # user_history is provided to the REST API and does not include CONTEXT or metadata
+            user_history.append(final_message)
+
+            # Return the REST API with metadata
+            yield {"status": "done", "message": final_message, "metadata": metadata}
 
         except Exception as e:
             logging.exception({ 'model': self.model, 'messages': messages, 'error': str(e) })

src/tools.py

@@ -41,7 +41,7 @@ import requests
 import yfinance as yf
 
 # %%
-def get_weather_by_location(city, state, country="USA"):
+def WeatherForecast(city, state, country="USA"):
     """
     Get weather information from weather.gov based on city, state, and country.
 
@@ -174,7 +174,7 @@ def do_weather():
     country = input("Enter country (default USA): ") or "USA"
     
     print(f"Getting weather for {city}, {state}, {country}...")
-    weather_data = get_weather_by_location(city, state, country)
+    weather_data = WeatherForecast(city, state, country)
     
     if "error" in weather_data:
         print(f"Error: {weather_data['error']}")
@@ -194,7 +194,7 @@ def do_weather():
 # %%
 
 # Stock related function
-def get_ticker_price(ticker_symbols):
+def TickerValue(ticker_symbols):
     """
     Look up the current price of a stock using its ticker symbol.
     
@@ -205,7 +205,7 @@ def get_ticker_price(ticker_symbols):
         dict: Current stock information including price
     """
     results = []
-    print(f"get_ticker_price('{ticker_symbols}')")
+    print(f"TickerValue('{ticker_symbols}')")
     for ticker_symbol in ticker_symbols.split(','):
         ticker_symbol = ticker_symbol.strip()
         if ticker_symbol == "":
@@ -243,7 +243,7 @@ def get_ticker_price(ticker_symbols):
 
 
 # %%
-def get_current_datetime(timezone="America/Los_Angeles"):
+def DateTime(timezone="America/Los_Angeles"):
     """
     Returns the current date and time in the specified timezone in ISO 8601 format.
     
@@ -274,7 +274,7 @@ def get_current_datetime(timezone="America/Los_Angeles"):
 tools = [ {
     "type": "function", 
     "function": {
-        "name": "get_ticker_price",
+        "name": "TickerValue",
         "description": "Get the current stock price of one or more ticker symbols. Returns an array of objects with 'symbol' and 'price' fields. Call this whenever you need to know the latest value of stock ticker symbols, for example when a user asks 'How much is Intel trading at?' or 'What are the prices of AAPL and MSFT?'",
         "parameters": {
             "type": "object",
@@ -291,8 +291,8 @@ tools = [ {
 }, {
     "type": "function", 
     "function": {
-        "name": "summarize_site",
+        "name": "AnalyzeSite",
-        "description": "Requests a second LLM agent to download the requested site and answer a question about the site. For example if the user says 'What are the top headlines on cnn.com?' you would use summarize_site to get the answer.",
+        "description": "Downloads the requested site and asks a second LLM agent to answer the question based on the site content. For example if the user says 'What are the top headlines on cnn.com?' you would use AnalyzeSite to get the answer. Only use this if the user asks about a specific site or company.",
         "parameters": {
             "type": "object",
             "properties": {
@@ -329,8 +329,8 @@ tools = [ {
 }, {
     "type": "function", 
     "function": {
-        "name": "get_current_datetime",
+        "name": "DateTime",
-        "description": "Get the current date and time in a specified timezone. For example if a user asks 'What time is it in Poland?' you would pass the Warsaw timezone to get_current_datetime.",
+        "description": "Get the current date and time in a specified timezone. For example if a user asks 'What time is it in Poland?' you would pass the Warsaw timezone to DateTime.",
         "parameters": {
             "type": "object",
             "properties": {
@@ -345,7 +345,7 @@ tools = [ {
 }, {
     "type": "function", 
     "function": {
-        "name": "get_weather_by_location",
+        "name": "WeatherForecast",
         "description": "Get the full weather forecast as structured data for a given CITY and STATE location in the United States. For example, if the user asks 'What is the weather in Portland?' or 'What is the forecast for tomorrow?' use the provided data to answer the question.",
         "parameters": {
             "type": "object",
@@ -367,4 +367,4 @@ tools = [ {
     }
 }]
 
-__all__ = [ 'tools', 'get_current_datetime', 'get_weather_by_location', 'get_ticker_price' ]
+__all__ = [ 'tools', 'DateTime', 'WeatherForecast', 'TickerValue' ]

src/utils/rag.py

@@ -68,23 +68,33 @@ def add_embeddings_to_collection(llm, collection, chunks):
       # If input is a Document, extract the text content
       if isinstance(text_or_doc, Document):
           text = text_or_doc.page_content
+          metadata = text_or_doc.metadata
       else:
           text = text_or_doc  # Assume it's already a string
+          metadata = { "index": i }
 
       embedding = get_embedding(llm, text)
       collection.add(
           ids=[str(i)], 
           documents=[text], 
-          embeddings=[embedding]
+          embeddings=[embedding],
+          metadatas=[metadata]
       )
 
 def find_similar(llm, collection, query, top_k=3):
     query_embedding = get_embedding(llm, query)
     results = collection.query(
         query_embeddings=[query_embedding], 
-        n_results=top_k
+        n_results=top_k,
+        include=["documents", "metadatas", "distances"]
     )
-    return results["documents"][0]  # List of top_k matching documents
+    return {
+       "query_embedding": query_embedding,
+       "ids": results["ids"][0],
+       "documents": results["documents"][0],
+       "distances": results["distances"][0],
+       "metadatas": results["metadatas"][0],
+    }
 
 def create_chunks_from_documents(docs):
   text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)