Merge pull request ubiquity-os-marketplace#35 from ShivTestOrg/context

fix: Structured Comment Exploration

evals/handlers/setup-context.ts

@@ -1,20 +1,17 @@
 import { SupabaseClient } from "@supabase/supabase-js";
 import { createAdapters } from "../../src/adapters";
-import { CommentSimilaritySearchResult } from "../../src/adapters/supabase/helpers/comment";
-import { IssueSimilaritySearchResult } from "../../src/adapters/supabase/helpers/issues";
 import { fetchRepoLanguageStats, fetchRepoDependencies } from "../../src/handlers/ground-truths/chat-bot";
 import { findGroundTruths } from "../../src/handlers/ground-truths/find-ground-truths";
 import { logger } from "../../src/helpers/errors";
 import { formatChatHistory } from "../../src/helpers/format-chat-history";
-import { recursivelyFetchLinkedIssues } from "../../src/helpers/issue-fetching";
 import { Context } from "../../src/types";
 import { VoyageAIClient } from "voyageai";
 import OpenAI from "openai";
+import { fetchSimilarContent } from "../../src/helpers/issue-fetching";
 
 const SEPERATOR = "######################################################\n";
 
 export interface FetchContext {
-  rerankedText: string[];
   formattedChat: string[];
   groundTruths: string[];
 }
@@ -42,50 +39,69 @@ export const initAdapters = (context: Context, clients: EvalClients): Context =>
 
 export async function fetchContext(context: Context, question: string): Promise<FetchContext> {
   const {
-    config: { similarityThreshold },
+    config: { similarityThreshold, model, maxDepth },
     adapters: {
       supabase: { comment, issue },
       voyage: { reranker },
+      openai: { completions },
     },
   } = context;
-  const { specAndBodies, streamlinedComments } = await recursivelyFetchLinkedIssues({
-    context,
-    owner: context.payload.repository.owner.login,
-    repo: context.payload.repository.name,
-  });
-  let formattedChat = await formatChatHistory(context, streamlinedComments, specAndBodies);
-  logger.info(`${formattedChat.join("")}`);
-  // using db functions to find similar comments and issues
-  const [similarComments, similarIssues] = await Promise.all([
+  // Calculate total available tokens
+  const modelMaxTokens = completions.getModelMaxTokenLimit(model);
+  const maxCompletionTokens = completions.getModelMaxOutputLimit(model);
+  let availableTokens = modelMaxTokens - maxCompletionTokens;
+
+  // Calculate base prompt tokens (system message + query template)
+  const basePromptTokens = await completions.getPromptTokens();
+  availableTokens -= basePromptTokens;
+  logger.debug(`Base prompt tokens: ${basePromptTokens}`);
+
+  // Find similar comments and issues from Supabase
+  const [similarCommentsSearch, similarIssuesSearch] = await Promise.all([
     comment.findSimilarComments(question, 1 - similarityThreshold, ""),
     issue.findSimilarIssues(question, 1 - similarityThreshold, ""),
   ]);
-  // combine the similar comments and issues into a single array
+
+  // Fetch full content for similar items using GitHub API
+  const { similarIssues, similarComments } = await fetchSimilarContent(context, similarIssuesSearch || [], similarCommentsSearch || []);
+
+  logger.debug(`Fetched similar comments: ${JSON.stringify(similarComments)}`);
+  logger.debug(`Fetched similar issues: ${JSON.stringify(similarIssues)}`);
+
+  // Rerank similar content
+  const { similarIssues: rerankedIssues, similarComments: rerankedComments } = await reranker.reRankSimilarContent(question, similarIssues, similarComments);
+
+  // Calculate token usage from reranked content
   const similarText = [
-    ...(similarComments?.map((comment: CommentSimilaritySearchResult) => comment.comment_plaintext) || []),
-    ...(similarIssues?.map((issue: IssueSimilaritySearchResult) => issue.issue_plaintext) || []),
+    ...rerankedComments.map((comment) => comment.body).filter((body): body is string => !!body),
+    ...rerankedIssues.map((issue) => issue.body).filter((body): body is string => !!body),
   ];
-  // filter out any empty strings
-  formattedChat = formattedChat.filter((text) => text);
-  // rerank the similar text using voyageai
-  const rerankedText = similarText.length > 0 ? await reranker.reRankResults(similarText, question) : [];
-  // gather structural data about the payload repository
+  const similarTextTokens = await completions.findTokenLength(similarText.join("\n"));
+  availableTokens -= similarTextTokens;
+  logger.debug(`Similar text tokens: ${similarTextTokens}`);
+
+  // Gather repository data and calculate ground truths
   const [languages, { dependencies, devDependencies }] = await Promise.all([fetchRepoLanguageStats(context), fetchRepoDependencies(context)]);
+
+  // Initialize ground truths
   let groundTruths: string[] = [];
-  if (!languages.length) {
-    groundTruths.push("No languages found in the repository");
-  }
-  if (!Reflect.ownKeys(dependencies).length) {
-    groundTruths.push("No dependencies found in the repository");
-  }
-  if (!Reflect.ownKeys(devDependencies).length) {
-    groundTruths.push("No devDependencies found in the repository");
-  }
-  if (groundTruths.length > 3) {
+  if (!languages.length) groundTruths.push("No languages found in the repository");
+  if (!Reflect.ownKeys(dependencies).length) groundTruths.push("No dependencies found in the repository");
+  if (!Reflect.ownKeys(devDependencies).length) groundTruths.push("No devDependencies found in the repository");
+
+  // If not all empty, get full ground truths
+  if (groundTruths.length !== 3) {
     groundTruths = await findGroundTruths(context, "chat-bot", { languages, dependencies, devDependencies });
   }
+
+  // Calculate ground truths tokens
+  const groundTruthsTokens = await completions.findTokenLength(groundTruths.join("\n"));
+  availableTokens -= groundTruthsTokens;
+  logger.debug(`Ground truths tokens: ${groundTruthsTokens}`);
+
+  // Get formatted chat history with remaining tokens and reranked content
+  const formattedChat = await formatChatHistory(context, maxDepth, rerankedIssues, rerankedComments, availableTokens);
   return {
-    rerankedText,
     formattedChat,
     groundTruths,
   };
@@ -98,12 +114,6 @@ export function formattedHistory(fetchContext: FetchContext): string {
     formattedChat += chat;
   });
   formattedChat += SEPERATOR;
-  //Iterate through the reranked text and add it to the final formatted chat
-  formattedChat += "#################### Reranked Text ####################\n";
-  fetchContext.rerankedText.forEach((reranked) => {
-    formattedChat += reranked;
-  });
-  formattedChat += SEPERATOR;
   //Iterate through the ground truths and add it to the final formatted chat
   formattedChat += "#################### Ground Truths ####################\n";
   fetchContext.groundTruths.forEach((truth) => {

evals/llm.eval.ts

@@ -12,6 +12,9 @@ import { writeFileSync } from "fs";
 import { fetchContext, formattedHistory, initAdapters } from "./handlers/setup-context";
 import { LOG_LEVEL, Logs } from "@ubiquity-os/ubiquity-os-logger";
 
+import { config } from "dotenv";
+config();
+
 // Required environment variables with type assertion
 const requiredEnvVars = {
   OPENAI_API_KEY: process.env.OPENAI_API_KEY as string,
@@ -66,7 +69,6 @@ const inputs = {
   config: {
     model: "gpt-4o",
     similarityThreshold: 0.8,
-    maxTokens: 1000,
   },
   settings: {
     openAiBaseUrl: "https://openrouter.ai/api/v1",
@@ -118,7 +120,7 @@ export async function main() {
               body: scenario.issue.body,
               html_url: scenario.issue.html_url,
               number: scenario.issue.number,
-            } as unknown as Context["payload"]["issue"],
+            } as unknown as Context<"issue_comment.created">["payload"]["issue"],
             sender: scenario.sender,
             repository: {
               name: scenario.repository.name,
@@ -139,20 +141,17 @@ export async function main() {
 
         initialContext = initAdapters(initialContext, clients);
         const chatHistory = await fetchContext(initialContext, scenario.issue.question);
-        const formattedContextHistory = formattedHistory(chatHistory);
         const result = await initialContext.adapters.openai.completions.createCompletion(
           scenario.issue.question,
           initialContext.config.model || "gpt-4o",
-          chatHistory.rerankedText,
           chatHistory.formattedChat,
           chatHistory.groundTruths,
-          initialContext.env.UBIQUITY_OS_APP_NAME,
-          initialContext.config.maxTokens
+          initialContext.env.UBIQUITY_OS_APP_NAME
         );
 
         return {
           output: result.answer,
-          context: formattedContextHistory,
+          context: formattedHistory(chatHistory),
           expected: scenario.expectedResponse,
         };
       },

manifest.json

@@ -1,7 +1,7 @@
 {
   "name": "command-ask",
   "description": "A highly context aware organization integrated chatbot",
-  "ubiquity:listeners": ["issue_comment.created"],
+  "ubiquity:listeners": ["issue_comment.created", "pull_request_review_comment.created"],
   "skipBotEvents": true,
   "commands": {
     "ask": {
@@ -32,8 +32,8 @@
         "default": 0.9,
         "type": "number"
       },
-      "maxTokens": {
-        "default": 10000,
+      "maxDepth": {
+        "default": 3,
         "type": "number"
       }
     }

src/adapters/openai/helpers/completions.ts

@@ -59,32 +59,41 @@ export class Completions extends SuperOpenAi {
     return this.getModelMaxTokenLimit("o1-mini");
   }
 
-  async createCompletion(
-    query: string,
-    model: string = "o1-mini",
-    additionalContext: string[],
-    localContext: string[],
-    groundTruths: string[],
-    botName: string,
-    maxTokens: number
-  ): Promise<CompletionsType> {
-    const numTokens = await this.findTokenLength(query, additionalContext, localContext, groundTruths);
-    logger.info(`Number of tokens: ${numTokens}`);
-
-    const sysMsg = [
+  private _getSystemPromptTemplate(groundTruths: string = "{groundTruths}", botName: string = "{botName}", localContext: string = "{localContext}"): string {
+    return [
       "You Must obey the following ground truths: ",
-      JSON.stringify(groundTruths) + "\n",
+      groundTruths + "\n",
       "You are tasked with assisting as a GitHub bot by generating responses based on provided chat history and similar responses, focusing on using available knowledge within the provided corpus, which may contain code, documentation, or incomplete information. Your role is to interpret and use this knowledge effectively to answer user questions.\n\n# Steps\n\n1. **Understand Context**: Review the chat history and any similar provided responses to understand the context.\n2. **Extract Relevant Information**: Identify key pieces of information, even if they are incomplete, from the available corpus.\n3. **Apply Knowledge**: Use the extracted information and relevant documentation to construct an informed response.\n4. **Draft Response**: Compile the gathered insights into a coherent and concise response, ensuring it's clear and directly addresses the user's query.\n5. **Review and Refine**: Check for accuracy and completeness, filling any gaps with logical assumptions where necessary.\n\n# Output Format\n\n- Concise and coherent responses in paragraphs that directly address the user's question.\n- Incorporate inline code snippets or references from the documentation if relevant.\n\n# Examples\n\n**Example 1**\n\n*Input:*\n- Chat History: \"What was the original reason for moving the LP tokens?\"\n- Corpus Excerpts: \"It isn't clear to me if we redid the staking yet and if we should migrate. If so, perhaps we should make a new issue instead. We should investigate whether the missing LP tokens issue from the MasterChefV2.1 contract is critical to the decision of migrating or not.\"\n\n*Output:*\n\"It was due to missing LP tokens issue from the MasterChefV2.1 Contract.\n\n# Notes\n\n- Ensure the response is crafted from the corpus provided, without introducing information outside of what's available or relevant to the query.\n- Consider edge cases where the corpus might lack explicit answers, and justify responses with logical reasoning based on the existing information.",
       `Your name is: ${botName}`,
       "\n",
-      "Main Context (Provide additional precedence in terms of information): ",
-      localContext.join("\n"),
-      "Secondary Context: ",
-      additionalContext.join("\n"),
+      "Main Context",
+      localContext,
     ].join("\n");
+  }
+
+  async getPromptTokens(query: string = "{query}"): Promise<number> {
+    const systemTemplate = this._getSystemPromptTemplate();
+    const messages = [
+      {
+        role: "system",
+        content: [{ type: "text", text: systemTemplate }],
+      },
+      {
+        role: "user",
+        content: [{ type: "text", text: query }],
+      },
+    ];
+
+    // Convert messages to string to count tokens
+    const messagesStr = JSON.stringify(messages);
+    return encode(messagesStr, { disallowedSpecial: new Set() }).length;
+  }
 
+  async createCompletion(query: string, model: string = "o1-mini", localContext: string[], groundTruths: string[], botName: string): Promise<CompletionsType> {
+    const numTokens = await this.findTokenLength(query, localContext, groundTruths);
+    logger.debug(`Number of tokens: ${numTokens}`);
+    const sysMsg = this._getSystemPromptTemplate(JSON.stringify(groundTruths), botName, localContext.join("\n"));
     logger.info(`System message: ${sysMsg}`);
-    logger.info(`Query: ${query}`);
 
     const res: OpenAI.Chat.Completions.ChatCompletion = await this.client.chat.completions.create({
       model: model,
@@ -109,7 +118,6 @@ export class Completions extends SuperOpenAi {
         },
       ],
       temperature: 0.2,
-      max_tokens: maxTokens,
       top_p: 0.5,
       frequency_penalty: 0,
       presence_penalty: 0,
@@ -118,8 +126,8 @@ export class Completions extends SuperOpenAi {
       },
     });
 
-    if (!res.choices || !res.choices.length) {
-      logger.debug(`No completion found for query: ${query} Response: ${JSON.stringify(res)}`, { res });
+    if (!res.choices || !res.choices[0].message) {
+      logger.error(`Failed to generate completion: ${JSON.stringify(res)}`);
       return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 }, groundTruths };
     }
 
@@ -155,6 +163,11 @@ export class Completions extends SuperOpenAi {
       model: model,
     });
 
+    if (!res.choices || !res.choices[0].message || !res.choices[0].message.content) {
+      this.context.logger.error(`Failed to generate ground truth completion: ${JSON.stringify(res)}`);
+      return null;
+    }
+
     return res.choices[0].message.content;
   }