diff --git a/evals/handlers/setup-context.ts b/evals/handlers/setup-context.ts index ddb5483..62fb763 100644 --- a/evals/handlers/setup-context.ts +++ b/evals/handlers/setup-context.ts @@ -100,7 +100,7 @@ export async function fetchContext(context: Context, question: string): Promise< logger.debug(`Ground truths tokens: ${groundTruthsTokens}`); // Get formatted chat history with remaining tokens and reranked content - const formattedChat = await formatChatHistory(context, maxDepth, availableTokens, rerankedIssues, rerankedComments); + const formattedChat = await formatChatHistory(context, maxDepth, rerankedIssues, rerankedComments, availableTokens); return { formattedChat, groundTruths, diff --git a/src/adapters/voyage/helpers/rerankers.ts b/src/adapters/voyage/helpers/rerankers.ts index 6d418cd..e9d213a 100644 --- a/src/adapters/voyage/helpers/rerankers.ts +++ b/src/adapters/voyage/helpers/rerankers.ts @@ -2,6 +2,7 @@ import { VoyageAIClient } from "voyageai"; import { Context } from "../../../types"; import { SimilarIssue, SimilarComment } from "../../../types/github-types"; import { SuperVoyage } from "./voyage"; +import { TreeNode } from "../../../helpers/format-chat-history"; interface DocumentWithMetadata { document: string; @@ -19,6 +20,87 @@ export class Rerankers extends SuperVoyage { this.context = context; } + private async _reRankNodesAtLevel(nodes: TreeNode[], query: string, topK: number = 100): Promise { + if (nodes.length === 0) return nodes; + + // Extract content from each node to create documents for reranking + const documents = nodes.map((node) => { + const content = [ + node.body || "", + ...(node.comments?.map((comment) => comment.body || "") || []), + ...(node.similarIssues?.map((issue) => issue.body || "") || []), + ...(node.similarComments?.map((comment) => comment.body || "") || []), + ...(node.codeSnippets?.map((snippet) => snippet.body || "") || []), + node.readmeSection || "", + ] + .filter(Boolean) + .join("\n"); + + return { + document: content, + metadata: { originalNode: node }, + }; + }); + + // Rerank the documents + const response = await this.client.rerank({ + query, + documents: documents.map((doc) => doc.document), + model: "rerank-2", + returnDocuments: true, + topK: Math.min(topK, documents.length), + }); + + const rerankedResults = response.data || []; + + // Map the reranked results back to their original nodes with scores + return rerankedResults + .map((result, index) => { + const originalNode = documents[index].metadata.originalNode; + // Try different possible score properties from the API response + const score = result.relevanceScore || 0; + if (originalNode && typeof score === "number") { + return { + node: originalNode, + score, + }; + } + return null; + }) + .filter((item): item is { node: TreeNode; score: number } => item !== null) + .sort((a, b) => b.score - a.score) // Sort by score in descending order + .map((item) => item.node); + } + + async reRankTreeNodes(rootNode: TreeNode, query: string, topK: number = 100): Promise { + try { + // Helper function to process a node and its children recursively + const processNode = async (node: TreeNode, parentNode?: TreeNode): Promise => { + // Create a new node with all properties from the original + const processedNode: TreeNode = { + ...node, + parent: parentNode, // Set the parent reference + children: [], // Clear children array to be populated with reranked children + }; + + // Rerank children if they exist + if (node.children.length > 0) { + const rerankedChildren = await this._reRankNodesAtLevel(node.children, query, topK); + // Process each reranked child recursively, passing the current node as parent + processedNode.children = await Promise.all(rerankedChildren.map((child) => processNode(child, processedNode))); + } + + return processedNode; + }; + + // Process the entire tree starting from the root (no parent for root node) + return await processNode(rootNode); + } catch (e: unknown) { + this.context.logger.error("Reranking tree nodes failed!", { e }); + return rootNode; + } + } + async reRankResults(results: string[], query: string, topK: number = 5): Promise { let response; try { diff --git a/src/helpers/format-chat-history.ts b/src/helpers/format-chat-history.ts index 2a4d3dd..07f4f01 100644 --- a/src/helpers/format-chat-history.ts +++ b/src/helpers/format-chat-history.ts @@ -3,19 +3,24 @@ import { StreamlinedComment, TokenLimits } from "../types/llm"; import { fetchIssueComments } from "./issue-fetching"; import { splitKey } from "./issue"; import { logger } from "./errors"; -import { Issue } from "../types/github-types"; +import { PullRequestDetails } from "../types/github-types"; import { updateTokenCount, createDefaultTokenLimits } from "./token-utils"; - import { SimilarIssue, SimilarComment } from "../types/github-types"; -interface TreeNode { +const SIMILAR_ISSUE_IDENTIFIER = "Similar Issues:"; +const SIMILAR_COMMENT_IDENTIFIER = "Similar Comments:"; + +export interface TreeNode { key: string; - issue: Issue; children: TreeNode[]; - parent?: TreeNode; + number: number; + html_url: string; depth: number; + parent?: TreeNode; + type: "issue" | "pull_request"; comments?: StreamlinedComment[]; body?: string; + prDetails?: PullRequestDetails; similarIssues?: SimilarIssue[]; similarComments?: SimilarComment[]; codeSnippets?: { body: string; path: string }[]; @@ -194,14 +199,16 @@ async function buildTree( const node: TreeNode = { key, - issue, children: [], depth, + number: issue.number, + html_url: issue.html_url, comments: response.comments.map((comment) => ({ ...comment, user: comment.user?.login || undefined, body: comment.body || undefined, })), + type: issue.pull_request ? "pull_request" : "issue", body: specAndBodies[key] || issue.body || undefined, }; @@ -300,129 +307,190 @@ async function buildTree( } } -async function processTreeNode(node: TreeNode, prefix: string, output: string[], tokenLimits: TokenLimits): Promise { - // Create header - const typeStr = node.issue.pull_request ? "PR" : "Issue"; - const headerLine = `${prefix}${node.parent ? "└── " : ""}${typeStr} #${node.issue.number} (${node.issue.html_url})`; +// Helper function to process node content +async function processNodeContent( + node: TreeNode, + prefix: string, + includeDiffs: boolean, + tokenLimits: TokenLimits +): Promise<{ output: string[]; isSuccess: boolean; childrenOutput: string[]; tokenLimits: TokenLimits }> { + const testTokenLimits = { ...tokenLimits }; + const output: string[] = []; + const childrenOutput: string[] = []; + + // Early token limit check + if (testTokenLimits.runningTokenCount >= testTokenLimits.tokensRemaining) { + return { output, isSuccess: false, childrenOutput, tokenLimits: testTokenLimits }; + } + + // Essential information first + const typeStr = node.type == "issue" ? "Issue" : "PR"; + const headerLine = `${prefix}${node.parent ? "└── " : ""}${typeStr} #${node.number} (${node.html_url})`; - if (!updateTokenCount(headerLine, tokenLimits)) { - return; + if (!updateTokenCount(headerLine, testTokenLimits)) { + return { output, isSuccess: false, childrenOutput, tokenLimits: testTokenLimits }; } output.push(headerLine); const childPrefix = prefix + (node.parent ? " " : ""); const contentPrefix = childPrefix + " "; - // Process body and similar content for root node - if (!node.parent) { - // Process body if exists - if (node.body?.trim()) { - const bodyContent = formatContent("Body", node.body, childPrefix, contentPrefix, tokenLimits); - if (bodyContent.length > 0) { - output.push(...bodyContent); - output.push(""); - } + // Helper function to add content if tokens allow + const tryAddContent = (content: string[], tokenLimit: TokenLimits): boolean => { + const tempLimit = { ...tokenLimit }; + if (content.every((line) => updateTokenCount(line, tempLimit))) { + content.forEach((line) => updateTokenCount(line, tokenLimit)); + output.push(...content); + return true; } + return false; + }; - // Process similar issues - if (node.similarIssues?.length) { - output.push(`${childPrefix}Similar Issues:`); - for (const issue of node.similarIssues) { - const line = `${contentPrefix}- Issue #${issue.issueNumber} (${issue.url}) - Similarity: ${(issue.similarity * 100).toFixed(2)}%`; - if (!updateTokenCount(line, tokenLimits)) break; - output.push(line); - - if (issue.body) { - const bodyLine = `${contentPrefix} ${issue.body}`; - if (!updateTokenCount(bodyLine, tokenLimits)) break; - output.push(bodyLine); - } - } + // Process body (truncate if needed) + if (node.body?.trim()) { + const bodyLines = formatContent("Body", node.body, childPrefix, contentPrefix, testTokenLimits); + if (bodyLines.length > 0) { + tryAddContent(bodyLines, testTokenLimits); output.push(""); } + } - // Process similar comments - if (node.similarComments?.length) { - output.push(`${childPrefix}Similar Comments:`); - for (const comment of node.similarComments) { - const line = `${contentPrefix}- Comment by ${comment.user?.login} - Similarity: ${(comment.similarity * 100).toFixed(2)}%`; - if (!updateTokenCount(line, tokenLimits)) break; - output.push(line); - - if (comment.body) { - const bodyLine = `${contentPrefix} ${comment.body}`; - if (!updateTokenCount(bodyLine, tokenLimits)) break; - output.push(bodyLine); - } - } + // Process PR diffs if space allows + if (includeDiffs && node.type === "pull_request" && node.prDetails?.diff) { + const diffLines = formatContent("Diff", node.prDetails.diff, childPrefix, contentPrefix, testTokenLimits); + if (diffLines.length > 0) { + tryAddContent(diffLines, testTokenLimits); output.push(""); } - } else if (node.body?.trim()) { - // Process body for non-root nodes - const bodyContent = formatContent("Body", node.body, childPrefix, contentPrefix, tokenLimits); - if (bodyContent.length > 0) { - output.push(...bodyContent); + } + + // Process comments (most recent first) + if (node.comments?.length) { + const commentsHeader = `${childPrefix}Comments: ${node.comments.length}`; + if (updateTokenCount(commentsHeader, testTokenLimits)) { + output.push(commentsHeader); + + // Sort comments by recency + const sortedComments = [...node.comments].sort((a, b) => parseInt(b.id) - parseInt(a.id)); + + for (const comment of sortedComments) { + if (!comment.body?.trim()) continue; + + const commentLine = `${childPrefix}├── ${comment.commentType || "issuecomment"}-${comment.id}: ${comment.user}: ${comment.body.trim()}`; + + if (!updateTokenCount(commentLine, testTokenLimits)) { + break; + } + output.push(commentLine); + + // Add referenced code if space allows + if (includeDiffs && comment.commentType === "pull_request_review_comment" && comment.referencedCode) { + const codeLines = [ + `${childPrefix} Referenced code in ${comment.referencedCode.path}:`, + `${childPrefix} Lines ${comment.referencedCode.startLine}-${comment.referencedCode.endLine}:`, + ...comment.referencedCode.content.split("\n").map((line) => `${childPrefix} ${line}`), + ]; + + tryAddContent(codeLines, testTokenLimits); + } + } output.push(""); } } - // Process PR details if available - if (node.issue.prDetails) { - const { diff } = node.issue.prDetails; + // Process similar content only for root node if space allows + if (!node.parent && testTokenLimits.runningTokenCount < testTokenLimits.tokensRemaining - 1000) { + for (const [type, items] of [ + [SIMILAR_ISSUE_IDENTIFIER, node.similarIssues], + [SIMILAR_COMMENT_IDENTIFIER, node.similarComments], + ] as const) { + if (!items?.length) continue; + + const typeHeader = `${childPrefix}${type}:`; + if (!updateTokenCount(typeHeader, testTokenLimits)) break; + output.push(typeHeader); + + // Sort by similarity + const sortedItems = [...items].sort((a, b) => b.similarity - a.similarity); + + for (const item of sortedItems) { + const similarity = (item.similarity * 100).toFixed(2); + const identifier = + type === SIMILAR_ISSUE_IDENTIFIER ? "Issue #" + (item as SimilarIssue).issueNumber : "Comment by " + (item as SimilarComment).user?.login; + const url = type === SIMILAR_ISSUE_IDENTIFIER ? (item as SimilarIssue).url : ""; + + const itemHeader = contentPrefix + "- " + identifier; + const urlPart = url ? " (" + url + ")" : ""; + const similarityPart = " - Similarity: " + similarity + "%"; + const itemLines = [itemHeader + urlPart + similarityPart]; + + if (item.body) { + const maxLength = 500; + const truncatedBody = item.body.length > maxLength ? item.body.slice(0, maxLength) + "..." : item.body; + itemLines.push(contentPrefix + " " + truncatedBody); + } - // Add diff information - if (diff) { - const diffContent = formatContent("Diff", diff, childPrefix, contentPrefix, tokenLimits); - if (diffContent.length > 0) { - output.push(...diffContent); - output.push(""); + if (!tryAddContent(itemLines, testTokenLimits)) { + break; + } } + output.push(""); } } - // Process comments if any - if (node.comments?.length) { - const commentsHeader = `${childPrefix}Comments: ${node.comments.length}`; - if (updateTokenCount(commentsHeader, tokenLimits)) { - output.push(commentsHeader); + const isSuccess = testTokenLimits.runningTokenCount <= testTokenLimits.tokensRemaining; + return { output, isSuccess, childrenOutput, tokenLimits: testTokenLimits }; +} - for (let i = 0; i < node.comments.length; i++) { - const comment = node.comments[i]; - if (!comment.body?.trim()) continue; +async function processTreeNode(node: TreeNode, prefix: string, output: string[], tokenLimits: TokenLimits): Promise { + // Early check for token limit + if (tokenLimits.runningTokenCount >= tokenLimits.tokensRemaining) { + return false; + } - const commentPrefix = i === node.comments.length - 1 ? "└── " : "├── "; - let commentLine = `${childPrefix}${commentPrefix}${comment.commentType || "issuecomment"}-${comment.id}: ${comment.user}: ${comment.body.trim()}`; + // Process current node first to ensure most relevant content is included + const result = await processNodeContent(node, prefix, false, tokenLimits); // Start without diffs + + if (result.isSuccess) { + // If we have room for diffs and it's a PR, try adding them + if (node.type === "pull_request" && tokenLimits.tokensRemaining - tokenLimits.runningTokenCount > 1000) { + // Buffer for diffs + const withDiffs = await processNodeContent(node, prefix, true, { ...tokenLimits }); + if (withDiffs.isSuccess) { + output.push(...withDiffs.output); + Object.assign(tokenLimits, withDiffs.tokenLimits); + } else { + output.push(...result.output); + Object.assign(tokenLimits, result.tokenLimits); + } + } else { + output.push(...result.output); + Object.assign(tokenLimits, result.tokenLimits); + } - // Add referenced code for PR review comments if available - if (comment.commentType === "pull_request_review_comment" && comment.referencedCode) { - const lineNumbers = `Lines ${comment.referencedCode.startLine}-${comment.referencedCode.endLine}:`; - const codePath = `Referenced code in ${comment.referencedCode.path}:`; - const content = comment.referencedCode.content.split("\n"); - const indentedContent = content.map((line) => childPrefix + " " + line).join("\n"); - const codeLines = [childPrefix + " " + codePath, childPrefix + " " + lineNumbers, childPrefix + " " + indentedContent]; + // Process children only if we have enough tokens left + if (tokenLimits.runningTokenCount < tokenLimits.tokensRemaining) { + // Sort children by recency (assuming newer items are more relevant) + const sortedChildren = [...node.children].sort((a, b) => b.number - a.number); - if (!updateTokenCount(codeLines.join("\n"), tokenLimits)) { - break; - } - commentLine = `${commentLine}\n${codeLines.join("\n")}`; + for (const child of sortedChildren) { + // Check if we have enough tokens for more content + if (tokenLimits.runningTokenCount >= tokenLimits.tokensRemaining - 500) { + // Leave buffer + break; } - if (!updateTokenCount(commentLine, tokenLimits)) { + const isChildProcessingComplete = await processTreeNode(child, prefix + " ", output, tokenLimits); + if (!isChildProcessingComplete) { break; } - output.push(commentLine); } - output.push(""); } - } - // Process children - for (let i = 0; i < node.children.length; i++) { - const child = node.children[i]; - const isLast = i === node.children.length - 1; - const nextPrefix = childPrefix + (isLast ? " " : "│ "); - await processTreeNode(child, nextPrefix, output, tokenLimits); + return true; } + + return false; } function formatContent(type: string, content: string, prefix: string, contentPrefix: string, tokenLimits: TokenLimits): string[] { @@ -448,16 +516,21 @@ function formatContent(type: string, content: string, prefix: string, contentPre return output; } -export async function buildChatHistoryTree(context: Context, maxDepth: number = 2): Promise<{ tree: TreeNode | null; tokenLimits: TokenLimits }> { +export async function buildChatHistoryTree( + context: Context, + maxDepth: number = 2, + similarComments: SimilarComment[], + similarIssues: SimilarIssue[] +): Promise<{ tree: TreeNode | null; tokenLimits: TokenLimits }> { const specAndBodies: Record = {}; const tokenLimits = createDefaultTokenLimits(context); - const { tree } = await buildTree(context, specAndBodies, maxDepth, tokenLimits); + const { tree } = await buildTree(context, specAndBodies, maxDepth, tokenLimits, similarIssues, similarComments); if (tree && "pull_request" in context.payload) { const { diff_hunk, position, original_position, path, body } = context.payload.comment || {}; if (diff_hunk) { tree.body += `\nPrimary Context: ${body || ""}\nDiff: ${diff_hunk}\nPath: ${path || ""}\nLines: ${position || ""}-${original_position || ""}`; - tree.comments = tree.comments?.filter((comment) => comment.id !== context.payload.comment?.id); + tree.comments = tree.comments?.filter((comment) => comment.id !== String(context.payload.comment?.id)); } } @@ -467,16 +540,19 @@ export async function buildChatHistoryTree(context: Context, maxDepth: number = export async function formatChatHistory( context: Context, maxDepth: number = 2, - availableTokens?: number, - similarIssues?: SimilarIssue[], - similarComments?: SimilarComment[] + similarIssues: SimilarIssue[], + similarComments: SimilarComment[], + availableTokens?: number ): Promise { - const { tree, tokenLimits } = await buildChatHistoryTree(context, maxDepth); + const { tree, tokenLimits } = await buildChatHistoryTree(context, maxDepth, similarComments, similarIssues); if (!tree) { return ["No main issue found."]; } + // Rerank the chat history + const reRankedChat = await context.adapters.voyage.reranker.reRankTreeNodes(tree, context.payload.comment.body); + // If availableTokens is provided, override the default tokensRemaining if (availableTokens !== undefined) { tokenLimits.tokensRemaining = availableTokens; @@ -494,7 +570,10 @@ export async function formatChatHistory( const headerLine = "Issue Tree Structure:"; treeOutput.push(headerLine, ""); - await processTreeNode(tree, "", treeOutput, tokenLimits); - logger.debug(`Final tokens: ${tokenLimits.runningTokenCount}/${tokenLimits.tokensRemaining}`); + const tokenLimitsnew = createDefaultTokenLimits(context); + + const isSuccess = await processTreeNode(reRankedChat, "", treeOutput, tokenLimitsnew); + logger.debug(`Tree processing ${isSuccess ? "succeeded" : "failed"} with tokens: ${tokenLimitsnew.runningTokenCount}/${tokenLimitsnew.tokensRemaining}`); + logger.debug(`Tree fetching tokens: ${tokenLimits.runningTokenCount}/${tokenLimits.tokensRemaining}`); return treeOutput; } diff --git a/src/helpers/pull-request-fetching.ts b/src/helpers/pull-request-fetching.ts index e4d36a7..0d4ab67 100644 --- a/src/helpers/pull-request-fetching.ts +++ b/src/helpers/pull-request-fetching.ts @@ -1,74 +1,9 @@ import { Context } from "../types"; -import { FetchParams, SimplifiedComment } from "../types/github-types"; +import { FetchParams, PullRequestGraphQlResponse, PullRequestLinkedIssue, SimplifiedComment } from "../types/github-types"; import { TokenLimits } from "../types/llm"; import { logger } from "./errors"; import { processPullRequestDiff } from "./pull-request-parsing"; -interface PullRequestGraphQlResponse { - repository: { - pullRequest: { - body: string; - closingIssuesReferences: { - nodes: Array<{ - number: number; - url: string; - body: string; - repository: { - owner: { - login: string; - }; - name: string; - }; - }>; - }; - reviews: { - pageInfo: { - hasNextPage: boolean; - endCursor: string | null; - }; - nodes: Array<{ - comments: { - nodes: Array<{ - id: string; - body: string; - author: { - login: string; - type: string; - }; - path?: string; - line?: number; - startLine?: number; - diffHunk?: string; - }>; - }; - }>; - }; - comments: { - pageInfo: { - hasNextPage: boolean; - endCursor: string | null; - }; - nodes: Array<{ - id: string; - body: string; - author: { - login: string; - type: string; - }; - }>; - }; - }; - }; -} - -interface PullRequestLinkedIssue { - number: number; - owner: string; - repo: string; - url: string; - body: string; -} - /** * Fetch both PR review comments and regular PR comments */ diff --git a/src/helpers/token-utils.ts b/src/helpers/token-utils.ts index 4e7b497..111f301 100644 --- a/src/helpers/token-utils.ts +++ b/src/helpers/token-utils.ts @@ -3,11 +3,8 @@ import { TokenLimits } from "../types/llm"; import { encode } from "gpt-tokenizer"; export function createDefaultTokenLimits(context: Context): TokenLimits { - // const modelMaxTokenLimit = context.adapters.openai.completions.getModelMaxTokenLimit(context.config.model); - // const maxCompletionTokens = context.adapters.openai.completions.getModelMaxOutputLimit(context.config.model); - - const modelMaxTokenLimit = 128_000; - const maxCompletionTokens = 16_384; + const modelMaxTokenLimit = context.adapters.openai.completions.getModelMaxTokenLimit(context.config.model); + const maxCompletionTokens = context.adapters.openai.completions.getModelMaxOutputLimit(context.config.model); return { modelMaxTokenLimit, maxCompletionTokens, diff --git a/src/types/github-types.ts b/src/types/github-types.ts index 9eda371..2beafa4 100644 --- a/src/types/github-types.ts +++ b/src/types/github-types.ts @@ -25,49 +25,6 @@ type Repository = { name: string; }; -type IssueData = { - number: number; - url: string; - body: string; - repository: Repository; -}; - -type PullRequestNode = { - id: string; - body: string; - closingIssuesReferences: { - nodes: IssueData[]; - }; -}; - -type PullRequestReviewCommentNode = { - id: string; - body: string; - pullRequest: PullRequestNode; -}; - -type IssueCommentNode = { - id: string; - body: string; - issue: IssueData; -}; - -export type GqlIssueSearchResult = { - node: IssueData; -}; - -export type GqlPullRequestSearchResult = { - node: PullRequestNode; -}; - -export type GqlPullRequestReviewCommentSearchResult = { - node: PullRequestReviewCommentNode; -}; - -export type GqlIssueCommentSearchResult = { - node: IssueCommentNode; -}; - export interface PullRequestDetails { diff: string | null; } @@ -204,3 +161,68 @@ export interface CommentIssueSearchResult { }; }; } + +export interface PullRequestGraphQlResponse { + repository: { + pullRequest: { + body: string; + closingIssuesReferences: { + nodes: Array<{ + number: number; + url: string; + body: string; + repository: { + owner: { + login: string; + }; + name: string; + }; + }>; + }; + reviews: { + pageInfo: { + hasNextPage: boolean; + endCursor: string | null; + }; + nodes: Array<{ + comments: { + nodes: Array<{ + id: string; + body: string; + author: { + login: string; + type: string; + }; + path?: string; + line?: number; + startLine?: number; + diffHunk?: string; + }>; + }; + }>; + }; + comments: { + pageInfo: { + hasNextPage: boolean; + endCursor: string | null; + }; + nodes: Array<{ + id: string; + body: string; + author: { + login: string; + type: string; + }; + }>; + }; + }; + }; +} + +export interface PullRequestLinkedIssue { + number: number; + owner: string; + repo: string; + url: string; + body: string; +} diff --git a/src/types/llm.ts b/src/types/llm.ts index ee2af3b..427cdbf 100644 --- a/src/types/llm.ts +++ b/src/types/llm.ts @@ -34,7 +34,7 @@ export type GroundTruthsSystemMessageTemplate = { }; export type StreamlinedComment = { - id: string | number; + id: string; user?: string; body?: string; org: string;