Merge pull request #4560 from 2hy6v/add-tools-support-for-fireworks-ai

added tools support for fireworks.ai
continuedev · Mar 9, 2025 · 66335c6 · 66335c6
2 parents e45bd6a + ba80343
commit 66335c6
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 1 deletion.
diff --git a/core/llm/llms/OpenAI.ts b/core/llm/llms/OpenAI.ts
@@ -97,6 +97,10 @@ class OpenAI extends BaseLLM {
     return !!model && (model.startsWith("o1") || model.startsWith("o3"));
   }
 
+  private isFireworksAiModel(model?: string): boolean {
+    return !!model && model.startsWith("accounts/fireworks/models");
+  }
+
   protected supportsPrediction(model: string): boolean {
     const SUPPORTED_MODELS = ["gpt-4o-mini", "gpt-4o", "mistral-large"];
     return SUPPORTED_MODELS.some((m) => model.includes(m));
@@ -274,7 +278,13 @@ class OpenAI extends BaseLLM {
       body.max_completion_tokens = undefined;
     }
 
-    if (body.tools?.length && !body.model?.startsWith("o3")) {
+    if (body.tools?.length) {
+      if (this.isFireworksAiModel(body.model)) {
+        // fireworks.ai does not support parallel tool calls, but their api expects this to be true anyway otherwise they return an error.
+        // tooling works with them as a inference provider once this is set to true.
+        // https://docs.fireworks.ai/guides/function-calling#openai-compatibility
+        body.parallel_tool_calls = true;
+      }
       // To ensure schema adherence: https://platform.openai.com/docs/guides/function-calling#parallel-function-calling-and-structured-outputs
       // In practice, setting this to true and asking for multiple tool calls
       // leads to "arguments" being something like '{"file": "test.ts"}{"file": "test.js"}'

diff --git a/core/llm/toolSupport.ts b/core/llm/toolSupport.ts
@@ -39,6 +39,19 @@ export const PROVIDER_TOOL_SUPPORT: Record<
     ) {
       return true;
     }
+    // firworks-ai https://docs.fireworks.ai/guides/function-calling
+    if (model.startsWith("accounts/fireworks/models/")) {
+      switch (model.substring(26)) {
+        case "llama-v3p1-405b-instruct":
+        case "llama-v3p1-70b-instruct":
+        case "qwen2p5-72b-instruct":
+        case "firefunction-v1":
+        case "firefunction-v2":
+          return true;
+        default:
+          return false;
+      }
+    }
   },
   gemini: (model) => {
     // All gemini models support function calling