feat: add function call in SGLang (#1666)

NeilJohnson0930 · web-flow · commit 1dc816d379a2 · 2025-03-11T17:43:16.000+08:00
diff --git a/camel/configs/sglang_config.py b/camel/configs/sglang_config.py
@@ -13,7 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 from __future__ import annotations
 
-from typing import Sequence, Union
+from typing import Any, Dict, List, Optional, Sequence, Union
 
 from camel.configs.base_config import BaseConfig
 from camel.types import NOT_GIVEN, NotGiven
@@ -56,10 +56,11 @@ class SGLangConfig(BaseConfig):
             in the chat completion. The total length of input tokens and
             generated tokens is limited by the model's context length.
             (default: :obj:`None`)
-        tools (list[FunctionTool], optional): A list of tools the model may
-            call. Currently, only functions are supported as a tool. Use this
-            to provide a list of functions the model may generate JSON inputs
-            for. A max of 128 functions are supported.
+        tools (list[Dict[str, Any]], optional): A list of tool definitions
+            that the model can dynamically invoke. Each tool should be
+            defined as a dictionary following OpenAI's function calling
+            specification format. For more details, refer to the OpenAI
+            documentation.
     """
 
     stop: Union[str, Sequence[str], NotGiven] = NOT_GIVEN
@@ -70,6 +71,7 @@ class SGLangConfig(BaseConfig):
     presence_penalty: float = 0.0
     stream: bool = False
     max_tokens: Union[int, NotGiven] = NOT_GIVEN
+    tools: Optional[Union[List[Dict[str, Any]]]] = None
 
 
 SGLANG_API_PARAMS = {param for param in SGLangConfig.model_fields.keys()}
diff --git a/camel/models/sglang_model.py b/camel/models/sglang_model.py
@@ -97,9 +97,16 @@ def __init__(
     def _start_server(self) -> None:
         try:
             if not self._url:
+                tool_call_flag = self.model_config_dict.get("tools")
+                tool_call_arg = (
+                    f"--tool-call-parser {self._api_key} "
+                    if tool_call_flag
+                    else ""
+                )
                 cmd = (
                     f"python -m sglang.launch_server "
                     f"--model-path {self.model_type} "
+                    f"{tool_call_arg}"
                     f"--port 30000 "
                     f"--host 0.0.0.0"
                 )
@@ -265,6 +272,19 @@ def stream(self) -> bool:
         """
         return self.model_config_dict.get('stream', False)
 
+    def __del__(self):
+        r"""Properly clean up resources when the model is destroyed."""
+        self.cleanup()
+
+    def cleanup(self):
+        r"""Terminate the server process and clean up resources."""
+        with self._lock:
+            if self.server_process:
+                _terminate_process(self.server_process)
+                self.server_process = None
+                self._client = None
+                logging.info("Server process terminated during cleanup.")
+
 
 # Below are helper functions from sglang.utils
 def _terminate_process(process):
@@ -326,21 +346,25 @@ def _execute_shell_command(command: str) -> subprocess.Popen:
     return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT)
 
 
-def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:
+def _wait_for_server(base_url: str, timeout: Optional[int] = 30) -> None:
     r"""Wait for the server to be ready by polling the /v1/models endpoint.
 
     Args:
         base_url: The base URL of the server
-        timeout: Maximum time to wait in seconds. None means wait forever.
+        timeout: Maximum time to wait in seconds. Default is 30 seconds.
     """
     import requests
 
+    # Set a default value if timeout is None
+    actual_timeout = 30 if timeout is None else timeout
+
     start_time = time.time()
     while True:
         try:
             response = requests.get(
                 f"{base_url}/v1/models",
                 headers={"Authorization": "Bearer None"},
+                timeout=5,  # Add a timeout for the request itself
             )
             if response.status_code == 200:
                 time.sleep(5)
@@ -356,9 +380,15 @@ def _wait_for_server(base_url: str, timeout: Optional[int] = None) -> None:
                 )
                 break
 
-            if timeout and time.time() - start_time > timeout:
+            if time.time() - start_time > actual_timeout:
+                raise TimeoutError(
+                    f"Server did not become ready within "
+                    f"{actual_timeout} seconds"
+                )
+        except (requests.exceptions.RequestException, TimeoutError) as e:
+            if time.time() - start_time > actual_timeout:
                 raise TimeoutError(
-                    "Server did not become ready within timeout period"
+                    f"Server did not become ready within "
+                    f"{actual_timeout} seconds: {e}"
                 )
-        except requests.exceptions.RequestException:
             time.sleep(1)
diff --git a/examples/models/sglang_model_example.py b/examples/models/sglang_model_example.py
@@ -51,6 +51,84 @@
 
 """
 ===============================================================================
-CAMEL AI ReferentialAction
+Hello CAMEL AI. How can I assist you today?
+===============================================================================
+"""
+
+weather_tool = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "city": {
+                        "type": "string",
+                        "description": "The city to find the weather for,\n"
+                        "e.g. 'San Francisco'",
+                    },
+                    "state": {
+                        "type": "string",
+                        "description": "The two-letter abbreviation for,\n"
+                        "the state (e.g., 'CA'), e.g. CA for California",
+                    },
+                    "unit": {
+                        "type": "string",
+                        "description": "Temperature unit (celsius/fahrenheit)",
+                        "enum": ["celsius", "fahrenheit"],
+                    },
+                },
+                "required": ["city", "state", "unit"],
+            },
+        },
+    }
+]
+
+
+r"""
+Note that api_key defines the parser used to interpret responses.
+Currently supported parsers include:
+llama3: Llama 3.1 / 3.2 (e.g. meta-llama/Llama-3.1-8B-Instruct,
+        meta-llama/Llama-3.2-1B-Instruct).
+mistral: Mistral (e.g. mistralai/Mistral-7B-Instruct-v0.3,
+         mistralai/Mistral-Nemo-Instruct-2407, 
+         mistralai/ Mistral-Nemo-Instruct-2407, mistralai/Mistral-7B-v0.3).
+qwen25: Qwen 2.5 (e.g. Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-7B-Instruct).
+"""
+sglang_model_with_tool = ModelFactory.create(
+    model_platform=ModelPlatformType.SGLANG,
+    model_type="meta-llama/Llama-3.2-1B-Instruct",
+    model_config_dict={"temperature": 0.0, "tools": weather_tool},
+    api_key="llama3",
+)
+
+assistant_sys_msg = (
+    "You are a helpful assistant.\n"
+    "Use the get_current_weather tool when asked about weather."
+)
+agent_with_tool = ChatAgent(
+    assistant_sys_msg,
+    model=sglang_model_with_tool,
+    token_limit=4096,
+    external_tools=weather_tool,
+)
+user_msg = "What's the weather in Boston today?"
+
+assistant_response = agent_with_tool.step(user_msg)
+external_tool_call = assistant_response.info.get('external_tool_call_request')
+if external_tool_call:
+    print(f"Detected external tool call: {external_tool_call.tool_name}")
+    print(f"Arguments: {external_tool_call.args}")
+    print(f"Tool Call ID: {external_tool_call.tool_call_id}")
+else:
+    print("No external tool call detected")
+
+"""
+===============================================================================
+Detected external tool call: get_current_weather
+Arguments: {'city': 'Boston', 'state': 'MA', 'unit': 'celsius'}
+Tool Call ID: 0
 ===============================================================================
 """
diff --git a/test/models/test_sglang_model.py b/test/models/test_sglang_model.py
@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import re
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -21,6 +22,47 @@
 from camel.utils import OpenAITokenCounter
 
 
+@pytest.fixture
+def sglang_model_cleanup():
+    r"""Fixture to ensure SGLang model is cleaned up after each test."""
+    models = []
+
+    # Mock the server-related functions to avoid actual server startup
+    with (
+        patch(
+            'camel.models.sglang_model._execute_shell_command'
+        ) as mock_execute,
+        patch('camel.models.sglang_model._wait_for_server') as mock_wait,
+        patch('camel.models.sglang_model.OpenAI') as mock_client,
+        patch('camel.models.sglang_model.AsyncOpenAI') as mock_async_client,
+    ):
+        # Configure mocks
+        mock_execute.return_value = MagicMock()
+        mock_wait.return_value = None
+        mock_client.return_value = MagicMock()
+        mock_async_client.return_value = MagicMock()
+
+        def _create_model(
+            model_type, model_config_dict=None, api_key="sglang"
+        ):
+            model = SGLangModel(model_type, model_config_dict, api_key=api_key)
+            # Set up the model to use our mocks
+            model._url = "http://mock-server:30000/v1"
+            model._client = mock_client.return_value
+            model._async_client = mock_async_client.return_value
+            models.append(model)
+            return model
+
+        yield _create_model
+
+    # Clean up all models after test
+    for model in models:
+        try:
+            model.cleanup()
+        except Exception as e:
+            print(f"Error during cleanup: {e}")
+
+
 @pytest.mark.model_backend
 @pytest.mark.parametrize(
     "model_type",
@@ -31,8 +73,8 @@
         ModelType.GPT_4O_MINI,
     ],
 )
-def test_sglang_model(model_type: ModelType):
-    model = SGLangModel(model_type, api_key="sglang")
+def test_sglang_model(model_type: ModelType, sglang_model_cleanup):
+    model = sglang_model_cleanup(model_type)
     assert model.model_type == model_type
     assert model.model_config_dict == SGLangConfig().as_dict()
     assert isinstance(model.token_counter, OpenAITokenCounter)
@@ -41,7 +83,7 @@ def test_sglang_model(model_type: ModelType):
 
 
 @pytest.mark.model_backend
-def test_sglang_model_unexpected_argument():
+def test_sglang_model_unexpected_argument(sglang_model_cleanup):
     model_type = ModelType.GPT_4
     model_config_dict = {"model_path": "vicuna-7b-v1.5"}
 
@@ -54,4 +96,75 @@ def test_sglang_model_unexpected_argument():
             )
         ),
     ):
-        _ = SGLangModel(model_type, model_config_dict, api_key="sglang")
+        _ = sglang_model_cleanup(model_type, model_config_dict)
+
+
+@pytest.mark.model_backend
+def test_sglang_function_call(sglang_model_cleanup):
+    test_tool = {
+        "type": "function",
+        "function": {
+            "name": "test_tool",
+            "description": "Test function",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    }
+
+    model = sglang_model_cleanup(
+        ModelType.GPT_4,
+        model_config_dict={"tools": [test_tool]},
+    )
+
+    # Create a mock response object
+    from camel.types import (
+        ChatCompletion,
+        ChatCompletionMessage,
+        ChatCompletionMessageToolCall,
+        Choice,
+        CompletionUsage,
+    )
+
+    # create mock response
+    mock_response = ChatCompletion(
+        id="mock_id",
+        object="chat.completion",
+        created=1234567890,
+        model="meta-llama/Meta-Llama-3.1-8B-Instruct",
+        choices=[
+            Choice(
+                index=0,
+                message=ChatCompletionMessage(
+                    role="assistant",
+                    content=None,
+                    tool_calls=[
+                        ChatCompletionMessageToolCall(
+                            id="0",
+                            type="function",
+                            function={"name": "test_tool", "arguments": "{}"},
+                        )
+                    ],
+                ),
+                finish_reason="tool_calls",
+            )
+        ],
+        usage=CompletionUsage(
+            prompt_tokens=10,
+            completion_tokens=20,
+            total_tokens=30,
+        ),
+    )
+
+    # Patch the run method to return our mock response
+    with patch.object(model, '_run', return_value=mock_response):
+        messages = [
+            {
+                "role": "user",
+                "content": "Use test_tool and respond with result",
+            }
+        ]
+
+        response = model.run(messages=messages)
+
+        assert len(response.choices[0].message.tool_calls) > 0
+        tool_call = response.choices[0].message.tool_calls[0]
+        assert tool_call.function.name == "test_tool"
