mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-03 22:11:14 +02:00
feat(agent): improve tiktoken initialization and device registration
- Configure tiktoken cache directory to isolate encoding files - Add logging and timing for tiktoken initialization process - Make Huawei CE and Ruijie Telnet device registration idempotent - Add duplicate registration prevention with global flags - Add logging for device type registration status - Update .gitignore to exclude tiktoken cache files
This commit is contained in:
1 parent
9a01e2194e
commit
b71b323fd4
5 files changed
+67
-1
No files matched your search
@@ -43,10 +43,17 @@ Requirements:
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Callable
|
||||
|
||||
# Configure tiktoken cache directory (must be set before importing tiktoken)
|
||||
_cache_dir = Path(__file__).parent.parent / "cache" / "tiktoken"
|
||||
_cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
os.environ["TIKTOKEN_CACHE_DIR"] = str(_cache_dir)
|
||||
|
||||
import tiktoken
|
||||
from langchain_core.messages import BaseMessage
|
||||
from langchain_core.messages import SystemMessage
|
||||
@@ -59,8 +66,14 @@ logger = logging.getLogger(__name__)
|
||||
# ============================================================================
|
||||
|
||||
# Initialize tiktoken encoding (required dependency)
|
||||
import time
|
||||
logger.info("Initializing tiktoken encoding (cl100k_base)...")
|
||||
logger.info(f"Cache directory: {_cache_dir}")
|
||||
logger.info("This may take a moment on first run (downloading ~1.6MB encoding file from openaipublic.blob.core.windows.net)")
|
||||
start_time = time.time()
|
||||
_tiktoken_encoding = tiktoken.get_encoding("cl100k_base")
|
||||
logger.info("Using tiktoken (cl100k_base) for accurate token counting")
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"✓ tiktoken encoding loaded successfully (took {elapsed:.2f}s)")
|
||||
|
||||
# ============================================================================
|
||||
# Constants
|
||||
|
||||
Reference in new issue
Block a user