From 3b7d633b07009248e152843b12b13eed54a286e1 Mon Sep 17 00:00:00 2001 From: Diogo Soares Rodrigues Date: Wed, 17 Dec 2025 16:47:34 -0300 Subject: [PATCH 1/5] feat: migrate to new google-genai SDK and update embedding model - Migrate from deprecated google-generativeai to google-genai SDK - Add support for thinking_level config (Gemini 3 models) - Update default model to gemini-3-flash-preview with thinking_level: high - Update embedder to use gemini-embedding-001 (replacing deprecated text-embedding-004) - Set embedding output_dimensionality to 3072 for optimal quality - Remove unused google.generativeai imports from api.py and main.py - Update pyproject.toml dependencies Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- api/api.py | 1 - api/config/embedder.json | 5 +- api/config/generator.json | 8 ++- api/main.py | 10 ++- api/poetry.lock | 136 +++++++++++++++++++++++++------------- api/pyproject.toml | 3 +- api/simple_chat.py | 73 +++++++++++++------- api/websocket_wiki.py | 73 +++++++++++++------- 8 files changed, 203 insertions(+), 106 deletions(-) diff --git a/api/api.py b/api/api.py index d40e73f96..4decd7247 100644 --- a/api/api.py +++ b/api/api.py @@ -7,7 +7,6 @@ import json from datetime import datetime from pydantic import BaseModel, Field -import google.generativeai as genai import asyncio # Configure logging diff --git a/api/config/embedder.json b/api/config/embedder.json index f0ab52d1e..89d4a2722 100644 --- a/api/config/embedder.json +++ b/api/config/embedder.json @@ -18,8 +18,9 @@ "client_class": "GoogleEmbedderClient", "batch_size": 100, "model_kwargs": { - "model": "text-embedding-004", - "task_type": "SEMANTIC_SIMILARITY" + "model": "gemini-embedding-001", + "task_type": "SEMANTIC_SIMILARITY", + "output_dimensionality": 3072 } }, "retriever": { diff --git a/api/config/generator.json b/api/config/generator.json index f88179098..a6c20e61b 100644 --- a/api/config/generator.json +++ b/api/config/generator.json @@ -20,9 +20,15 @@ } }, "google": { - "default_model": "gemini-2.5-flash", + "default_model": "gemini-3-flash-preview", "supportsCustomModel": true, "models": { + "gemini-3-flash-preview": { + "temperature": 1.0, + "top_p": 0.8, + "top_k": 20, + "thinking_level": "high" + }, "gemini-2.5-flash": { "temperature": 1.0, "top_p": 0.8, diff --git a/api/main.py b/api/main.py index fe083f550..63e4fd5c8 100644 --- a/api/main.py +++ b/api/main.py @@ -50,14 +50,12 @@ def patched_watch(*args, **kwargs): logger.warning(f"Missing environment variables: {', '.join(missing_vars)}") logger.warning("Some functionality may not work correctly without these variables.") -# Configure Google Generative AI -import google.generativeai as genai +# Note: Google GenAI configuration is now done at client instantiation time +# with the new google-genai SDK. The API key is passed to genai.Client() from api.config import GOOGLE_API_KEY -if GOOGLE_API_KEY: - genai.configure(api_key=GOOGLE_API_KEY) -else: - logger.warning("GOOGLE_API_KEY not configured") +if not GOOGLE_API_KEY: + logger.warning("GOOGLE_API_KEY not configured - Google Gemini models will not work") if __name__ == "__main__": # Get port from environment variable or use default diff --git a/api/poetry.lock b/api/poetry.lock index a2446bba9..e07df94ab 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "adalflow" @@ -37,7 +37,7 @@ faiss-cpu = ["faiss-cpu (>=1.8.0)"] google-generativeai = ["google-generativeai (>=0.7.2)"] groq = ["groq (>=0.9.0)"] lancedb = ["lancedb (>=0.5.2)"] -mcp = ["mcp (>=1.9.4,<2.0.0)"] +mcp = ["mcp (>=1.9.4,<2.0.0) ; python_version >= \"3.10\""] ollama = ["ollama (>=0.2.1)"] openai = ["openai (>=1.97.1)"] pgvector = ["pgvector (>=0.3.1)"] @@ -197,7 +197,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.3.0)", "backports.zstd", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -261,8 +261,8 @@ files = [ [package.extras] doc = ["sphinx", "sphinxcontrib-trio"] -test = ["black", "coverage", "flake8", "flake8-2020", "flake8-bugbear", "mypy", "pytest", "pytest-cov"] -typetest = ["mypy", "pyright", "typing-extensions"] +test = ["black ; implementation_name == \"cpython\"", "coverage", "flake8", "flake8-2020", "flake8-bugbear", "mypy ; implementation_name == \"cpython\"", "pytest", "pytest-cov"] +typetest = ["mypy ; implementation_name == \"cpython\"", "pyright", "typing-extensions"] [[package]] name = "attrs" @@ -705,7 +705,7 @@ files = [ ] [package.dependencies] -cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9\" and platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] @@ -1000,51 +1000,51 @@ requests = ">=2.18.0,<3.0.0" [package.extras] async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] -grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0)", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-api-core" -version = "2.26.0" +version = "2.28.1" description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main"] -markers = "python_version < \"3.14\"" +markers = "python_version <= \"3.13\"" files = [ - {file = "google_api_core-2.26.0-py3-none-any.whl", hash = "sha256:2b204bd0da2c81f918e3582c48458e24c11771f987f6258e6e227212af78f3ed"}, - {file = "google_api_core-2.26.0.tar.gz", hash = "sha256:e6e6d78bd6cf757f4aee41dcc85b07f485fbb069d5daa3afb126defba1e91a62"}, + {file = "google_api_core-2.28.1-py3-none-any.whl", hash = "sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c"}, + {file = "google_api_core-2.28.1.tar.gz", hash = "sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8"}, ] [package.dependencies] google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.2,<2.0.0" grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""} -grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""} +grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} proto-plus = [ {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, - {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" requests = ">=2.18.0,<3.0.0" [package.extras] async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] -grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0)", "grpcio (>=1.75.1,<2.0.0)", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0)", "grpcio-status (>=1.75.1,<2.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-api-python-client" -version = "2.185.0" +version = "2.187.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_python_client-2.185.0-py3-none-any.whl", hash = "sha256:00fe173a4b346d2397fbe0d37ac15368170dfbed91a0395a66ef2558e22b93fc"}, - {file = "google_api_python_client-2.185.0.tar.gz", hash = "sha256:aa1b338e4bb0f141c2df26743f6b46b11f38705aacd775b61971cbc51da089c3"}, + {file = "google_api_python_client-2.187.0-py3-none-any.whl", hash = "sha256:d8d0f6d85d7d1d10bdab32e642312ed572bdc98919f72f831b44b9a9cebba32f"}, + {file = "google_api_python_client-2.187.0.tar.gz", hash = "sha256:e98e8e8f49e1b5048c2f8276473d6485febc76c9c47892a8b4d1afa2c9ec8278"}, ] [package.dependencies] @@ -1056,56 +1056,86 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.41.1" +version = "2.45.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_auth-2.41.1-py2.py3-none-any.whl", hash = "sha256:754843be95575b9a19c604a848a41be03f7f2afd8c019f716dc1f51ee41c639d"}, - {file = "google_auth-2.41.1.tar.gz", hash = "sha256:b76b7b1f9e61f0cb7e88870d14f6a94aeef248959ef6992670efee37709cbfd2"}, + {file = "google_auth-2.45.0-py2.py3-none-any.whl", hash = "sha256:82344e86dc00410ef5382d99be677c6043d72e502b625aa4f4afa0bdacca0f36"}, + {file = "google_auth-2.45.0.tar.gz", hash = "sha256:90d3f41b6b72ea72dd9811e765699ee491ab24139f34ebf1ca2b9cc0c38708f3"}, ] [package.dependencies] cachetools = ">=2.0.0,<7.0" pyasn1-modules = ">=0.2.1" +requests = {version = ">=2.20.0,<3.0.0", optional = true, markers = "extra == \"requests\""} rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] +cryptography = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)"] enterprise-cert = ["cryptography", "pyopenssl"] -pyjwt = ["cryptography (<39.0.0)", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] -pyopenssl = ["cryptography (<39.0.0)", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +pyjwt = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] +pyopenssl = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0)"] -testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0)", "cryptography (<39.0.0)", "cryptography (>=38.0.3)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] +testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] urllib3 = ["packaging", "urllib3"] [[package]] name = "google-auth-httplib2" -version = "0.2.0" +version = "0.3.0" description = "Google Authentication Library: httplib2 transport" optional = false -python-versions = "*" +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_auth_httplib2-0.3.0-py3-none-any.whl", hash = "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776"}, + {file = "google_auth_httplib2-0.3.0.tar.gz", hash = "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b"}, +] + +[package.dependencies] +google-auth = ">=1.32.0,<3.0.0" +httplib2 = ">=0.19.0,<1.0.0" + +[[package]] +name = "google-genai" +version = "1.56.0" +description = "GenAI Python SDK" +optional = false +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, - {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, + {file = "google_genai-1.56.0-py3-none-any.whl", hash = "sha256:9e6b11e0c105ead229368cb5849a480e4d0185519f8d9f538d61ecfcf193b052"}, + {file = "google_genai-1.56.0.tar.gz", hash = "sha256:0491af33c375f099777ae207d9621f044e27091fafad4c50e617eba32165e82f"}, ] [package.dependencies] -google-auth = "*" -httplib2 = ">=0.19.0" +anyio = ">=4.8.0,<5.0.0" +distro = ">=1.7.0,<2" +google-auth = {version = ">=2.45.0,<3.0.0", extras = ["requests"]} +httpx = ">=0.28.1,<1.0.0" +pydantic = ">=2.9.0,<3.0.0" +requests = ">=2.28.1,<3.0.0" +sniffio = "*" +tenacity = ">=8.2.3,<9.2.0" +typing-extensions = ">=4.11.0,<5.0.0" +websockets = ">=13.0.0,<15.1.0" + +[package.extras] +aiohttp = ["aiohttp (<3.13.3)"] +local-tokenizer = ["protobuf", "sentencepiece (>=0.2.0)"] [[package]] name = "google-generativeai" -version = "0.8.5" +version = "0.8.6" description = "Google Generative AI High level API client library and tools." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "google_generativeai-0.8.5-py3-none-any.whl", hash = "sha256:22b420817fb263f8ed520b33285f45976d5b21e904da32b80d4fd20c055123a2"}, + {file = "google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa"}, ] [package.dependencies] @@ -1123,14 +1153,14 @@ dev = ["Pillow", "absl-py", "black", "ipython", "nose2", "pandas", "pytype", "py [[package]] name = "googleapis-common-protos" -version = "1.71.0" +version = "1.72.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "googleapis_common_protos-1.71.0-py3-none-any.whl", hash = "sha256:59034a1d849dc4d18971997a72ac56246570afdd17f9369a0ff68218d50ab78c"}, - {file = "googleapis_common_protos-1.71.0.tar.gz", hash = "sha256:1aec01e574e29da63c80ba9f7bbf1ccfaacf1da877f23609fe236ca7c72a2e2e"}, + {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, + {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, ] [package.dependencies] @@ -1354,7 +1384,7 @@ httpcore = "==1.*" idna = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] @@ -1675,7 +1705,7 @@ PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} requests = ">=2.0.0,<3" [package.extras] -broker = ["pymsalruntime (>=0.14,<0.19)", "pymsalruntime (>=0.17,<0.19)", "pymsalruntime (>=0.18,<0.19)"] +broker = ["pymsalruntime (>=0.14,<0.19) ; python_version >= \"3.6\" and platform_system == \"Windows\"", "pymsalruntime (>=0.17,<0.19) ; python_version >= \"3.8\" and platform_system == \"Darwin\"", "pymsalruntime (>=0.18,<0.19) ; python_version >= \"3.8\" and platform_system == \"Linux\""] [[package]] name = "msal-extensions" @@ -2153,14 +2183,14 @@ files = [ [[package]] name = "proto-plus" -version = "1.26.1" +version = "1.27.0" description = "Beautiful, Pythonic protocol buffers" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, - {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, + {file = "proto_plus-1.27.0-py3-none-any.whl", hash = "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82"}, + {file = "proto_plus-1.27.0.tar.gz", hash = "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4"}, ] [package.dependencies] @@ -2250,7 +2280,7 @@ typing-inspection = ">=0.4.2" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" @@ -2811,6 +2841,22 @@ typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\"" [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +[[package]] +name = "tenacity" +version = "9.1.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, + {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tiktoken" version = "0.12.0" @@ -2959,7 +3005,7 @@ files = [ ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2983,12 +3029,12 @@ h11 = ">=0.8" httptools = {version = ">=0.6.3", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} -uvloop = {version = ">=0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +uvloop = {version = ">=0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "uvloop" @@ -2997,7 +3043,7 @@ description = "Fast implementation of asyncio event loop on top of libuv" optional = false python-versions = ">=3.8.1" groups = ["main"] -markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\"" +markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"" files = [ {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c"}, {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792"}, @@ -3404,4 +3450,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "b558e94d5d8bdcc4273f47c52c8bfa6f4e003df0cf754f56340b8b98283d4a8d" +content-hash = "1933fdd12b1645fa83d6e66b8f1683ca2d927836a0bacd2a631f04adefb115d2" diff --git a/api/pyproject.toml b/api/pyproject.toml index 09760f8b1..3fef92c11 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -12,7 +12,8 @@ python = "^3.11" fastapi = ">=0.95.0" uvicorn = { extras = ["standard"], version = ">=0.21.1" } pydantic = ">=2.0.0" -google-generativeai = ">=0.3.0" +google-genai = ">=1.0.0" +google-generativeai = ">=0.8.0" tiktoken = ">=0.5.0" adalflow = ">=0.1.0" numpy = ">=1.24.0" diff --git a/api/simple_chat.py b/api/simple_chat.py index 41a184ed8..f5f771d02 100644 --- a/api/simple_chat.py +++ b/api/simple_chat.py @@ -3,7 +3,8 @@ from typing import List, Optional from urllib.parse import unquote -import google.generativeai as genai +from google import genai +from google.genai import types as genai_types from adalflow.components.model_client.ollama_client import OllamaClient from adalflow.core.types import ModelType from fastapi import FastAPI, HTTPException @@ -11,7 +12,7 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field -from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, GOOGLE_API_KEY from api.data_pipeline import count_tokens, get_file_content from api.openai_client import OpenAIClient from api.openrouter_client import OpenRouterClient @@ -450,14 +451,23 @@ async def chat_completions_stream(request: ChatCompletionRequest): model_type=ModelType.LLM, ) else: - # Initialize Google Generative AI model (default provider) - model = genai.GenerativeModel( - model_name=model_config["model"], - generation_config={ - "temperature": model_config["temperature"], - "top_p": model_config["top_p"], - "top_k": model_config["top_k"], - }, + # Initialize Google Generative AI client (new google-genai SDK) + google_client = genai.Client(api_key=GOOGLE_API_KEY) + google_model_name = model_config["model"] + + # Build thinking_config if thinking_level is specified (for Gemini 3 models) + thinking_config = None + if "thinking_level" in model_config: + thinking_config = genai_types.ThinkingConfig( + thinking_level=model_config["thinking_level"].upper() + ) + logger.info(f"Using thinking_level: {model_config['thinking_level']}") + + google_generation_config = genai_types.GenerateContentConfig( + temperature=model_config["temperature"], + top_p=model_config["top_p"], + top_k=model_config["top_k"], + thinking_config=thinking_config ) # Create a streaming response @@ -550,10 +560,14 @@ async def response_stream(): "DASHSCOPE_WORKSPACE_ID) environment variables with valid values." ) else: - # Google Generative AI (default provider) - response = model.generate_content(prompt, stream=True) + # Google Generative AI (default provider) - using new google-genai SDK + response = google_client.models.generate_content_stream( + model=google_model_name, + contents=prompt, + config=google_generation_config + ) for chunk in response: - if hasattr(chunk, "text"): + if hasattr(chunk, "text") and chunk.text: yield chunk.text except Exception as e_outer: @@ -711,22 +725,31 @@ async def response_stream(): "DASHSCOPE_WORKSPACE_ID) environment variables with valid values." ) else: - # Google Generative AI fallback (default provider) + # Google Generative AI fallback (default provider) - using new google-genai SDK model_config = get_model_config(request.provider, request.model) - fallback_model = genai.GenerativeModel( - model_name=model_config["model_kwargs"]["model"], - generation_config={ - "temperature": model_config["model_kwargs"].get("temperature", 0.7), - "top_p": model_config["model_kwargs"].get("top_p", 0.8), - "top_k": model_config["model_kwargs"].get("top_k", 40), - }, + + # Build thinking_config if thinking_level is specified + fallback_thinking_config = None + if "thinking_level" in model_config["model_kwargs"]: + fallback_thinking_config = genai_types.ThinkingConfig( + thinking_level=model_config["model_kwargs"]["thinking_level"].upper() + ) + + fallback_generation_config = genai_types.GenerateContentConfig( + temperature=model_config["model_kwargs"].get("temperature", 0.7), + top_p=model_config["model_kwargs"].get("top_p", 0.8), + top_k=model_config["model_kwargs"].get("top_k", 40), + thinking_config=fallback_thinking_config ) - - fallback_response = fallback_model.generate_content( - simplified_prompt, stream=True + + fallback_client = genai.Client(api_key=GOOGLE_API_KEY) + fallback_response = fallback_client.models.generate_content_stream( + model=model_config["model_kwargs"]["model"], + contents=simplified_prompt, + config=fallback_generation_config ) for chunk in fallback_response: - if hasattr(chunk, "text"): + if hasattr(chunk, "text") and chunk.text: yield chunk.text except Exception as e2: logger.error(f"Error in fallback streaming response: {str(e2)}") diff --git a/api/websocket_wiki.py b/api/websocket_wiki.py index 6556bdf1d..fc32e764d 100644 --- a/api/websocket_wiki.py +++ b/api/websocket_wiki.py @@ -3,13 +3,14 @@ from typing import List, Optional, Dict, Any from urllib.parse import unquote -import google.generativeai as genai +from google import genai +from google.genai import types as genai_types from adalflow.components.model_client.ollama_client import OllamaClient from adalflow.core.types import ModelType from fastapi import WebSocket, WebSocketDisconnect, HTTPException from pydantic import BaseModel, Field -from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY +from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY, GOOGLE_API_KEY from api.data_pipeline import count_tokens, get_file_content from api.openai_client import OpenAIClient from api.openrouter_client import OpenRouterClient @@ -529,14 +530,23 @@ async def handle_websocket_chat(websocket: WebSocket): model_type=ModelType.LLM ) else: - # Initialize Google Generative AI model - model = genai.GenerativeModel( - model_name=model_config["model"], - generation_config={ - "temperature": model_config["temperature"], - "top_p": model_config["top_p"], - "top_k": model_config["top_k"] - } + # Initialize Google Generative AI client (new google-genai SDK) + google_client = genai.Client(api_key=GOOGLE_API_KEY) + google_model_name = model_config["model"] + + # Build thinking_config if thinking_level is specified (for Gemini 3 models) + thinking_config = None + if "thinking_level" in model_config: + thinking_config = genai_types.ThinkingConfig( + thinking_level=model_config["thinking_level"].upper() + ) + logger.info(f"Using thinking_level: {model_config['thinking_level']}") + + google_generation_config = genai_types.GenerateContentConfig( + temperature=model_config["temperature"], + top_p=model_config["top_p"], + top_k=model_config["top_k"], + thinking_config=thinking_config ) # Process the response based on the provider @@ -637,10 +647,14 @@ async def handle_websocket_chat(websocket: WebSocket): # Close the WebSocket connection after sending the error message await websocket.close() else: - # Google Generative AI (default provider) - response = model.generate_content(prompt, stream=True) + # Google Generative AI (default provider) - using new google-genai SDK + response = google_client.models.generate_content_stream( + model=google_model_name, + contents=prompt, + config=google_generation_config + ) for chunk in response: - if hasattr(chunk, 'text'): + if hasattr(chunk, 'text') and chunk.text: await websocket.send_text(chunk.text) await websocket.close() @@ -781,22 +795,31 @@ async def handle_websocket_chat(websocket: WebSocket): ) await websocket.send_text(error_msg) else: - # Google Generative AI fallback (default provider) + # Google Generative AI fallback (default provider) - using new google-genai SDK model_config = get_model_config(request.provider, request.model) - fallback_model = genai.GenerativeModel( - model_name=model_config["model_kwargs"]["model"], - generation_config={ - "temperature": model_config["model_kwargs"].get("temperature", 0.7), - "top_p": model_config["model_kwargs"].get("top_p", 0.8), - "top_k": model_config["model_kwargs"].get("top_k", 40), - }, + + # Build thinking_config if thinking_level is specified + fallback_thinking_config = None + if "thinking_level" in model_config["model_kwargs"]: + fallback_thinking_config = genai_types.ThinkingConfig( + thinking_level=model_config["model_kwargs"]["thinking_level"].upper() + ) + + fallback_generation_config = genai_types.GenerateContentConfig( + temperature=model_config["model_kwargs"].get("temperature", 0.7), + top_p=model_config["model_kwargs"].get("top_p", 0.8), + top_k=model_config["model_kwargs"].get("top_k", 40), + thinking_config=fallback_thinking_config ) - - fallback_response = fallback_model.generate_content( - simplified_prompt, stream=True + + fallback_client = genai.Client(api_key=GOOGLE_API_KEY) + fallback_response = fallback_client.models.generate_content_stream( + model=model_config["model_kwargs"]["model"], + contents=simplified_prompt, + config=fallback_generation_config ) for chunk in fallback_response: - if hasattr(chunk, "text"): + if hasattr(chunk, "text") and chunk.text: await websocket.send_text(chunk.text) except Exception as e2: logger.error(f"Error in fallback streaming response: {str(e2)}") From f45c03932afc98ddd190f456df79b7d893374b1b Mon Sep 17 00:00:00 2001 From: Diogo Soares Rodrigues Date: Wed, 17 Dec 2025 16:51:01 -0300 Subject: [PATCH 2/5] docs(ai-docs): add comprehensive AI-generated documentation for project architecture - Added 5 new documentation files in .ai/docs/ directory covering: - API analysis: Complete overview of FastAPI endpoints, authentication, and external dependencies - Data flow analysis: Data models, Pydantic schemas, and information flow patterns - Dependency analysis: External service integrations and configuration requirements - Request flow analysis: End-to-end request processing and streaming implementation - Structure analysis: High-level system architecture and component interactions These AI-generated docs provide developers with a comprehensive understanding of the project's architecture, API contracts, data models, and integration patterns, serving as both onboarding material and architectural reference documentation. --- .ai/docs/api_analysis.md | 134 ++++++++++++++++++++++ .ai/docs/data_flow_analysis.md | 101 ++++++++++++++++ .ai/docs/dependency_analysis.md | 110 ++++++++++++++++++ .ai/docs/request_flow_analysis.md | 68 +++++++++++ .ai/docs/structure_analysis.md | 70 +++++++++++ .cursor/rules/backend-rag-conventions.mdc | 31 +++++ .cursor/rules/frontend-conventions.mdc | 33 ++++++ .cursor/rules/project-overview.mdc | 36 ++++++ AGENTS.md | 37 ++++++ CLAUDE.md | 68 +++++++++++ 10 files changed, 688 insertions(+) create mode 100644 .ai/docs/api_analysis.md create mode 100644 .ai/docs/data_flow_analysis.md create mode 100644 .ai/docs/dependency_analysis.md create mode 100644 .ai/docs/request_flow_analysis.md create mode 100644 .ai/docs/structure_analysis.md create mode 100644 .cursor/rules/backend-rag-conventions.mdc create mode 100644 .cursor/rules/frontend-conventions.mdc create mode 100644 .cursor/rules/project-overview.mdc create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/.ai/docs/api_analysis.md b/.ai/docs/api_analysis.md new file mode 100644 index 000000000..7f348a8bc --- /dev/null +++ b/.ai/docs/api_analysis.md @@ -0,0 +1,134 @@ +# API Documentation + +This document provides a comprehensive overview of the APIs exposed and consumed by the `deepwiki-open` project. The project consists of a Next.js frontend and a Python FastAPI backend. + +## APIs Served by This Project + +The primary API is served by a Python FastAPI backend (default port 8001). The Next.js application (default port 3000) provides proxy routes to this backend for frontend consumption. + +### Endpoints + +#### 1. Chat Completion (Streaming) +- **Method and Path**: `POST /chat/completions/stream` (HTTP) or `WS /ws/chat` (WebSocket) +- **Description**: Generates a streaming chat response using RAG (Retrieval-Augmented Generation) based on a repository's content. +- **Request (Body)**: + ```json + { + "repo_url": "string", + "messages": [ + {"role": "user", "content": "string"} + ], + "filePath": "string (optional)", + "token": "string (optional, for private repos)", + "type": "github | gitlab | bitbucket (default: github)", + "provider": "google | openai | openrouter | ollama | bedrock | azure | dashscope", + "model": "string (optional)", + "language": "en | ja | zh | es | kr | vi (default: en)", + "excluded_dirs": "string (newline separated)", + "excluded_files": "string (newline separated)", + "included_dirs": "string (newline separated)", + "included_files": "string (newline separated)" + } + ``` +- **Response**: + - **HTTP**: `text/event-stream` containing chunks of the generated text. + - **WebSocket**: Text messages containing chunks of the generated text. +- **Authentication**: Requires valid LLM provider API keys configured on the server. +- **Example**: + ```bash + curl -X POST http://localhost:8001/chat/completions/stream \ + -H "Content-Type: application/json" \ + -d '{"repo_url": "https://github.com/user/repo", "messages": [{"role": "user", "content": "Explain this project"}]}' + ``` + +#### 2. Model Configuration +- **Method and Path**: `GET /models/config` +- **Description**: Retrieves available LLM providers and their supported models. +- **Response**: + ```json + { + "providers": [ + { + "id": "google", + "name": "Google", + "supportsCustomModel": true, + "models": [{"id": "gemini-2.5-flash", "name": "gemini-2.5-flash"}] + } + ], + "defaultProvider": "google" + } + ``` + +#### 3. Wiki Cache Management +- **GET /api/wiki_cache**: Retrieves cached wiki data for a repository. + - **Params**: `owner`, `repo`, `repo_type`, `language`. +- **POST /api/wiki_cache**: Stores generated wiki data in the server-side cache. +- **DELETE /api/wiki_cache**: Deletes a specific wiki cache. + - **Params**: `owner`, `repo`, `repo_type`, `language`, `authorization_code` (if auth enabled). + +#### 4. Processed Projects +- **Method and Path**: `GET /api/processed_projects` +- **Description**: Lists all projects that have been processed and cached on the server. +- **Response**: A list of project entries including `owner`, `repo`, `language`, and `submittedAt`. + +#### 5. Wiki Export +- **Method and Path**: `POST /export/wiki` +- **Description**: Exports wiki content as a downloadable Markdown or JSON file. +- **Request**: + ```json + { + "repo_url": "string", + "pages": [...], + "format": "markdown | json" + } + ``` + +#### 6. Authentication & Status +- **GET /auth/status**: Checks if the application requires an authorization code (`DEEPWIKI_AUTH_MODE`). +- **POST /auth/validate**: Validates the provided authorization code. +- **GET /health**: Returns the health status of the API service. + +--- + +### Authentication & Security + +- **Internal Authentication**: Controlled by `DEEPWIKI_AUTH_MODE` and `DEEPWIKI_AUTH_CODE` environment variables. If enabled, certain operations (like deleting cache) require the code. +- **External API Keys**: The backend requires API keys for the LLM providers (e.g., `OPENAI_API_KEY`, `GOOGLE_API_KEY`). These are stored as environment variables on the server. +- **Repository Access**: For private repositories, users can provide a Personal Access Token (`token` in request body), which is used by the backend to clone the repository via HTTPS. + +### Rate Limiting & Constraints + +- **Token Limits**: The backend includes a token counter (`tiktoken`). Requests exceeding ~8000 tokens trigger a warning or may be truncated depending on the provider. +- **Concurrency**: The Python backend uses FastAPI with `uvicorn`, supporting asynchronous request handling. + +--- + +## External API Dependencies + +### Services Consumed + +| Service Name | Purpose | Configuration | Authentication | +| :--- | :--- | :--- | :--- | +| **Google Generative AI** | Default LLM provider (Gemini) | `GOOGLE_API_KEY` | API Key | +| **OpenAI** | LLM and Embeddings | `OPENAI_API_KEY` | API Key | +| **OpenRouter** | Unified LLM access | `OPENROUTER_API_KEY` | API Key | +| **AWS Bedrock** | Enterprise LLM access | `AWS_ACCESS_KEY_ID`, `AWS_REGION`, etc. | IAM Credentials | +| **Azure AI** | Enterprise LLM access | `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` | API Key | +| **Alibaba Dashscope** | Qwen models | `DASHSCOPE_API_KEY` | API Key | +| **Ollama** | Local LLM execution | `OLLAMA_HOST` (default: localhost:11434) | None (Local) | +| **GitHub/GitLab/Bitbucket** | Source code retrieval | `git clone` via subprocess | Optional PAT | + +### Integration Patterns + +- **RAG Pipeline**: The project uses `adalflow` to manage the RAG pipeline. It clones repositories, splits text, generates embeddings, and stores them in a local vector database for retrieval during chat. +- **Streaming**: Both HTTP (Server-Sent Events) and WebSockets are used to provide real-time streaming of LLM responses to the frontend. +- **Proxying**: The Next.js frontend proxies requests to the Python backend to avoid CORS issues and centralize API management. + +--- + +## Available Documentation + +- **API Analysis**: Located at `/.ai/docs/api_analysis.md`. +- **Structure Analysis**: Located at `/.ai/docs/structure_analysis.md`. +- **OpenAPI Spec**: FastAPI automatically generates an OpenAPI spec at `/openapi.json` and interactive docs at `/docs` (Swagger UI) when the backend is running. +- **Quality**: The internal documentation in `.ai/docs/` provides a high-level overview of the system architecture and data flow. The code itself is well-structured with Pydantic models defining request/response contracts. \ No newline at end of file diff --git a/.ai/docs/data_flow_analysis.md b/.ai/docs/data_flow_analysis.md new file mode 100644 index 000000000..5e31ed63c --- /dev/null +++ b/.ai/docs/data_flow_analysis.md @@ -0,0 +1,101 @@ +# Data Flow Analysis + +## Data Models Overview + +The application uses a consistent set of data models across the Python (FastAPI) backend and the TypeScript (Next.js) frontend to represent repositories, wiki structures, and chat interactions. + +### Core Backend Models (Pydantic) +- **`WikiPage`**: Represents a single documentation page. + - `id`, `title`, `content`, `filePaths` (List[str]), `importance` (high/medium/low), `relatedPages` (List[str]). +- **`WikiStructureModel`**: Defines the hierarchy of the generated wiki. + - `id`, `title`, `description`, `pages` (List[WikiPage]), `sections` (List[WikiSection]), `rootSections` (List[str]). +- **`RepoInfo`**: Captures repository metadata. + - `owner`, `repo`, `type` (github/gitlab/bitbucket/local), `token` (optional), `localPath`, `repoUrl`. +- **`ChatCompletionRequest`**: Standardizes the payload for chat interactions. + - `repo_url`, `messages` (List[ChatMessage]), `filePath` (optional), `token`, `provider`, `model`, `language`, `filters` (included/excluded dirs/files). +- **`WikiCacheData`**: The structure used for persisting generated wikis. + - `wiki_structure`, `generated_pages` (Dict[str, WikiPage]), `repo`, `provider`, `model`. + +### Core Frontend Models (TypeScript) +- **`WikiPage`**, **`WikiStructure`**, **`RepoInfo`**: Mirror the backend Pydantic models for type safety. +- **`Message`**: Represents a chat message with `role` (user/assistant/system) and `content`. +- **`ResearchStage`**: Used in "Deep Research" mode to track the progress of multi-turn investigations. + +## Data Transformation Map + +Data undergoes several transformations as it moves from raw repository files to structured documentation and AI-generated responses. + +1. **Repository Ingestion**: + - Raw files are fetched via `git clone` or API calls (GitHub/GitLab/Bitbucket). + - `read_all_documents` transforms files into `Document` objects, attaching metadata like `file_path`, `token_count`, and `is_code`. +2. **RAG Pipeline Transformation**: + - `TextSplitter` breaks documents into manageable chunks based on token limits. + - `ToEmbeddings` (via `adalflow`) transforms text chunks into vector representations using the configured embedder (OpenAI, Google, or Ollama). + - `LocalDB` persists these transformed documents and their vectors into a FAISS index. +3. **Retrieval Transformation**: + - User queries are embedded using the same embedding model. + - `FAISSRetriever` performs a similarity search to find the most relevant document chunks. +4. **Generation Transformation**: + - A prompt is constructed by combining a `system_prompt`, `conversation_history`, `retrieved_context`, and the `user_query`. + - The LLM transforms this prompt into a structured `RAGAnswer` or a streamed text response. +5. **Wiki Generation**: + - The repository structure is analyzed by the LLM to generate a `WikiStructureModel`. + - Each page in the structure is then individually generated by the LLM using relevant file contexts. + +## Storage Interactions + +The system utilizes both server-side and client-side storage for persistence and performance. + +### Server-Side Storage (`~/.adalflow/`) +- **`/repos/`**: Stores cloned Git repositories. +- **`/databases/`**: Stores serialized `LocalDB` states (FAISS indices and document metadata) as `.pkl` files. +- **`/wikicache/`**: Stores generated wiki data as JSON files, named using the pattern `deepwiki_cache_{repo_type}_{owner}_{repo}_{language}.json`. + +### Client-Side Storage +- **`localStorage`**: The frontend caches generated wiki structures and pages to provide near-instant loading for previously visited repositories. +- **Session State**: React state manages the current conversation history and active wiki page during a user session. + +## Validation Mechanisms + +Data integrity and system stability are maintained through several validation layers: + +- **Pydantic Validation**: All API endpoints use Pydantic models to validate incoming request bodies and outgoing responses. +- **Token Limit Validation**: `tiktoken` is used to estimate token counts for user queries and document chunks, ensuring they stay within LLM and embedding model limits (e.g., `MAX_INPUT_TOKENS = 7500`). +- **Embedding Consistency**: The `RAG` component includes a `_validate_and_filter_embeddings` method that ensures all vectors in the FAISS index have consistent dimensions, filtering out any documents that failed to embed correctly. +- **Authentication**: Sensitive operations like deleting a wiki cache are protected by `WIKI_AUTH_MODE` and `WIKI_AUTH_CODE` environment variables. +- **File Filtering**: Inclusion and exclusion rules (e.g., `DEFAULT_EXCLUDED_DIRS`) prevent sensitive or irrelevant files (like `.git`, `node_modules`, or binary files) from entering the data pipeline. + +## State Management Analysis + +### Backend State +- **`Memory` Component**: Manages conversation state within the `RAG` pipeline using a list of `DialogTurn` objects. +- **`DatabaseManager`**: Maintains the state of the currently loaded vector database and repository paths. + +### Frontend State +- **`Ask` Component**: Manages `conversationHistory` (array of `Message`) and `researchStages` for the Deep Research feature. +- **`RepoWikiPage`**: Manages the `wikiStructure`, `generatedPages` (a map of page IDs to content), and `pagesInProgress` (a set of IDs currently being generated to prevent duplicate requests). +- **`LanguageContext`**: Manages the application's internationalization state. + +## Serialization Processes + +- **JSON Serialization**: Used for all REST API communication, WebSocket messages, and the persistence of wiki cache files. +- **Pickle Serialization**: Used by `adalflow`'s `LocalDB` to save and load the state of the FAISS index and document objects. +- **Base64 Encoding**: Used when fetching file contents via the GitHub API. +- **URL Encoding**: Used for safely passing repository URLs and file paths in API requests. + +## Data Lifecycle Diagrams + +### Chat Interaction Lifecycle +1. **Input**: User sends a message via the `Ask` component. +2. **Transport**: Message is wrapped in a `ChatCompletionRequest` and sent over a **WebSocket** (`/ws/chat`). +3. **Processing**: Backend prepares the `RAG` instance, retrieves context from the FAISS index, and constructs the LLM prompt. +4. **Generation**: LLM generates a response, which is streamed back to the client. +5. **Update**: Frontend appends the response to `conversationHistory` and renders it via the `Markdown` component. + +### Wiki Generation Lifecycle +1. **Trigger**: User navigates to a repository page. +2. **Cache Check**: Frontend checks `localStorage`, then queries `/api/wiki_cache`. +3. **Structure Generation**: If not cached, backend analyzes the repo and returns a `WikiStructureModel`. +4. **Page Generation**: Frontend iterates through the structure, requesting content for each `WikiPage` via the backend. +5. **Persistence**: As pages are generated, they are saved to the server-side cache and the client's `localStorage`. +6. **Display**: `WikiTreeView` renders the hierarchy, and selecting a page displays its content in the main view. \ No newline at end of file diff --git a/.ai/docs/dependency_analysis.md b/.ai/docs/dependency_analysis.md new file mode 100644 index 000000000..fe8cfd2c3 --- /dev/null +++ b/.ai/docs/dependency_analysis.md @@ -0,0 +1,110 @@ +# Dependency Analysis + +## Internal Dependencies Map + +The project follows a clear separation between the frontend (Next.js) and the backend (FastAPI), with a layered architecture in the backend to handle RAG (Retrieval-Augmented Generation) and data processing. + +### Backend (Python) +- **Entry Point**: `api/main.py` initializes the environment and starts the `uvicorn` server, importing the FastAPI app from `api/api.py`. +- **API Layer**: `api/api.py` defines REST endpoints for configuration, authentication, and project management. It delegates real-time chat to `api/websocket_wiki.py`. +- **WebSocket Layer**: `api/websocket_wiki.py` handles bidirectional communication for chat, instantiating the `RAG` component for each session. +- **Logic Layer (RAG)**: `api/rag.py` contains the core RAG logic using the `adalflow` framework. It relies on: + - `api/data_pipeline.py`: For repository cloning, document processing, and database management. + - `api/tools/embedder.py`: A factory for creating embedder instances. + - `api/prompts.py`: Centralized storage for system prompts and templates. +- **Data Layer**: `api/data_pipeline.py` manages the lifecycle of repository data, from `git clone` to tokenization and embedding. +- **Client Layer**: `api/config.py` maps provider IDs to specific client implementations (e.g., `OpenAIClient`, `GoogleGenAIClient`). Custom clients are located in `api/*_client.py`. + +### Frontend (TypeScript/Next.js) +- **App Router**: `src/app` contains the page definitions. + - `page.tsx`: Landing page and repository configuration. + - `[owner]/[repo]/page.tsx`: Main wiki view. +- **Components**: `src/components` contains reusable UI elements. + - `Ask.tsx`: Chat interface, uses `src/utils/websocketClient.ts`. + - `WikiTreeView.tsx`: Navigation for the generated wiki. + - `Mermaid.tsx`: Renders architecture diagrams using the `mermaid` library. +- **State & Logic**: + - `src/contexts/LanguageContext.tsx`: Manages i18n state. + - `src/hooks/useProcessedProjects.ts`: Fetches and manages the list of analyzed repositories. + - `src/utils/websocketClient.ts`: Encapsulates WebSocket logic for backend communication. + +## External Libraries Analysis + +### Backend (Python) +- **Core Framework**: `adalflow` (LLM/RAG orchestration), `fastapi` (Web API), `uvicorn` (ASGI server). +- **LLM SDKs**: `google-generativeai`, `openai`, `ollama`, `boto3` (AWS Bedrock), `azure-identity`, `azure-core`. +- **Data Processing**: `faiss-cpu` (Vector DB), `tiktoken` (Token counting), `numpy`, `langid` (Language detection). +- **Utilities**: `pydantic` (Data validation), `python-dotenv` (Env management), `aiohttp` (Async HTTP), `websockets` (WebSocket protocol). + +### Frontend (React/Next.js) +- **Framework**: `next` (v15.3.1), `react` (v19). +- **Visualization**: `mermaid` (Diagrams), `svg-pan-zoom`. +- **Content Rendering**: `react-markdown`, `rehype-raw`, `remark-gfm`, `react-syntax-highlighter`. +- **Internationalization**: `next-intl`. +- **Styling**: `tailwindcss`, `next-themes`. + +## Service Integrations + +- **LLM Providers**: + - **Google**: Gemini models via `google-generativeai`. + - **OpenAI**: GPT models via `openai`. + - **OpenRouter**: Aggregator for various models. + - **Ollama**: Local LLM execution. + - **AWS Bedrock**: Enterprise LLM access. + - **Azure AI**: Microsoft's AI services. + - **DashScope**: Alibaba Cloud's AI models. +- **Git Platforms**: + - **GitHub/GitLab/Bitbucket**: Integrated via `git` CLI for cloning and personal access tokens for private repo access. + +## Dependency Injection Patterns + +- **Factory Pattern**: The backend uses a factory pattern for creating LLM clients and embedders. `api/config.py` maintains a mapping of client classes, and `api/tools/embedder.py` provides a `get_embedder` function that resolves the correct instance based on environment configuration. +- **Adalflow Components**: The `RAG` class in `api/rag.py` inherits from `adal.Component`, allowing it to compose other `adalflow` building blocks (like `FAISSRetriever` and `Memory`) in a declarative manner. +- **Context Providers**: The frontend uses React Context (`LanguageContext`) to inject internationalization state and functions throughout the component tree. + +## Module Coupling Assessment + +- **Backend Modularity**: High. The separation of clients, data pipeline, and RAG logic allows for easy addition of new LLM providers or data sources. +- **Frontend-Backend Coupling**: Moderate. The frontend is coupled to the backend's API schema (Pydantic models). Changes in backend models (e.g., `WikiPage`, `RepoInfo`) require manual updates to TypeScript interfaces in `src/types`. +- **Framework Coupling**: High. The backend is deeply integrated with `adalflow`. While this provides powerful RAG capabilities, it makes the core logic dependent on the `adalflow` API. +- **Cohesion**: High. Each module has a well-defined responsibility (e.g., `data_pipeline.py` only handles data ingestion and processing). + +## Dependency Graph + +```mermaid +graph TD + subgraph Frontend + UI[Next.js Pages] --> Comp[React Components] + Comp --> Hooks[Custom Hooks] + Comp --> WS_Client[WebSocket Client] + Comp --> Mermaid[Mermaid.js] + end + + subgraph Backend_API + WS_Handler[WebSocket Handler] --> RAG_Comp[RAG Component] + REST_API[FastAPI Endpoints] --> Config[Config Manager] + end + + subgraph Backend_Logic + RAG_Comp --> Retriever[FAISS Retriever] + RAG_Comp --> Memory[Conversation Memory] + RAG_Comp --> Embedder_Factory[Embedder Factory] + RAG_Comp --> Pipeline[Data Pipeline] + end + + subgraph External_Services + Pipeline --> Git[Git CLI / Repo Providers] + Embedder_Factory --> LLM_Clients[LLM Providers: OpenAI, Google, etc.] + LLM_Clients --> Adalflow[Adalflow Framework] + end + + WS_Client -.-> WS_Handler + UI -.-> REST_API +``` + +## Potential Dependency Issues + +- **Adalflow Versioning**: The project relies on `adalflow >= 0.1.0`. As a relatively new framework, breaking changes in `adalflow` could significantly impact the `RAG` and `DataPipeline` implementations. +- **Duplicate Model Definitions**: Data models are defined in Python (Pydantic) and TypeScript. This duplication increases the risk of desynchronization during API updates. +- **Manual Client Mapping**: The mapping of providers to client classes in `api/config.py` is manual. While functional, it requires code changes to support new providers even if they follow standard protocols (like OpenAI-compatible APIs). +- **Local Environment Dependencies**: The backend relies on the `git` binary being present in the system path for repository cloning, which might cause issues in restricted container environments if not properly configured. \ No newline at end of file diff --git a/.ai/docs/request_flow_analysis.md b/.ai/docs/request_flow_analysis.md new file mode 100644 index 000000000..76acce6e6 --- /dev/null +++ b/.ai/docs/request_flow_analysis.md @@ -0,0 +1,68 @@ +# Request Flow Analysis + +## Entry Points Overview +The system exposes two primary layers of entry points: the Next.js frontend and the FastAPI backend. + +- **Frontend Entry Points (Next.js - Port 3000)**: + - **Web UI**: The main application interface served at the root and dynamic repository routes (e.g., `/[owner]/[repo]`). + - **API Proxy Routes**: Located in `src/app/api/`, these routes act as proxies to the backend: + - `POST /api/chat/stream`: Fallback HTTP streaming for chat. + - `GET /api/models/config`: Fetches available LLM providers and models. + - `GET /api/wiki/projects`: Lists processed repositories. + - `GET /api/auth/status` & `POST /api/auth/validate`: Authentication checks. + +- **Backend Entry Points (FastAPI - Port 8001)**: + - **WebSocket**: `WS /ws/chat` - The primary entry point for real-time, streaming RAG chat. + - **REST API**: + - `POST /chat/completions/stream`: HTTP streaming chat endpoint. + - `GET /models/config`: Returns the system's LLM configuration. + - `GET/POST/DELETE /api/wiki_cache`: Manages the server-side JSON cache for generated wikis. + - `GET /api/processed_projects`: Scans the cache directory to list all indexed repositories. + - `POST /export/wiki`: Generates downloadable Markdown or JSON exports. + - `GET /local_repo/structure`: Analyzes local directories for the "Local Path" feature. + +## Request Routing Map +- **Client-to-Proxy**: The frontend UI components (like `Ask.tsx` or `ProcessedProjects.tsx`) typically call the Next.js `/api/*` routes. +- **Proxy-to-Backend**: Next.js routes (e.g., `src/app/api/chat/stream/route.ts`) use the `fetch` API to forward requests to the `TARGET_SERVER_BASE_URL` (defaulting to `http://localhost:8001`). +- **Direct WebSocket**: For the core chat functionality, `src/utils/websocketClient.ts` bypasses the Next.js proxy and establishes a direct `WebSocket` connection to the backend's `/ws/chat` endpoint. +- **Backend Internal Routing**: FastAPI routes requests based on the path and method defined in `api/api.py`. It uses `app.add_api_route` and `app.add_websocket_route` to link paths to handler functions in `api/simple_chat.py` and `api/websocket_wiki.py`. + +## Middleware Pipeline +- **CORS Middleware**: The FastAPI application in `api/api.py` uses `CORSMiddleware` configured with `allow_origins=["*"]`. This is critical for allowing the Next.js frontend (on port 3000) to communicate with the backend (on port 8001) and for direct WebSocket connections. +- **Next.js Request Handling**: While no custom `middleware.ts` is present, Next.js handles standard request preprocessing, including environment variable injection and route matching. +- **Logging Middleware**: A custom logging configuration in `api/logging_config.py` sets up a `RotatingFileHandler`. It includes a filter (`IgnoreLogChangeDetectedFilter`) to suppress development-time noise from file watchers. + +## Controller/Handler Analysis +- **`api/api.py`**: The main controller for administrative and metadata operations. It handles Pydantic model validation for wiki structures and manages the filesystem-based cache in `~/.adalflow/wikicache`. +- **`api/websocket_wiki.py`**: The primary handler for the chat lifecycle. It: + 1. Accepts the WebSocket connection. + 2. Parses the `ChatCompletionRequest`. + 3. Initializes the `RAG` component. + 4. Streams LLM chunks back to the client. +- **`api/rag.py`**: The core logic controller for the RAG pipeline. It orchestrates the `DatabaseManager` for retrieval and manages conversation `Memory`. +- **`api/data_pipeline.py`**: The low-level handler for data operations. It manages `git clone` operations, file parsing, and interacts with the vector database (FAISS). + +## Authentication & Authorization Flow +- **System-Level Auth**: Controlled by `WIKI_AUTH_MODE`. If enabled, the `WIKI_AUTH_CODE` must be provided for sensitive operations like deleting a wiki cache (`DELETE /api/wiki_cache`). +- **Repository-Level Auth**: The `ChatCompletionRequest` and `WikiCacheRequest` models include an optional `token` field. This Personal Access Token is used by `download_repo` in `api/data_pipeline.py` to authenticate Git clones for private GitHub, GitLab, or Bitbucket repositories. +- **Provider Auth**: LLM provider authentication (e.g., OpenAI, Google Gemini) is handled server-side via environment variables (`OPENAI_API_KEY`, `GOOGLE_API_KEY`) which are used to initialize the respective AI clients. + +## Error Handling Pathways +- **REST API Errors**: Handled via FastAPI's `HTTPException`. Errors are returned as JSON objects with a `detail` field and appropriate HTTP status codes (e.g., 401 for unauthorized, 404 for missing cache). +- **WebSocket Errors**: If an error occurs during the WebSocket session (e.g., "No valid documents with embeddings found"), the backend sends a descriptive text message to the client and then calls `websocket.close()`. +- **Proxy Errors**: The Next.js proxy routes include `try-catch` blocks that capture backend failures or network issues, returning a 500 status with the error message to the frontend. +- **Validation Errors**: Pydantic models in the backend automatically catch and return 422 Unprocessable Entity errors if the request body does not match the expected schema. + +## Request Lifecycle Diagram +1. **User Action**: User enters a repository URL and a question in the `Ask` component. +2. **Connection**: The frontend `websocketClient` initiates a connection to `ws://localhost:8001/ws/chat`. +3. **Handshake**: FastAPI accepts the connection and the client sends the `ChatCompletionRequest` JSON. +4. **Retrieval Preparation**: + - Backend checks if the repository is already cloned and indexed. + - If not, `data_pipeline.py` clones the repo and `RAG.prepare_retriever` builds the vector index. +5. **Context Retrieval**: The `RAG` component searches the index for documents relevant to the user's query. +6. **Prompt Construction**: A system prompt is generated, combining the user's query, conversation history from `Memory`, and the retrieved code context. +7. **Streaming Response**: + - The backend calls the configured LLM provider (e.g., Google Gemini). + - As chunks of text arrive from the LLM, they are immediately sent over the WebSocket to the frontend. +8. **Termination**: Once the LLM finishes the response, the backend closes the WebSocket, and the frontend updates the UI with the complete message. \ No newline at end of file diff --git a/.ai/docs/structure_analysis.md b/.ai/docs/structure_analysis.md new file mode 100644 index 000000000..2713d63c1 --- /dev/null +++ b/.ai/docs/structure_analysis.md @@ -0,0 +1,70 @@ +# Code Structure Analysis + +## Architectural Overview +The **DeepWiki-Open** project is a full-stack application designed to generate and interact with documentation (wikis) for software repositories using Retrieval-Augmented Generation (RAG). The architecture is split into two main parts: + +1. **Frontend (Next.js)**: A React-based web interface using the App Router. It provides a dynamic UI for exploring repository structures and a chat interface for querying the codebase. It communicates with the backend via REST APIs for configuration and WebSockets for real-time, streaming chat completions. +2. **Backend (FastAPI)**: A Python-based API that orchestrates the RAG pipeline. It leverages the `adalflow` library to build modular data processing and retrieval components. The backend is provider-agnostic, supporting multiple LLM and embedding providers (OpenAI, Google Gemini, Ollama, Azure, OpenRouter, etc.). + +The system follows a **modular monolith** pattern where core responsibilities (data ingestion, retrieval, generation, and API serving) are separated into distinct modules within the `api/` directory. + +## Core Components +### Backend (`/api`) +- **`api.py`**: The central FastAPI application defining REST endpoints for wiki management, configuration retrieval, and authentication. +- **`rag.py`**: Implements the RAG pipeline using `adalflow`. It defines the `RAG` component, `Memory` for conversation history, and custom data classes for structured answers. +- **`data_pipeline.py`**: Handles repository lifecycle management, including cloning (GitHub, GitLab, Bitbucket), document parsing, token counting, and vector database management via the `DatabaseManager`. +- **`websocket_wiki.py`**: Manages real-time WebSocket connections for chat, handling the orchestration between user queries, RAG retrieval, and LLM generation. +- **`config.py`**: A centralized configuration manager that loads JSON settings from `/api/config/` and handles environment variable substitution. +- **`prompts.py`**: Contains all system prompts and templates for RAG, simple chat, and the "Deep Research" multi-turn process. + +### Frontend (`/src`) +- **`src/app`**: Contains the page routes, including dynamic routes for specific repositories (`[owner]/[repo]`) and specialized views like `slides` and `workshop`. +- **`src/components`**: Reusable UI components such as `WikiTreeView`, `Ask` (chat interface), `Markdown` (rendering), and `ModelSelectionModal`. +- **`src/utils/websocketClient.ts`**: A dedicated client for managing WebSocket communication with the backend. + +## Service Definitions +- **Wiki Service**: Managed through `api.py`, it handles the creation, caching, and exporting (Markdown/JSON) of wiki structures. +- **Chat Service**: Implemented in `websocket_wiki.py` (WebSocket) and `simple_chat.py` (HTTP Stream), providing a stateful, streaming chat experience. +- **Data Pipeline Service**: Encapsulated in `data_pipeline.py`, responsible for transforming raw repository files into searchable embeddings stored in a local FAISS index. +- **Model Configuration Service**: Provides a unified interface to query available LLM providers and models via the `/models/config` endpoint, dynamically resolved from `generator.json`. +- **Deep Research Service**: A specialized logic within the chat handlers that uses multi-turn prompts to perform deep, iterative analysis of specific topics. + +## Interface Contracts +- **Pydantic Models (`api.py`, `websocket_wiki.py`)**: + - `WikiPage`: Defines the structure of a single documentation page. + - `WikiStructureModel`: Defines the hierarchy and metadata of the entire wiki. + - `RepoInfo`: Captures repository details (owner, repo, type, token). + - `ChatCompletionRequest`: Standardizes the payload for chat interactions, including model parameters and file filters. +- **WebSocket Protocol**: A JSON-based message exchange format for real-time chat, where the client sends a `ChatCompletionRequest` and receives streamed text responses. +- **Adalflow Components**: The `RAG` class and `Memory` class follow the `adal.Component` and `adal.DataComponent` interfaces, ensuring compatibility with the Adalflow ecosystem. + +## Design Patterns Identified +- **Strategy Pattern**: Used in the LLM and Embedder clients (`openai_client.py`, `google_embedder_client.py`, `azureai_client.py`, etc.) to provide a uniform interface for different AI providers. +- **Factory Pattern**: `get_embedder` (in `api/tools/embedder.py`) and `load_generator_config` act as factories for creating configured clients based on environment settings. +- **Repository Pattern**: The `DatabaseManager` abstracts the complexities of Git operations and vector database persistence. +- **Component-based Architecture**: Both in React (frontend) and Adalflow (backend), where logic is encapsulated into reusable, composable units. +- **Observer Pattern**: Utilized via WebSockets to push real-time updates from the backend to the frontend during long-running generation tasks. + +## Component Relationships +- **Frontend `Ask` Component → `websocketClient` → Backend `websocket_wiki.py`**: The primary path for user interaction. +- **`websocket_wiki.py` → `RAG` Component**: The WebSocket handler instantiates and calls the RAG pipeline for each request. +- **`RAG` → `FAISSRetriever` & `Memory`**: The RAG component orchestrates retrieval from the vector store and manages conversation state. +- **`FAISSRetriever` → `DatabaseManager`**: The retriever relies on the data pipeline to ensure the repository is cloned and indexed. +- **`DatabaseManager` → `Embedder`**: The pipeline uses the configured embedder to generate vectors for repository documents. + +## Key Methods & Functions +- **`RAG.prepare_retriever` (`api/rag.py`)**: Orchestrates the setup of the vector database for a given repository, including cloning and indexing. +- **`handle_websocket_chat` (`api/websocket_wiki.py`)**: The main entry point for processing chat queries, performing retrieval, and streaming LLM responses. +- **`download_repo` (`api/data_pipeline.py`)**: A robust utility for cloning GitHub, GitLab, and Bitbucket repositories with support for access tokens. +- **`read_all_documents` (`api/data_pipeline.py`)**: Recursively parses repository files while respecting complex exclusion/inclusion filters defined in `repo.json`. +- **`get_embedder` (`api/tools/embedder.py`)**: Dynamically initializes the correct embedder (OpenAI, Google, or Ollama) based on configuration. + +## Available Documentation +- **`README.md`**: High-level project overview and setup instructions. +- **`api/README.md`**: Documentation specifically for the backend API and RAG logic. +- **`Ollama-instruction.md`**: Detailed guide for running the system with local Ollama models. +- **`tests/README.md`**: Instructions for running unit and integration tests. +- **`.ai/docs/api_analysis.md`**: Detailed analysis of the API endpoints and data models. +- **`.ai/docs/structure_analysis.md`**: Previous structural analysis of the codebase. + +**Documentation Quality**: The documentation is high-quality and comprehensive. It covers installation, configuration, and specific deployment scenarios (like local LLMs). The presence of multi-language READMEs and specialized guides for Ollama indicates a mature project with a focus on developer experience. The code is well-structured with clear naming conventions and descriptive docstrings in key modules. \ No newline at end of file diff --git a/.cursor/rules/backend-rag-conventions.mdc b/.cursor/rules/backend-rag-conventions.mdc new file mode 100644 index 000000000..dbf712ff6 --- /dev/null +++ b/.cursor/rules/backend-rag-conventions.mdc @@ -0,0 +1,31 @@ +--- +description: Rules for Python backend development, Adalflow RAG pipeline, and API conventions. +globs: + - "api/**/*.py" +alwaysApply: false +--- + +# Backend & RAG Conventions + +This project uses FastAPI and the `adalflow` framework to build a robust RAG pipeline. + +## Python & FastAPI Patterns +- **Pydantic Models**: Use Pydantic for all request/response schemas. Define them in `api/api.py` or `api/websocket_wiki.py`. +- **Async Handlers**: Use `async def` for API endpoints and WebSocket handlers to maintain high concurrency. +- **Error Handling**: Use `fastapi.HTTPException` for REST errors. For WebSockets, send a descriptive text message before closing the connection. + +## Adalflow & RAG Logic +- **Component Pattern**: The `RAG` class in `api/rag.py` must inherit from `adal.Component`. +- **Memory**: Use the `Memory` class (from `adalflow`) for conversation history management. +- **Data Pipeline**: Use `api/data_pipeline.py` for repository cloning and document parsing. Respect the exclusion filters defined in `repo.json`. +- **Embedders**: Always initialize embedders via `get_embedder` in `api/tools/embedder.py`. +- **Prompts**: Centralize all LLM prompts in `api/prompts.py`. Use templates for dynamic content. + +## Vector Database (FAISS) +- Vector indices are managed by `DatabaseManager` in `api/data_pipeline.py`. +- Ensure embedding dimensions are validated before insertion (see `_validate_and_filter_embeddings` in `api/rag.py`). + +## Anti-Patterns +- Do not hardcode API keys; use environment variables via `api/config.py`. +- Avoid blocking synchronous calls in async routes (use `run_in_threadpool` if necessary). +- Do not bypass the `DatabaseManager` for file system operations in `~/.adalflow`. \ No newline at end of file diff --git a/.cursor/rules/frontend-conventions.mdc b/.cursor/rules/frontend-conventions.mdc new file mode 100644 index 000000000..821819324 --- /dev/null +++ b/.cursor/rules/frontend-conventions.mdc @@ -0,0 +1,33 @@ +--- +description: Rules for Next.js frontend, TypeScript types, and UI component development. +globs: + - "src/**/*.{ts,tsx}" +alwaysApply: false +--- + +# Frontend & UI Conventions + +The frontend is a Next.js application focused on repository visualization and AI interaction. + +## Next.js & React Patterns +- **App Router**: Use the `src/app` directory for routing. Dynamic repository routes follow the `[owner]/[repo]` pattern. +- **Server vs. Client**: Use `"use client"` only for components requiring state, effects, or browser APIs (e.g., `Ask.tsx`, `WikiTreeView.tsx`). +- **TypeScript**: Strictly type all component props and API responses. Sync with backend Pydantic models. + +## AI & Chat UI +- **WebSockets**: Use `src/utils/websocketClient.ts` for all chat interactions. Do not implement raw WebSocket logic in components. +- **Markdown Rendering**: Use the `Markdown` component for AI responses to support syntax highlighting and Mermaid diagrams. +- **Streaming**: Handle partial message updates in the UI to provide a "typing" effect. + +## State & Data +- **Local Storage**: Cache generated wiki structures in `localStorage` to allow instant navigation. +- **i18n**: Use `next-intl` for all user-facing strings. Access via `useTranslations` hook. +- **Context**: Use `LanguageContext` for global language state. + +## Styling +- **Tailwind CSS**: Use Tailwind for all styling. Follow the existing design system (dark/light mode support via `next-themes`). +- **Icons**: Use `lucide-react` for consistent iconography. + +## Anti-Patterns +- Do not fetch data directly from the backend in components; use the Next.js API proxy routes in `src/app/api/`. +- Avoid large monolithic components; break down complex views like the Wiki page into smaller sub-components. \ No newline at end of file diff --git a/.cursor/rules/project-overview.mdc b/.cursor/rules/project-overview.mdc new file mode 100644 index 000000000..891063060 --- /dev/null +++ b/.cursor/rules/project-overview.mdc @@ -0,0 +1,36 @@ +--- +description: General project context, architecture, and tech stack overview. +globs: + - "**/*" +alwaysApply: true +--- + +# DeepWiki-Open Project Overview + +DeepWiki-Open is a full-stack RAG (Retrieval-Augmented Generation) application designed to generate and interact with documentation for software repositories. + +## Tech Stack +- **Frontend**: Next.js 15 (App Router), React 19, TypeScript, Tailwind CSS, Mermaid.js. +- **Backend**: FastAPI (Python 3.10+), Adalflow (RAG framework), FAISS (Vector DB). +- **AI**: Provider-agnostic (OpenAI, Google Gemini, Ollama, Azure, OpenRouter, etc.). + +## Core Architecture +- **Modular Monolith**: The backend is organized into functional modules within `api/`. +- **RAG Pipeline**: Uses `adalflow` to orchestrate document ingestion, embedding, and retrieval. +- **Communication**: REST for configuration/metadata; WebSockets (`/ws/chat`) for real-time streaming chat. +- **Persistence**: + - Repositories: `~/.adalflow/repos/` + - Vector DBs: `~/.adalflow/databases/` + - Wiki Cache: `~/.adalflow/wikicache/` (JSON) + +## Key Directories +- `api/`: FastAPI backend logic. +- `api/tools/`: Embedders and utility clients. +- `src/app/`: Next.js pages and API routes. +- `src/components/`: Reusable React components. +- `src/utils/`: Frontend utilities (WebSocket client, etc.). + +## Development Principles +- **Provider Agnostic**: Always use the factory patterns in `api/config.py` and `api/tools/embedder.py` when adding LLM support. +- **Type Safety**: Maintain parity between Pydantic models in `api/` and TypeScript interfaces in `src/types`. +- **Streaming First**: Prioritize streaming responses for AI interactions to improve UX. \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..3fe6b907b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,37 @@ +# DeepWiki-Open Configuration + +Full-stack RAG application for generating documentation (wikis) and interacting with codebases via AI. + +## Build & Test Commands +- **Backend Setup**: `cd api && pip install -r requirements.txt` +- **Backend Run**: `python api/main.py` (Starts FastAPI on port 8001) +- **Frontend Setup**: `npm install` +- **Frontend Run**: `npm run dev` (Starts Next.js on port 3000) +- **Testing**: `pytest` (Backend), `npm test` (Frontend) +- **Linting**: `flake8 api/` (Python), `npm run lint` (TypeScript) + +## Architecture Overview +- **Frontend**: Next.js (App Router), React 19, Tailwind CSS, Mermaid.js for diagrams. +- **Backend**: FastAPI (Python) orchestrating the RAG pipeline. +- **RAG Framework**: `adalflow` for modular data processing and retrieval. +- **Vector Store**: Local FAISS index managed via `DatabaseManager`. +- **Communication**: REST for config/metadata; WebSockets (`/ws/chat`) for streaming chat. +- **Storage**: Local filesystem at `~/.adalflow/` (repos, databases, wikicache). + +## Code Style Conventions +- **Python**: PEP8, type hints, Pydantic models for all API schemas. +- **TypeScript**: Strict typing, functional components, mirrored interfaces from Pydantic. +- **Naming**: PascalCase for components/classes, camelCase for TS variables, snake_case for Python. +- **Error Handling**: `HTTPException` in FastAPI; try-catch with UI feedback in React. + +## Key Conventions +- **Data Models**: Keep `api/api.py` Pydantic models and `src/types/` TS interfaces in sync. +- **RAG Pipeline**: Logic resides in `api/rag.py` using Adalflow components. +- **Provider Agnostic**: Support for OpenAI, Google Gemini, Ollama, etc., via `api/config.py`. +- **Caching**: Wikis are cached as JSON in `~/.adalflow/wikicache/`. +- **Auth**: Sensitive operations require `WIKI_AUTH_CODE` if `WIKI_AUTH_MODE` is enabled. + +## Git Workflow +- **Commits**: Conventional Commits (feat, fix, docs, refactor). +- **Branches**: `main` for stable releases, feature branches for development. +- **PRs**: Ensure both frontend and backend tests pass before merging. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..16dcc96ac --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,68 @@ +# CLAUDE.md - DeepWiki-Open Configuration + +DeepWiki-Open is a full-stack RAG (Retrieval-Augmented Generation) application designed to generate comprehensive wikis and provide interactive chat capabilities for software repositories. + +## 🛠 Common Commands + +### Backend (FastAPI) +- **Install Dependencies**: `pip install -r requirements.txt` (inside `api/` directory) +- **Run Server**: `python api/main.py` (Runs on port 8001 by default) +- **Run Tests**: `pytest` +- **Linting**: `flake8 api/` or `black api/` + +### Frontend (Next.js) +- **Install Dependencies**: `npm install` +- **Run Development**: `npm run dev` (Runs on port 3000) +- **Build Production**: `npm run build` +- **Linting**: `npm run lint` + +## 🏗 Architecture Overview + +The project follows a modular monolith pattern split into a Next.js frontend and a Python FastAPI backend. + +### Backend (`api/`) +- **`api.py`**: Main FastAPI application and REST endpoints. +- **`rag.py`**: Core RAG pipeline using the `adalflow` library. +- **`data_pipeline.py`**: Repository lifecycle (clone, parse, index) and FAISS management. +- **`websocket_wiki.py`**: Real-time WebSocket handler for streaming chat. +- **`config.py`**: Centralized configuration and LLM provider mapping. +- **`tools/embedder.py`**: Factory for embedding models (OpenAI, Google, Ollama). + +### Frontend (`src/`) +- **App Router**: Dynamic routes in `src/app/[owner]/[repo]` for repository views. +- **Components**: Reusable UI in `src/components` (e.g., `Ask.tsx` for chat, `WikiTreeView.tsx`). +- **WebSocket Client**: `src/utils/websocketClient.ts` manages real-time backend communication. + +## 📏 Code Style & Conventions + +### Python (Backend) +- **Type Hinting**: Use Python type hints for all function signatures. +- **Validation**: Use Pydantic models for all request/response bodies. +- **Documentation**: Use Google-style docstrings for complex logic. +- **Framework**: Leverage `adalflow` components (`adal.Component`) for RAG logic. + +### TypeScript/React (Frontend) +- **Typing**: Maintain strict TypeScript interfaces that mirror backend Pydantic models. +- **Components**: Use functional components with Tailwind CSS for styling. +- **State**: Use React Context for global state (e.g., `LanguageContext`) and local state for UI. +- **Icons**: Use `lucide-react` for iconography. + +## 📦 Data & Storage +- **Local Storage**: The system stores data in `~/.adalflow/`: + - `/repos/`: Cloned source code. + - `/databases/`: FAISS vector indices and metadata (.pkl). + - `/wikicache/`: Generated wiki structures (JSON). +- **Client Cache**: The frontend uses `localStorage` to cache wiki pages for instant loading. + +## ⚠️ Development Gotchas +- **WebSocket vs REST**: Chat uses WebSockets (`/ws/chat`) for streaming. Ensure the frontend client handles connection lifecycle and errors. +- **Token Limits**: Be mindful of the `MAX_INPUT_TOKENS` (approx. 7500-8000). The system uses `tiktoken` for counting. +- **Environment Variables**: Ensure `OPENAI_API_KEY` or `GOOGLE_API_KEY` are set. Check `api/config/generator.json` for provider settings. +- **Auth**: If `WIKI_AUTH_MODE` is enabled, the `WIKI_AUTH_CODE` must be passed in headers/payloads for sensitive operations. +- **Git Dependencies**: The backend requires the `git` CLI to be installed on the host system for repository cloning. + +## 🔗 Key Components +- **`RAG` (api/rag.py)**: Orchestrates retrieval and generation. +- **`DatabaseManager` (api/data_pipeline.py)**: Handles the FAISS vector store. +- **`Ask` (src/components/Ask.tsx)**: The primary chat interface component. +- **`WikiStructureModel`**: The core data contract for the generated documentation hierarchy. From 98a411e36c2eb8b115bd6727261f145e759cef89 Mon Sep 17 00:00:00 2001 From: Diogo Soares Rodrigues Date: Wed, 17 Dec 2025 16:54:14 -0300 Subject: [PATCH 3/5] chore: remove project-specific AI assistant config files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove CLAUDE.md, AGENTS.md, and .ai/docs/ as these are project-specific configuration files not suitable for upstream. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .ai/docs/api_analysis.md | 134 ------------------------------ .ai/docs/data_flow_analysis.md | 101 ---------------------- .ai/docs/dependency_analysis.md | 110 ------------------------ .ai/docs/request_flow_analysis.md | 68 --------------- .ai/docs/structure_analysis.md | 70 ---------------- AGENTS.md | 37 --------- CLAUDE.md | 68 --------------- 7 files changed, 588 deletions(-) delete mode 100644 .ai/docs/api_analysis.md delete mode 100644 .ai/docs/data_flow_analysis.md delete mode 100644 .ai/docs/dependency_analysis.md delete mode 100644 .ai/docs/request_flow_analysis.md delete mode 100644 .ai/docs/structure_analysis.md delete mode 100644 AGENTS.md delete mode 100644 CLAUDE.md diff --git a/.ai/docs/api_analysis.md b/.ai/docs/api_analysis.md deleted file mode 100644 index 7f348a8bc..000000000 --- a/.ai/docs/api_analysis.md +++ /dev/null @@ -1,134 +0,0 @@ -# API Documentation - -This document provides a comprehensive overview of the APIs exposed and consumed by the `deepwiki-open` project. The project consists of a Next.js frontend and a Python FastAPI backend. - -## APIs Served by This Project - -The primary API is served by a Python FastAPI backend (default port 8001). The Next.js application (default port 3000) provides proxy routes to this backend for frontend consumption. - -### Endpoints - -#### 1. Chat Completion (Streaming) -- **Method and Path**: `POST /chat/completions/stream` (HTTP) or `WS /ws/chat` (WebSocket) -- **Description**: Generates a streaming chat response using RAG (Retrieval-Augmented Generation) based on a repository's content. -- **Request (Body)**: - ```json - { - "repo_url": "string", - "messages": [ - {"role": "user", "content": "string"} - ], - "filePath": "string (optional)", - "token": "string (optional, for private repos)", - "type": "github | gitlab | bitbucket (default: github)", - "provider": "google | openai | openrouter | ollama | bedrock | azure | dashscope", - "model": "string (optional)", - "language": "en | ja | zh | es | kr | vi (default: en)", - "excluded_dirs": "string (newline separated)", - "excluded_files": "string (newline separated)", - "included_dirs": "string (newline separated)", - "included_files": "string (newline separated)" - } - ``` -- **Response**: - - **HTTP**: `text/event-stream` containing chunks of the generated text. - - **WebSocket**: Text messages containing chunks of the generated text. -- **Authentication**: Requires valid LLM provider API keys configured on the server. -- **Example**: - ```bash - curl -X POST http://localhost:8001/chat/completions/stream \ - -H "Content-Type: application/json" \ - -d '{"repo_url": "https://github.com/user/repo", "messages": [{"role": "user", "content": "Explain this project"}]}' - ``` - -#### 2. Model Configuration -- **Method and Path**: `GET /models/config` -- **Description**: Retrieves available LLM providers and their supported models. -- **Response**: - ```json - { - "providers": [ - { - "id": "google", - "name": "Google", - "supportsCustomModel": true, - "models": [{"id": "gemini-2.5-flash", "name": "gemini-2.5-flash"}] - } - ], - "defaultProvider": "google" - } - ``` - -#### 3. Wiki Cache Management -- **GET /api/wiki_cache**: Retrieves cached wiki data for a repository. - - **Params**: `owner`, `repo`, `repo_type`, `language`. -- **POST /api/wiki_cache**: Stores generated wiki data in the server-side cache. -- **DELETE /api/wiki_cache**: Deletes a specific wiki cache. - - **Params**: `owner`, `repo`, `repo_type`, `language`, `authorization_code` (if auth enabled). - -#### 4. Processed Projects -- **Method and Path**: `GET /api/processed_projects` -- **Description**: Lists all projects that have been processed and cached on the server. -- **Response**: A list of project entries including `owner`, `repo`, `language`, and `submittedAt`. - -#### 5. Wiki Export -- **Method and Path**: `POST /export/wiki` -- **Description**: Exports wiki content as a downloadable Markdown or JSON file. -- **Request**: - ```json - { - "repo_url": "string", - "pages": [...], - "format": "markdown | json" - } - ``` - -#### 6. Authentication & Status -- **GET /auth/status**: Checks if the application requires an authorization code (`DEEPWIKI_AUTH_MODE`). -- **POST /auth/validate**: Validates the provided authorization code. -- **GET /health**: Returns the health status of the API service. - ---- - -### Authentication & Security - -- **Internal Authentication**: Controlled by `DEEPWIKI_AUTH_MODE` and `DEEPWIKI_AUTH_CODE` environment variables. If enabled, certain operations (like deleting cache) require the code. -- **External API Keys**: The backend requires API keys for the LLM providers (e.g., `OPENAI_API_KEY`, `GOOGLE_API_KEY`). These are stored as environment variables on the server. -- **Repository Access**: For private repositories, users can provide a Personal Access Token (`token` in request body), which is used by the backend to clone the repository via HTTPS. - -### Rate Limiting & Constraints - -- **Token Limits**: The backend includes a token counter (`tiktoken`). Requests exceeding ~8000 tokens trigger a warning or may be truncated depending on the provider. -- **Concurrency**: The Python backend uses FastAPI with `uvicorn`, supporting asynchronous request handling. - ---- - -## External API Dependencies - -### Services Consumed - -| Service Name | Purpose | Configuration | Authentication | -| :--- | :--- | :--- | :--- | -| **Google Generative AI** | Default LLM provider (Gemini) | `GOOGLE_API_KEY` | API Key | -| **OpenAI** | LLM and Embeddings | `OPENAI_API_KEY` | API Key | -| **OpenRouter** | Unified LLM access | `OPENROUTER_API_KEY` | API Key | -| **AWS Bedrock** | Enterprise LLM access | `AWS_ACCESS_KEY_ID`, `AWS_REGION`, etc. | IAM Credentials | -| **Azure AI** | Enterprise LLM access | `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` | API Key | -| **Alibaba Dashscope** | Qwen models | `DASHSCOPE_API_KEY` | API Key | -| **Ollama** | Local LLM execution | `OLLAMA_HOST` (default: localhost:11434) | None (Local) | -| **GitHub/GitLab/Bitbucket** | Source code retrieval | `git clone` via subprocess | Optional PAT | - -### Integration Patterns - -- **RAG Pipeline**: The project uses `adalflow` to manage the RAG pipeline. It clones repositories, splits text, generates embeddings, and stores them in a local vector database for retrieval during chat. -- **Streaming**: Both HTTP (Server-Sent Events) and WebSockets are used to provide real-time streaming of LLM responses to the frontend. -- **Proxying**: The Next.js frontend proxies requests to the Python backend to avoid CORS issues and centralize API management. - ---- - -## Available Documentation - -- **API Analysis**: Located at `/.ai/docs/api_analysis.md`. -- **Structure Analysis**: Located at `/.ai/docs/structure_analysis.md`. -- **OpenAPI Spec**: FastAPI automatically generates an OpenAPI spec at `/openapi.json` and interactive docs at `/docs` (Swagger UI) when the backend is running. -- **Quality**: The internal documentation in `.ai/docs/` provides a high-level overview of the system architecture and data flow. The code itself is well-structured with Pydantic models defining request/response contracts. \ No newline at end of file diff --git a/.ai/docs/data_flow_analysis.md b/.ai/docs/data_flow_analysis.md deleted file mode 100644 index 5e31ed63c..000000000 --- a/.ai/docs/data_flow_analysis.md +++ /dev/null @@ -1,101 +0,0 @@ -# Data Flow Analysis - -## Data Models Overview - -The application uses a consistent set of data models across the Python (FastAPI) backend and the TypeScript (Next.js) frontend to represent repositories, wiki structures, and chat interactions. - -### Core Backend Models (Pydantic) -- **`WikiPage`**: Represents a single documentation page. - - `id`, `title`, `content`, `filePaths` (List[str]), `importance` (high/medium/low), `relatedPages` (List[str]). -- **`WikiStructureModel`**: Defines the hierarchy of the generated wiki. - - `id`, `title`, `description`, `pages` (List[WikiPage]), `sections` (List[WikiSection]), `rootSections` (List[str]). -- **`RepoInfo`**: Captures repository metadata. - - `owner`, `repo`, `type` (github/gitlab/bitbucket/local), `token` (optional), `localPath`, `repoUrl`. -- **`ChatCompletionRequest`**: Standardizes the payload for chat interactions. - - `repo_url`, `messages` (List[ChatMessage]), `filePath` (optional), `token`, `provider`, `model`, `language`, `filters` (included/excluded dirs/files). -- **`WikiCacheData`**: The structure used for persisting generated wikis. - - `wiki_structure`, `generated_pages` (Dict[str, WikiPage]), `repo`, `provider`, `model`. - -### Core Frontend Models (TypeScript) -- **`WikiPage`**, **`WikiStructure`**, **`RepoInfo`**: Mirror the backend Pydantic models for type safety. -- **`Message`**: Represents a chat message with `role` (user/assistant/system) and `content`. -- **`ResearchStage`**: Used in "Deep Research" mode to track the progress of multi-turn investigations. - -## Data Transformation Map - -Data undergoes several transformations as it moves from raw repository files to structured documentation and AI-generated responses. - -1. **Repository Ingestion**: - - Raw files are fetched via `git clone` or API calls (GitHub/GitLab/Bitbucket). - - `read_all_documents` transforms files into `Document` objects, attaching metadata like `file_path`, `token_count`, and `is_code`. -2. **RAG Pipeline Transformation**: - - `TextSplitter` breaks documents into manageable chunks based on token limits. - - `ToEmbeddings` (via `adalflow`) transforms text chunks into vector representations using the configured embedder (OpenAI, Google, or Ollama). - - `LocalDB` persists these transformed documents and their vectors into a FAISS index. -3. **Retrieval Transformation**: - - User queries are embedded using the same embedding model. - - `FAISSRetriever` performs a similarity search to find the most relevant document chunks. -4. **Generation Transformation**: - - A prompt is constructed by combining a `system_prompt`, `conversation_history`, `retrieved_context`, and the `user_query`. - - The LLM transforms this prompt into a structured `RAGAnswer` or a streamed text response. -5. **Wiki Generation**: - - The repository structure is analyzed by the LLM to generate a `WikiStructureModel`. - - Each page in the structure is then individually generated by the LLM using relevant file contexts. - -## Storage Interactions - -The system utilizes both server-side and client-side storage for persistence and performance. - -### Server-Side Storage (`~/.adalflow/`) -- **`/repos/`**: Stores cloned Git repositories. -- **`/databases/`**: Stores serialized `LocalDB` states (FAISS indices and document metadata) as `.pkl` files. -- **`/wikicache/`**: Stores generated wiki data as JSON files, named using the pattern `deepwiki_cache_{repo_type}_{owner}_{repo}_{language}.json`. - -### Client-Side Storage -- **`localStorage`**: The frontend caches generated wiki structures and pages to provide near-instant loading for previously visited repositories. -- **Session State**: React state manages the current conversation history and active wiki page during a user session. - -## Validation Mechanisms - -Data integrity and system stability are maintained through several validation layers: - -- **Pydantic Validation**: All API endpoints use Pydantic models to validate incoming request bodies and outgoing responses. -- **Token Limit Validation**: `tiktoken` is used to estimate token counts for user queries and document chunks, ensuring they stay within LLM and embedding model limits (e.g., `MAX_INPUT_TOKENS = 7500`). -- **Embedding Consistency**: The `RAG` component includes a `_validate_and_filter_embeddings` method that ensures all vectors in the FAISS index have consistent dimensions, filtering out any documents that failed to embed correctly. -- **Authentication**: Sensitive operations like deleting a wiki cache are protected by `WIKI_AUTH_MODE` and `WIKI_AUTH_CODE` environment variables. -- **File Filtering**: Inclusion and exclusion rules (e.g., `DEFAULT_EXCLUDED_DIRS`) prevent sensitive or irrelevant files (like `.git`, `node_modules`, or binary files) from entering the data pipeline. - -## State Management Analysis - -### Backend State -- **`Memory` Component**: Manages conversation state within the `RAG` pipeline using a list of `DialogTurn` objects. -- **`DatabaseManager`**: Maintains the state of the currently loaded vector database and repository paths. - -### Frontend State -- **`Ask` Component**: Manages `conversationHistory` (array of `Message`) and `researchStages` for the Deep Research feature. -- **`RepoWikiPage`**: Manages the `wikiStructure`, `generatedPages` (a map of page IDs to content), and `pagesInProgress` (a set of IDs currently being generated to prevent duplicate requests). -- **`LanguageContext`**: Manages the application's internationalization state. - -## Serialization Processes - -- **JSON Serialization**: Used for all REST API communication, WebSocket messages, and the persistence of wiki cache files. -- **Pickle Serialization**: Used by `adalflow`'s `LocalDB` to save and load the state of the FAISS index and document objects. -- **Base64 Encoding**: Used when fetching file contents via the GitHub API. -- **URL Encoding**: Used for safely passing repository URLs and file paths in API requests. - -## Data Lifecycle Diagrams - -### Chat Interaction Lifecycle -1. **Input**: User sends a message via the `Ask` component. -2. **Transport**: Message is wrapped in a `ChatCompletionRequest` and sent over a **WebSocket** (`/ws/chat`). -3. **Processing**: Backend prepares the `RAG` instance, retrieves context from the FAISS index, and constructs the LLM prompt. -4. **Generation**: LLM generates a response, which is streamed back to the client. -5. **Update**: Frontend appends the response to `conversationHistory` and renders it via the `Markdown` component. - -### Wiki Generation Lifecycle -1. **Trigger**: User navigates to a repository page. -2. **Cache Check**: Frontend checks `localStorage`, then queries `/api/wiki_cache`. -3. **Structure Generation**: If not cached, backend analyzes the repo and returns a `WikiStructureModel`. -4. **Page Generation**: Frontend iterates through the structure, requesting content for each `WikiPage` via the backend. -5. **Persistence**: As pages are generated, they are saved to the server-side cache and the client's `localStorage`. -6. **Display**: `WikiTreeView` renders the hierarchy, and selecting a page displays its content in the main view. \ No newline at end of file diff --git a/.ai/docs/dependency_analysis.md b/.ai/docs/dependency_analysis.md deleted file mode 100644 index fe8cfd2c3..000000000 --- a/.ai/docs/dependency_analysis.md +++ /dev/null @@ -1,110 +0,0 @@ -# Dependency Analysis - -## Internal Dependencies Map - -The project follows a clear separation between the frontend (Next.js) and the backend (FastAPI), with a layered architecture in the backend to handle RAG (Retrieval-Augmented Generation) and data processing. - -### Backend (Python) -- **Entry Point**: `api/main.py` initializes the environment and starts the `uvicorn` server, importing the FastAPI app from `api/api.py`. -- **API Layer**: `api/api.py` defines REST endpoints for configuration, authentication, and project management. It delegates real-time chat to `api/websocket_wiki.py`. -- **WebSocket Layer**: `api/websocket_wiki.py` handles bidirectional communication for chat, instantiating the `RAG` component for each session. -- **Logic Layer (RAG)**: `api/rag.py` contains the core RAG logic using the `adalflow` framework. It relies on: - - `api/data_pipeline.py`: For repository cloning, document processing, and database management. - - `api/tools/embedder.py`: A factory for creating embedder instances. - - `api/prompts.py`: Centralized storage for system prompts and templates. -- **Data Layer**: `api/data_pipeline.py` manages the lifecycle of repository data, from `git clone` to tokenization and embedding. -- **Client Layer**: `api/config.py` maps provider IDs to specific client implementations (e.g., `OpenAIClient`, `GoogleGenAIClient`). Custom clients are located in `api/*_client.py`. - -### Frontend (TypeScript/Next.js) -- **App Router**: `src/app` contains the page definitions. - - `page.tsx`: Landing page and repository configuration. - - `[owner]/[repo]/page.tsx`: Main wiki view. -- **Components**: `src/components` contains reusable UI elements. - - `Ask.tsx`: Chat interface, uses `src/utils/websocketClient.ts`. - - `WikiTreeView.tsx`: Navigation for the generated wiki. - - `Mermaid.tsx`: Renders architecture diagrams using the `mermaid` library. -- **State & Logic**: - - `src/contexts/LanguageContext.tsx`: Manages i18n state. - - `src/hooks/useProcessedProjects.ts`: Fetches and manages the list of analyzed repositories. - - `src/utils/websocketClient.ts`: Encapsulates WebSocket logic for backend communication. - -## External Libraries Analysis - -### Backend (Python) -- **Core Framework**: `adalflow` (LLM/RAG orchestration), `fastapi` (Web API), `uvicorn` (ASGI server). -- **LLM SDKs**: `google-generativeai`, `openai`, `ollama`, `boto3` (AWS Bedrock), `azure-identity`, `azure-core`. -- **Data Processing**: `faiss-cpu` (Vector DB), `tiktoken` (Token counting), `numpy`, `langid` (Language detection). -- **Utilities**: `pydantic` (Data validation), `python-dotenv` (Env management), `aiohttp` (Async HTTP), `websockets` (WebSocket protocol). - -### Frontend (React/Next.js) -- **Framework**: `next` (v15.3.1), `react` (v19). -- **Visualization**: `mermaid` (Diagrams), `svg-pan-zoom`. -- **Content Rendering**: `react-markdown`, `rehype-raw`, `remark-gfm`, `react-syntax-highlighter`. -- **Internationalization**: `next-intl`. -- **Styling**: `tailwindcss`, `next-themes`. - -## Service Integrations - -- **LLM Providers**: - - **Google**: Gemini models via `google-generativeai`. - - **OpenAI**: GPT models via `openai`. - - **OpenRouter**: Aggregator for various models. - - **Ollama**: Local LLM execution. - - **AWS Bedrock**: Enterprise LLM access. - - **Azure AI**: Microsoft's AI services. - - **DashScope**: Alibaba Cloud's AI models. -- **Git Platforms**: - - **GitHub/GitLab/Bitbucket**: Integrated via `git` CLI for cloning and personal access tokens for private repo access. - -## Dependency Injection Patterns - -- **Factory Pattern**: The backend uses a factory pattern for creating LLM clients and embedders. `api/config.py` maintains a mapping of client classes, and `api/tools/embedder.py` provides a `get_embedder` function that resolves the correct instance based on environment configuration. -- **Adalflow Components**: The `RAG` class in `api/rag.py` inherits from `adal.Component`, allowing it to compose other `adalflow` building blocks (like `FAISSRetriever` and `Memory`) in a declarative manner. -- **Context Providers**: The frontend uses React Context (`LanguageContext`) to inject internationalization state and functions throughout the component tree. - -## Module Coupling Assessment - -- **Backend Modularity**: High. The separation of clients, data pipeline, and RAG logic allows for easy addition of new LLM providers or data sources. -- **Frontend-Backend Coupling**: Moderate. The frontend is coupled to the backend's API schema (Pydantic models). Changes in backend models (e.g., `WikiPage`, `RepoInfo`) require manual updates to TypeScript interfaces in `src/types`. -- **Framework Coupling**: High. The backend is deeply integrated with `adalflow`. While this provides powerful RAG capabilities, it makes the core logic dependent on the `adalflow` API. -- **Cohesion**: High. Each module has a well-defined responsibility (e.g., `data_pipeline.py` only handles data ingestion and processing). - -## Dependency Graph - -```mermaid -graph TD - subgraph Frontend - UI[Next.js Pages] --> Comp[React Components] - Comp --> Hooks[Custom Hooks] - Comp --> WS_Client[WebSocket Client] - Comp --> Mermaid[Mermaid.js] - end - - subgraph Backend_API - WS_Handler[WebSocket Handler] --> RAG_Comp[RAG Component] - REST_API[FastAPI Endpoints] --> Config[Config Manager] - end - - subgraph Backend_Logic - RAG_Comp --> Retriever[FAISS Retriever] - RAG_Comp --> Memory[Conversation Memory] - RAG_Comp --> Embedder_Factory[Embedder Factory] - RAG_Comp --> Pipeline[Data Pipeline] - end - - subgraph External_Services - Pipeline --> Git[Git CLI / Repo Providers] - Embedder_Factory --> LLM_Clients[LLM Providers: OpenAI, Google, etc.] - LLM_Clients --> Adalflow[Adalflow Framework] - end - - WS_Client -.-> WS_Handler - UI -.-> REST_API -``` - -## Potential Dependency Issues - -- **Adalflow Versioning**: The project relies on `adalflow >= 0.1.0`. As a relatively new framework, breaking changes in `adalflow` could significantly impact the `RAG` and `DataPipeline` implementations. -- **Duplicate Model Definitions**: Data models are defined in Python (Pydantic) and TypeScript. This duplication increases the risk of desynchronization during API updates. -- **Manual Client Mapping**: The mapping of providers to client classes in `api/config.py` is manual. While functional, it requires code changes to support new providers even if they follow standard protocols (like OpenAI-compatible APIs). -- **Local Environment Dependencies**: The backend relies on the `git` binary being present in the system path for repository cloning, which might cause issues in restricted container environments if not properly configured. \ No newline at end of file diff --git a/.ai/docs/request_flow_analysis.md b/.ai/docs/request_flow_analysis.md deleted file mode 100644 index 76acce6e6..000000000 --- a/.ai/docs/request_flow_analysis.md +++ /dev/null @@ -1,68 +0,0 @@ -# Request Flow Analysis - -## Entry Points Overview -The system exposes two primary layers of entry points: the Next.js frontend and the FastAPI backend. - -- **Frontend Entry Points (Next.js - Port 3000)**: - - **Web UI**: The main application interface served at the root and dynamic repository routes (e.g., `/[owner]/[repo]`). - - **API Proxy Routes**: Located in `src/app/api/`, these routes act as proxies to the backend: - - `POST /api/chat/stream`: Fallback HTTP streaming for chat. - - `GET /api/models/config`: Fetches available LLM providers and models. - - `GET /api/wiki/projects`: Lists processed repositories. - - `GET /api/auth/status` & `POST /api/auth/validate`: Authentication checks. - -- **Backend Entry Points (FastAPI - Port 8001)**: - - **WebSocket**: `WS /ws/chat` - The primary entry point for real-time, streaming RAG chat. - - **REST API**: - - `POST /chat/completions/stream`: HTTP streaming chat endpoint. - - `GET /models/config`: Returns the system's LLM configuration. - - `GET/POST/DELETE /api/wiki_cache`: Manages the server-side JSON cache for generated wikis. - - `GET /api/processed_projects`: Scans the cache directory to list all indexed repositories. - - `POST /export/wiki`: Generates downloadable Markdown or JSON exports. - - `GET /local_repo/structure`: Analyzes local directories for the "Local Path" feature. - -## Request Routing Map -- **Client-to-Proxy**: The frontend UI components (like `Ask.tsx` or `ProcessedProjects.tsx`) typically call the Next.js `/api/*` routes. -- **Proxy-to-Backend**: Next.js routes (e.g., `src/app/api/chat/stream/route.ts`) use the `fetch` API to forward requests to the `TARGET_SERVER_BASE_URL` (defaulting to `http://localhost:8001`). -- **Direct WebSocket**: For the core chat functionality, `src/utils/websocketClient.ts` bypasses the Next.js proxy and establishes a direct `WebSocket` connection to the backend's `/ws/chat` endpoint. -- **Backend Internal Routing**: FastAPI routes requests based on the path and method defined in `api/api.py`. It uses `app.add_api_route` and `app.add_websocket_route` to link paths to handler functions in `api/simple_chat.py` and `api/websocket_wiki.py`. - -## Middleware Pipeline -- **CORS Middleware**: The FastAPI application in `api/api.py` uses `CORSMiddleware` configured with `allow_origins=["*"]`. This is critical for allowing the Next.js frontend (on port 3000) to communicate with the backend (on port 8001) and for direct WebSocket connections. -- **Next.js Request Handling**: While no custom `middleware.ts` is present, Next.js handles standard request preprocessing, including environment variable injection and route matching. -- **Logging Middleware**: A custom logging configuration in `api/logging_config.py` sets up a `RotatingFileHandler`. It includes a filter (`IgnoreLogChangeDetectedFilter`) to suppress development-time noise from file watchers. - -## Controller/Handler Analysis -- **`api/api.py`**: The main controller for administrative and metadata operations. It handles Pydantic model validation for wiki structures and manages the filesystem-based cache in `~/.adalflow/wikicache`. -- **`api/websocket_wiki.py`**: The primary handler for the chat lifecycle. It: - 1. Accepts the WebSocket connection. - 2. Parses the `ChatCompletionRequest`. - 3. Initializes the `RAG` component. - 4. Streams LLM chunks back to the client. -- **`api/rag.py`**: The core logic controller for the RAG pipeline. It orchestrates the `DatabaseManager` for retrieval and manages conversation `Memory`. -- **`api/data_pipeline.py`**: The low-level handler for data operations. It manages `git clone` operations, file parsing, and interacts with the vector database (FAISS). - -## Authentication & Authorization Flow -- **System-Level Auth**: Controlled by `WIKI_AUTH_MODE`. If enabled, the `WIKI_AUTH_CODE` must be provided for sensitive operations like deleting a wiki cache (`DELETE /api/wiki_cache`). -- **Repository-Level Auth**: The `ChatCompletionRequest` and `WikiCacheRequest` models include an optional `token` field. This Personal Access Token is used by `download_repo` in `api/data_pipeline.py` to authenticate Git clones for private GitHub, GitLab, or Bitbucket repositories. -- **Provider Auth**: LLM provider authentication (e.g., OpenAI, Google Gemini) is handled server-side via environment variables (`OPENAI_API_KEY`, `GOOGLE_API_KEY`) which are used to initialize the respective AI clients. - -## Error Handling Pathways -- **REST API Errors**: Handled via FastAPI's `HTTPException`. Errors are returned as JSON objects with a `detail` field and appropriate HTTP status codes (e.g., 401 for unauthorized, 404 for missing cache). -- **WebSocket Errors**: If an error occurs during the WebSocket session (e.g., "No valid documents with embeddings found"), the backend sends a descriptive text message to the client and then calls `websocket.close()`. -- **Proxy Errors**: The Next.js proxy routes include `try-catch` blocks that capture backend failures or network issues, returning a 500 status with the error message to the frontend. -- **Validation Errors**: Pydantic models in the backend automatically catch and return 422 Unprocessable Entity errors if the request body does not match the expected schema. - -## Request Lifecycle Diagram -1. **User Action**: User enters a repository URL and a question in the `Ask` component. -2. **Connection**: The frontend `websocketClient` initiates a connection to `ws://localhost:8001/ws/chat`. -3. **Handshake**: FastAPI accepts the connection and the client sends the `ChatCompletionRequest` JSON. -4. **Retrieval Preparation**: - - Backend checks if the repository is already cloned and indexed. - - If not, `data_pipeline.py` clones the repo and `RAG.prepare_retriever` builds the vector index. -5. **Context Retrieval**: The `RAG` component searches the index for documents relevant to the user's query. -6. **Prompt Construction**: A system prompt is generated, combining the user's query, conversation history from `Memory`, and the retrieved code context. -7. **Streaming Response**: - - The backend calls the configured LLM provider (e.g., Google Gemini). - - As chunks of text arrive from the LLM, they are immediately sent over the WebSocket to the frontend. -8. **Termination**: Once the LLM finishes the response, the backend closes the WebSocket, and the frontend updates the UI with the complete message. \ No newline at end of file diff --git a/.ai/docs/structure_analysis.md b/.ai/docs/structure_analysis.md deleted file mode 100644 index 2713d63c1..000000000 --- a/.ai/docs/structure_analysis.md +++ /dev/null @@ -1,70 +0,0 @@ -# Code Structure Analysis - -## Architectural Overview -The **DeepWiki-Open** project is a full-stack application designed to generate and interact with documentation (wikis) for software repositories using Retrieval-Augmented Generation (RAG). The architecture is split into two main parts: - -1. **Frontend (Next.js)**: A React-based web interface using the App Router. It provides a dynamic UI for exploring repository structures and a chat interface for querying the codebase. It communicates with the backend via REST APIs for configuration and WebSockets for real-time, streaming chat completions. -2. **Backend (FastAPI)**: A Python-based API that orchestrates the RAG pipeline. It leverages the `adalflow` library to build modular data processing and retrieval components. The backend is provider-agnostic, supporting multiple LLM and embedding providers (OpenAI, Google Gemini, Ollama, Azure, OpenRouter, etc.). - -The system follows a **modular monolith** pattern where core responsibilities (data ingestion, retrieval, generation, and API serving) are separated into distinct modules within the `api/` directory. - -## Core Components -### Backend (`/api`) -- **`api.py`**: The central FastAPI application defining REST endpoints for wiki management, configuration retrieval, and authentication. -- **`rag.py`**: Implements the RAG pipeline using `adalflow`. It defines the `RAG` component, `Memory` for conversation history, and custom data classes for structured answers. -- **`data_pipeline.py`**: Handles repository lifecycle management, including cloning (GitHub, GitLab, Bitbucket), document parsing, token counting, and vector database management via the `DatabaseManager`. -- **`websocket_wiki.py`**: Manages real-time WebSocket connections for chat, handling the orchestration between user queries, RAG retrieval, and LLM generation. -- **`config.py`**: A centralized configuration manager that loads JSON settings from `/api/config/` and handles environment variable substitution. -- **`prompts.py`**: Contains all system prompts and templates for RAG, simple chat, and the "Deep Research" multi-turn process. - -### Frontend (`/src`) -- **`src/app`**: Contains the page routes, including dynamic routes for specific repositories (`[owner]/[repo]`) and specialized views like `slides` and `workshop`. -- **`src/components`**: Reusable UI components such as `WikiTreeView`, `Ask` (chat interface), `Markdown` (rendering), and `ModelSelectionModal`. -- **`src/utils/websocketClient.ts`**: A dedicated client for managing WebSocket communication with the backend. - -## Service Definitions -- **Wiki Service**: Managed through `api.py`, it handles the creation, caching, and exporting (Markdown/JSON) of wiki structures. -- **Chat Service**: Implemented in `websocket_wiki.py` (WebSocket) and `simple_chat.py` (HTTP Stream), providing a stateful, streaming chat experience. -- **Data Pipeline Service**: Encapsulated in `data_pipeline.py`, responsible for transforming raw repository files into searchable embeddings stored in a local FAISS index. -- **Model Configuration Service**: Provides a unified interface to query available LLM providers and models via the `/models/config` endpoint, dynamically resolved from `generator.json`. -- **Deep Research Service**: A specialized logic within the chat handlers that uses multi-turn prompts to perform deep, iterative analysis of specific topics. - -## Interface Contracts -- **Pydantic Models (`api.py`, `websocket_wiki.py`)**: - - `WikiPage`: Defines the structure of a single documentation page. - - `WikiStructureModel`: Defines the hierarchy and metadata of the entire wiki. - - `RepoInfo`: Captures repository details (owner, repo, type, token). - - `ChatCompletionRequest`: Standardizes the payload for chat interactions, including model parameters and file filters. -- **WebSocket Protocol**: A JSON-based message exchange format for real-time chat, where the client sends a `ChatCompletionRequest` and receives streamed text responses. -- **Adalflow Components**: The `RAG` class and `Memory` class follow the `adal.Component` and `adal.DataComponent` interfaces, ensuring compatibility with the Adalflow ecosystem. - -## Design Patterns Identified -- **Strategy Pattern**: Used in the LLM and Embedder clients (`openai_client.py`, `google_embedder_client.py`, `azureai_client.py`, etc.) to provide a uniform interface for different AI providers. -- **Factory Pattern**: `get_embedder` (in `api/tools/embedder.py`) and `load_generator_config` act as factories for creating configured clients based on environment settings. -- **Repository Pattern**: The `DatabaseManager` abstracts the complexities of Git operations and vector database persistence. -- **Component-based Architecture**: Both in React (frontend) and Adalflow (backend), where logic is encapsulated into reusable, composable units. -- **Observer Pattern**: Utilized via WebSockets to push real-time updates from the backend to the frontend during long-running generation tasks. - -## Component Relationships -- **Frontend `Ask` Component → `websocketClient` → Backend `websocket_wiki.py`**: The primary path for user interaction. -- **`websocket_wiki.py` → `RAG` Component**: The WebSocket handler instantiates and calls the RAG pipeline for each request. -- **`RAG` → `FAISSRetriever` & `Memory`**: The RAG component orchestrates retrieval from the vector store and manages conversation state. -- **`FAISSRetriever` → `DatabaseManager`**: The retriever relies on the data pipeline to ensure the repository is cloned and indexed. -- **`DatabaseManager` → `Embedder`**: The pipeline uses the configured embedder to generate vectors for repository documents. - -## Key Methods & Functions -- **`RAG.prepare_retriever` (`api/rag.py`)**: Orchestrates the setup of the vector database for a given repository, including cloning and indexing. -- **`handle_websocket_chat` (`api/websocket_wiki.py`)**: The main entry point for processing chat queries, performing retrieval, and streaming LLM responses. -- **`download_repo` (`api/data_pipeline.py`)**: A robust utility for cloning GitHub, GitLab, and Bitbucket repositories with support for access tokens. -- **`read_all_documents` (`api/data_pipeline.py`)**: Recursively parses repository files while respecting complex exclusion/inclusion filters defined in `repo.json`. -- **`get_embedder` (`api/tools/embedder.py`)**: Dynamically initializes the correct embedder (OpenAI, Google, or Ollama) based on configuration. - -## Available Documentation -- **`README.md`**: High-level project overview and setup instructions. -- **`api/README.md`**: Documentation specifically for the backend API and RAG logic. -- **`Ollama-instruction.md`**: Detailed guide for running the system with local Ollama models. -- **`tests/README.md`**: Instructions for running unit and integration tests. -- **`.ai/docs/api_analysis.md`**: Detailed analysis of the API endpoints and data models. -- **`.ai/docs/structure_analysis.md`**: Previous structural analysis of the codebase. - -**Documentation Quality**: The documentation is high-quality and comprehensive. It covers installation, configuration, and specific deployment scenarios (like local LLMs). The presence of multi-language READMEs and specialized guides for Ollama indicates a mature project with a focus on developer experience. The code is well-structured with clear naming conventions and descriptive docstrings in key modules. \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 3fe6b907b..000000000 --- a/AGENTS.md +++ /dev/null @@ -1,37 +0,0 @@ -# DeepWiki-Open Configuration - -Full-stack RAG application for generating documentation (wikis) and interacting with codebases via AI. - -## Build & Test Commands -- **Backend Setup**: `cd api && pip install -r requirements.txt` -- **Backend Run**: `python api/main.py` (Starts FastAPI on port 8001) -- **Frontend Setup**: `npm install` -- **Frontend Run**: `npm run dev` (Starts Next.js on port 3000) -- **Testing**: `pytest` (Backend), `npm test` (Frontend) -- **Linting**: `flake8 api/` (Python), `npm run lint` (TypeScript) - -## Architecture Overview -- **Frontend**: Next.js (App Router), React 19, Tailwind CSS, Mermaid.js for diagrams. -- **Backend**: FastAPI (Python) orchestrating the RAG pipeline. -- **RAG Framework**: `adalflow` for modular data processing and retrieval. -- **Vector Store**: Local FAISS index managed via `DatabaseManager`. -- **Communication**: REST for config/metadata; WebSockets (`/ws/chat`) for streaming chat. -- **Storage**: Local filesystem at `~/.adalflow/` (repos, databases, wikicache). - -## Code Style Conventions -- **Python**: PEP8, type hints, Pydantic models for all API schemas. -- **TypeScript**: Strict typing, functional components, mirrored interfaces from Pydantic. -- **Naming**: PascalCase for components/classes, camelCase for TS variables, snake_case for Python. -- **Error Handling**: `HTTPException` in FastAPI; try-catch with UI feedback in React. - -## Key Conventions -- **Data Models**: Keep `api/api.py` Pydantic models and `src/types/` TS interfaces in sync. -- **RAG Pipeline**: Logic resides in `api/rag.py` using Adalflow components. -- **Provider Agnostic**: Support for OpenAI, Google Gemini, Ollama, etc., via `api/config.py`. -- **Caching**: Wikis are cached as JSON in `~/.adalflow/wikicache/`. -- **Auth**: Sensitive operations require `WIKI_AUTH_CODE` if `WIKI_AUTH_MODE` is enabled. - -## Git Workflow -- **Commits**: Conventional Commits (feat, fix, docs, refactor). -- **Branches**: `main` for stable releases, feature branches for development. -- **PRs**: Ensure both frontend and backend tests pass before merging. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 16dcc96ac..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,68 +0,0 @@ -# CLAUDE.md - DeepWiki-Open Configuration - -DeepWiki-Open is a full-stack RAG (Retrieval-Augmented Generation) application designed to generate comprehensive wikis and provide interactive chat capabilities for software repositories. - -## 🛠 Common Commands - -### Backend (FastAPI) -- **Install Dependencies**: `pip install -r requirements.txt` (inside `api/` directory) -- **Run Server**: `python api/main.py` (Runs on port 8001 by default) -- **Run Tests**: `pytest` -- **Linting**: `flake8 api/` or `black api/` - -### Frontend (Next.js) -- **Install Dependencies**: `npm install` -- **Run Development**: `npm run dev` (Runs on port 3000) -- **Build Production**: `npm run build` -- **Linting**: `npm run lint` - -## 🏗 Architecture Overview - -The project follows a modular monolith pattern split into a Next.js frontend and a Python FastAPI backend. - -### Backend (`api/`) -- **`api.py`**: Main FastAPI application and REST endpoints. -- **`rag.py`**: Core RAG pipeline using the `adalflow` library. -- **`data_pipeline.py`**: Repository lifecycle (clone, parse, index) and FAISS management. -- **`websocket_wiki.py`**: Real-time WebSocket handler for streaming chat. -- **`config.py`**: Centralized configuration and LLM provider mapping. -- **`tools/embedder.py`**: Factory for embedding models (OpenAI, Google, Ollama). - -### Frontend (`src/`) -- **App Router**: Dynamic routes in `src/app/[owner]/[repo]` for repository views. -- **Components**: Reusable UI in `src/components` (e.g., `Ask.tsx` for chat, `WikiTreeView.tsx`). -- **WebSocket Client**: `src/utils/websocketClient.ts` manages real-time backend communication. - -## 📏 Code Style & Conventions - -### Python (Backend) -- **Type Hinting**: Use Python type hints for all function signatures. -- **Validation**: Use Pydantic models for all request/response bodies. -- **Documentation**: Use Google-style docstrings for complex logic. -- **Framework**: Leverage `adalflow` components (`adal.Component`) for RAG logic. - -### TypeScript/React (Frontend) -- **Typing**: Maintain strict TypeScript interfaces that mirror backend Pydantic models. -- **Components**: Use functional components with Tailwind CSS for styling. -- **State**: Use React Context for global state (e.g., `LanguageContext`) and local state for UI. -- **Icons**: Use `lucide-react` for iconography. - -## 📦 Data & Storage -- **Local Storage**: The system stores data in `~/.adalflow/`: - - `/repos/`: Cloned source code. - - `/databases/`: FAISS vector indices and metadata (.pkl). - - `/wikicache/`: Generated wiki structures (JSON). -- **Client Cache**: The frontend uses `localStorage` to cache wiki pages for instant loading. - -## ⚠️ Development Gotchas -- **WebSocket vs REST**: Chat uses WebSockets (`/ws/chat`) for streaming. Ensure the frontend client handles connection lifecycle and errors. -- **Token Limits**: Be mindful of the `MAX_INPUT_TOKENS` (approx. 7500-8000). The system uses `tiktoken` for counting. -- **Environment Variables**: Ensure `OPENAI_API_KEY` or `GOOGLE_API_KEY` are set. Check `api/config/generator.json` for provider settings. -- **Auth**: If `WIKI_AUTH_MODE` is enabled, the `WIKI_AUTH_CODE` must be passed in headers/payloads for sensitive operations. -- **Git Dependencies**: The backend requires the `git` CLI to be installed on the host system for repository cloning. - -## 🔗 Key Components -- **`RAG` (api/rag.py)**: Orchestrates retrieval and generation. -- **`DatabaseManager` (api/data_pipeline.py)**: Handles the FAISS vector store. -- **`Ask` (src/components/Ask.tsx)**: The primary chat interface component. -- **`WikiStructureModel`**: The core data contract for the generated documentation hierarchy. From 31caa8499537d760898e1efc220c0120f0cf6bf2 Mon Sep 17 00:00:00 2001 From: Diogo Soares Rodrigues Date: Wed, 17 Dec 2025 17:02:23 -0300 Subject: [PATCH 4/5] feat: complete google-genai SDK migration for embeddings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Migrate GoogleEmbedderClient from deprecated google-generativeai to google-genai SDK - Use new client.models.embed_content() API with EmbedContentConfig - Remove google-generativeai dependency entirely from pyproject.toml - Update poetry.lock to remove deprecated package This completes the SDK migration, ensuring consistency across all Google AI integrations (both generation and embeddings now use google-genai). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- api/google_embedder_client.py | 165 +++++++---------- api/poetry.lock | 334 +--------------------------------- api/pyproject.toml | 1 - 3 files changed, 69 insertions(+), 431 deletions(-) diff --git a/api/google_embedder_client.py b/api/google_embedder_client.py index b604fd8ed..54da4049a 100644 --- a/api/google_embedder_client.py +++ b/api/google_embedder_client.py @@ -1,4 +1,4 @@ -"""Google AI Embeddings ModelClient integration.""" +"""Google AI Embeddings ModelClient integration using the new google-genai SDK.""" import os import logging @@ -9,10 +9,10 @@ from adalflow.core.types import ModelType, EmbedderOutput try: - import google.generativeai as genai - from google.generativeai.types.text_types import EmbeddingDict, BatchEmbeddingDict + from google import genai + from google.genai import types as genai_types except ImportError: - raise ImportError("google-generativeai is required. Install it with 'pip install google-generativeai'") + raise ImportError("google-genai is required. Install it with 'pip install google-genai'") log = logging.getLogger(__name__) @@ -20,9 +20,9 @@ class GoogleEmbedderClient(ModelClient): __doc__ = r"""A component wrapper for Google AI Embeddings API client. - This client provides access to Google's embedding models through the Google AI API. - It supports text embeddings for various tasks including semantic similarity, - retrieval, and classification. + This client provides access to Google's embedding models through the Google AI API + using the new google-genai SDK. It supports text embeddings for various tasks + including semantic similarity, retrieval, and classification. Args: api_key (Optional[str]): Google AI API key. Defaults to None. @@ -39,15 +39,16 @@ class GoogleEmbedderClient(ModelClient): embedder = adal.Embedder( model_client=client, model_kwargs={ - "model": "text-embedding-004", - "task_type": "SEMANTIC_SIMILARITY" + "model": "gemini-embedding-001", + "task_type": "SEMANTIC_SIMILARITY", + "output_dimensionality": 3072 } ) ``` References: - Google AI Embeddings: https://ai.google.dev/gemini-api/docs/embeddings - - Available models: text-embedding-004, embedding-001 + - Available models: gemini-embedding-001, text-embedding-004 """ def __init__( @@ -56,7 +57,7 @@ def __init__( env_api_key_name: str = "GOOGLE_API_KEY", ): """Initialize Google AI Embeddings client. - + Args: api_key: Google AI API key. If not provided, uses environment variable. env_api_key_name: Name of environment variable containing API key. @@ -64,6 +65,7 @@ def __init__( super().__init__() self._api_key = api_key self._env_api_key_name = env_api_key_name + self._client = None self._initialize_client() def _initialize_client(self): @@ -73,58 +75,35 @@ def _initialize_client(self): raise ValueError( f"Environment variable {self._env_api_key_name} must be set" ) - genai.configure(api_key=api_key) + self._client = genai.Client(api_key=api_key) def parse_embedding_response(self, response) -> EmbedderOutput: """Parse Google AI embedding response to EmbedderOutput format. - + Args: - response: Google AI embedding response (EmbeddingDict or BatchEmbeddingDict) - + response: Google AI embedding response from embed_content + Returns: EmbedderOutput with parsed embeddings """ try: from adalflow.core.types import Embedding - + embedding_data = [] - - if isinstance(response, dict): - if 'embedding' in response: - embedding_value = response['embedding'] - if isinstance(embedding_value, list) and len(embedding_value) > 0: - # Check if it's a single embedding (list of floats) or batch (list of lists) - if isinstance(embedding_value[0], (int, float)): - # Single embedding response: {'embedding': [float, ...]} - embedding_data = [Embedding(embedding=embedding_value, index=0)] - else: - # Batch embeddings response: {'embedding': [[float, ...], [float, ...], ...]} - embedding_data = [ - Embedding(embedding=emb_list, index=i) - for i, emb_list in enumerate(embedding_value) - ] + + # New SDK returns response with .embeddings attribute + # Each embedding has .values which is a list of floats + if hasattr(response, 'embeddings') and response.embeddings: + for i, emb in enumerate(response.embeddings): + if hasattr(emb, 'values'): + embedding_data.append(Embedding(embedding=list(emb.values), index=i)) + elif isinstance(emb, (list, tuple)): + embedding_data.append(Embedding(embedding=list(emb), index=i)) else: - log.warning(f"Empty or invalid embedding data: {embedding_value}") - embedding_data = [] - elif 'embeddings' in response: - # Alternative batch format: {'embeddings': [{'embedding': [float, ...]}, ...]} - embedding_data = [ - Embedding(embedding=item['embedding'], index=i) - for i, item in enumerate(response['embeddings']) - ] - else: - log.warning(f"Unexpected response structure: {response.keys()}") - embedding_data = [] - elif hasattr(response, 'embeddings'): - # Custom batch response object from our implementation - embedding_data = [ - Embedding(embedding=emb, index=i) - for i, emb in enumerate(response.embeddings) - ] + log.warning(f"Unexpected embedding format at index {i}: {type(emb)}") else: - log.warning(f"Unexpected response type: {type(response)}") - embedding_data = [] - + log.warning(f"Unexpected response structure: {type(response)}") + return EmbedderOutput( data=embedding_data, error=None, @@ -145,43 +124,44 @@ def convert_inputs_to_api_kwargs( model_type: ModelType = ModelType.UNDEFINED, ) -> Dict: """Convert inputs to Google AI API format. - + Args: input: Text input(s) to embed - model_kwargs: Model parameters including model name and task_type + model_kwargs: Model parameters including model name, task_type, output_dimensionality model_type: Should be ModelType.EMBEDDER for this client - + Returns: Dict: API kwargs for Google AI embedding call """ if model_type != ModelType.EMBEDDER: raise ValueError(f"GoogleEmbedderClient only supports EMBEDDER model type, got {model_type}") - + # Ensure input is a list if isinstance(input, str): - content = [input] + contents = [input] elif isinstance(input, Sequence): - content = list(input) + contents = list(input) else: raise TypeError("input must be a string or sequence of strings") - - final_model_kwargs = model_kwargs.copy() - - # Handle single vs batch embedding - if len(content) == 1: - final_model_kwargs["content"] = content[0] - else: - final_model_kwargs["contents"] = content - - # Set default task type if not provided - if "task_type" not in final_model_kwargs: - final_model_kwargs["task_type"] = "SEMANTIC_SIMILARITY" - - # Set default model if not provided - if "model" not in final_model_kwargs: - final_model_kwargs["model"] = "text-embedding-004" - - return final_model_kwargs + + final_kwargs = { + "model": model_kwargs.get("model", "gemini-embedding-001"), + "contents": contents, + } + + # Build EmbedContentConfig if we have additional parameters + config_params = {} + + if "task_type" in model_kwargs: + config_params["task_type"] = model_kwargs["task_type"] + + if "output_dimensionality" in model_kwargs: + config_params["output_dimensionality"] = model_kwargs["output_dimensionality"] + + if config_params: + final_kwargs["config"] = genai_types.EmbedContentConfig(**config_params) + + return final_kwargs @backoff.on_exception( backoff.expo, @@ -190,42 +170,33 @@ def convert_inputs_to_api_kwargs( ) def call(self, api_kwargs: Dict = {}, model_type: ModelType = ModelType.UNDEFINED): """Call Google AI embedding API. - + Args: api_kwargs: API parameters model_type: Should be ModelType.EMBEDDER - + Returns: Google AI embedding response """ if model_type != ModelType.EMBEDDER: raise ValueError(f"GoogleEmbedderClient only supports EMBEDDER model type") - - log.info(f"Google AI Embeddings API kwargs: {api_kwargs}") - + + log.info(f"Google AI Embeddings API call with model: {api_kwargs.get('model')}") + try: - # Use embed_content for single text or batch embedding - if "content" in api_kwargs: - # Single embedding - response = genai.embed_content(**api_kwargs) - elif "contents" in api_kwargs: - # Batch embedding - Google AI supports batch natively - contents = api_kwargs.pop("contents") - response = genai.embed_content(content=contents, **api_kwargs) - else: - raise ValueError("Either 'content' or 'contents' must be provided") - + # Use the new SDK client.models.embed_content + response = self._client.models.embed_content(**api_kwargs) return response - + except Exception as e: log.error(f"Error calling Google AI Embeddings API: {e}") raise async def acall(self, api_kwargs: Dict = {}, model_type: ModelType = ModelType.UNDEFINED): """Async call to Google AI embedding API. - - Note: Google AI Python client doesn't have async support yet, - so this falls back to synchronous call. + + Note: Using synchronous call as the google-genai SDK async support + may vary. For true async, consider using asyncio.to_thread. """ - # Google AI client doesn't have async support yet - return self.call(api_kwargs, model_type) \ No newline at end of file + import asyncio + return await asyncio.to_thread(self.call, api_kwargs, model_type) diff --git a/api/poetry.lock b/api/poetry.lock index e07df94ab..887775c40 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -955,105 +955,6 @@ files = [ {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, ] -[[package]] -name = "google-ai-generativelanguage" -version = "0.6.15" -description = "Google Ai Generativelanguage API client library" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c"}, - {file = "google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3"}, -] - -[package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" -proto-plus = [ - {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""}, - {version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""}, -] -protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" - -[[package]] -name = "google-api-core" -version = "2.25.2" -description = "Google API client core library" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version >= \"3.14\"" -files = [ - {file = "google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7"}, - {file = "google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300"}, -] - -[package.dependencies] -google-auth = ">=2.14.1,<3.0.0" -googleapis-common-protos = ">=1.56.2,<2.0.0" -grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} -grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} -proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" -requests = ">=2.18.0,<3.0.0" - -[package.extras] -async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] -grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] - -[[package]] -name = "google-api-core" -version = "2.28.1" -description = "Google API client core library" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version <= \"3.13\"" -files = [ - {file = "google_api_core-2.28.1-py3-none-any.whl", hash = "sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c"}, - {file = "google_api_core-2.28.1.tar.gz", hash = "sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8"}, -] - -[package.dependencies] -google-auth = ">=2.14.1,<3.0.0" -googleapis-common-protos = ">=1.56.2,<2.0.0" -grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""} -grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} -proto-plus = [ - {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, - {version = ">=1.22.3,<2.0.0"}, -] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" -requests = ">=2.18.0,<3.0.0" - -[package.extras] -async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] -grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] - -[[package]] -name = "google-api-python-client" -version = "2.187.0" -description = "Google API Client Library for Python" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "google_api_python_client-2.187.0-py3-none-any.whl", hash = "sha256:d8d0f6d85d7d1d10bdab32e642312ed572bdc98919f72f831b44b9a9cebba32f"}, - {file = "google_api_python_client-2.187.0.tar.gz", hash = "sha256:e98e8e8f49e1b5048c2f8276473d6485febc76c9c47892a8b4d1afa2c9ec8278"}, -] - -[package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0" -google-auth = ">=1.32.0,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" -google-auth-httplib2 = ">=0.2.0,<1.0.0" -httplib2 = ">=0.19.0,<1.0.0" -uritemplate = ">=3.0.1,<5" - [[package]] name = "google-auth" version = "2.45.0" @@ -1083,22 +984,6 @@ requests = ["requests (>=2.20.0,<3.0.0)"] testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] urllib3 = ["packaging", "urllib3"] -[[package]] -name = "google-auth-httplib2" -version = "0.3.0" -description = "Google Authentication Library: httplib2 transport" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "google_auth_httplib2-0.3.0-py3-none-any.whl", hash = "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776"}, - {file = "google_auth_httplib2-0.3.0.tar.gz", hash = "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b"}, -] - -[package.dependencies] -google-auth = ">=1.32.0,<3.0.0" -httplib2 = ">=0.19.0,<1.0.0" - [[package]] name = "google-genai" version = "1.56.0" @@ -1127,142 +1012,6 @@ websockets = ">=13.0.0,<15.1.0" aiohttp = ["aiohttp (<3.13.3)"] local-tokenizer = ["protobuf", "sentencepiece (>=0.2.0)"] -[[package]] -name = "google-generativeai" -version = "0.8.6" -description = "Google Generative AI High level API client library and tools." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa"}, -] - -[package.dependencies] -google-ai-generativelanguage = "0.6.15" -google-api-core = "*" -google-api-python-client = "*" -google-auth = ">=2.15.0" -protobuf = "*" -pydantic = "*" -tqdm = "*" -typing-extensions = "*" - -[package.extras] -dev = ["Pillow", "absl-py", "black", "ipython", "nose2", "pandas", "pytype", "pyyaml"] - -[[package]] -name = "googleapis-common-protos" -version = "1.72.0" -description = "Common protobufs used in Google APIs" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, - {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, -] - -[package.dependencies] -protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" - -[package.extras] -grpc = ["grpcio (>=1.44.0,<2.0.0)"] - -[[package]] -name = "grpcio" -version = "1.76.0" -description = "HTTP/2-based RPC framework" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "grpcio-1.76.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc"}, - {file = "grpcio-1.76.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde"}, - {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:035d90bc79eaa4bed83f524331d55e35820725c9fbb00ffa1904d5550ed7ede3"}, - {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4215d3a102bd95e2e11b5395c78562967959824156af11fa93d18fdd18050990"}, - {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:49ce47231818806067aea3324d4bf13825b658ad662d3b25fada0bdad9b8a6af"}, - {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8cc3309d8e08fd79089e13ed4819d0af72aa935dd8f435a195fd152796752ff2"}, - {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:971fd5a1d6e62e00d945423a567e42eb1fa678ba89072832185ca836a94daaa6"}, - {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d9adda641db7207e800a7f089068f6f645959f2df27e870ee81d44701dd9db3"}, - {file = "grpcio-1.76.0-cp310-cp310-win32.whl", hash = "sha256:063065249d9e7e0782d03d2bca50787f53bd0fb89a67de9a7b521c4a01f1989b"}, - {file = "grpcio-1.76.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6ae758eb08088d36812dd5d9af7a9859c05b1e0f714470ea243694b49278e7b"}, - {file = "grpcio-1.76.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2e1743fbd7f5fa713a1b0a8ac8ebabf0ec980b5d8809ec358d488e273b9cf02a"}, - {file = "grpcio-1.76.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:a8c2cf1209497cf659a667d7dea88985e834c24b7c3b605e6254cbb5076d985c"}, - {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:08caea849a9d3c71a542827d6df9d5a69067b0a1efbea8a855633ff5d9571465"}, - {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f0e34c2079d47ae9f6188211db9e777c619a21d4faba6977774e8fa43b085e48"}, - {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8843114c0cfce61b40ad48df65abcfc00d4dba82eae8718fab5352390848c5da"}, - {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8eddfb4d203a237da6f3cc8a540dad0517d274b5a1e9e636fd8d2c79b5c1d397"}, - {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:32483fe2aab2c3794101c2a159070584e5db11d0aa091b2c0ea9c4fc43d0d749"}, - {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dcfe41187da8992c5f40aa8c5ec086fa3672834d2be57a32384c08d5a05b4c00"}, - {file = "grpcio-1.76.0-cp311-cp311-win32.whl", hash = "sha256:2107b0c024d1b35f4083f11245c0e23846ae64d02f40b2b226684840260ed054"}, - {file = "grpcio-1.76.0-cp311-cp311-win_amd64.whl", hash = "sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d"}, - {file = "grpcio-1.76.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8"}, - {file = "grpcio-1.76.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280"}, - {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4"}, - {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11"}, - {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6"}, - {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8"}, - {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980"}, - {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882"}, - {file = "grpcio-1.76.0-cp312-cp312-win32.whl", hash = "sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958"}, - {file = "grpcio-1.76.0-cp312-cp312-win_amd64.whl", hash = "sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347"}, - {file = "grpcio-1.76.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2"}, - {file = "grpcio-1.76.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468"}, - {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3"}, - {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb"}, - {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae"}, - {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77"}, - {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03"}, - {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42"}, - {file = "grpcio-1.76.0-cp313-cp313-win32.whl", hash = "sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f"}, - {file = "grpcio-1.76.0-cp313-cp313-win_amd64.whl", hash = "sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8"}, - {file = "grpcio-1.76.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62"}, - {file = "grpcio-1.76.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd"}, - {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc"}, - {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a"}, - {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba"}, - {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09"}, - {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc"}, - {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc"}, - {file = "grpcio-1.76.0-cp314-cp314-win32.whl", hash = "sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e"}, - {file = "grpcio-1.76.0-cp314-cp314-win_amd64.whl", hash = "sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e"}, - {file = "grpcio-1.76.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:8ebe63ee5f8fa4296b1b8cfc743f870d10e902ca18afc65c68cf46fd39bb0783"}, - {file = "grpcio-1.76.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:3bf0f392c0b806905ed174dcd8bdd5e418a40d5567a05615a030a5aeddea692d"}, - {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b7604868b38c1bfd5cf72d768aedd7db41d78cb6a4a18585e33fb0f9f2363fd"}, - {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e6d1db20594d9daba22f90da738b1a0441a7427552cc6e2e3d1297aeddc00378"}, - {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d099566accf23d21037f18a2a63d323075bebace807742e4b0ac210971d4dd70"}, - {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ebea5cc3aa8ea72e04df9913492f9a96d9348db876f9dda3ad729cfedf7ac416"}, - {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0c37db8606c258e2ee0c56b78c62fc9dee0e901b5dbdcf816c2dd4ad652b8b0c"}, - {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ebebf83299b0cb1721a8859ea98f3a77811e35dce7609c5c963b9ad90728f886"}, - {file = "grpcio-1.76.0-cp39-cp39-win32.whl", hash = "sha256:0aaa82d0813fd4c8e589fac9b65d7dd88702555f702fb10417f96e2a2a6d4c0f"}, - {file = "grpcio-1.76.0-cp39-cp39-win_amd64.whl", hash = "sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a"}, - {file = "grpcio-1.76.0.tar.gz", hash = "sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73"}, -] - -[package.dependencies] -typing-extensions = ">=4.12,<5.0" - -[package.extras] -protobuf = ["grpcio-tools (>=1.76.0)"] - -[[package]] -name = "grpcio-status" -version = "1.71.2" -description = "Status proto mapping for gRPC" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3"}, - {file = "grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50"}, -] - -[package.dependencies] -googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.71.2" -protobuf = ">=5.26.1,<6.0dev" - [[package]] name = "h11" version = "0.16.0" @@ -1297,21 +1046,6 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<1.0)"] -[[package]] -name = "httplib2" -version = "0.31.0" -description = "A comprehensive HTTP client library." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "httplib2-0.31.0-py3-none-any.whl", hash = "sha256:b9cd78abea9b4e43a7714c6e0f8b6b8561a6fc1e95d5dbd367f5bf0ef35f5d24"}, - {file = "httplib2-0.31.0.tar.gz", hash = "sha256:ac7ab497c50975147d4f7b1ade44becc7df2f8954d42b38b3d69c515f531135c"}, -] - -[package.dependencies] -pyparsing = ">=3.0.4,<4" - [[package]] name = "httptools" version = "0.7.1" @@ -2181,45 +1915,6 @@ files = [ {file = "propcache-0.4.1.tar.gz", hash = "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d"}, ] -[[package]] -name = "proto-plus" -version = "1.27.0" -description = "Beautiful, Pythonic protocol buffers" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "proto_plus-1.27.0-py3-none-any.whl", hash = "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82"}, - {file = "proto_plus-1.27.0.tar.gz", hash = "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4"}, -] - -[package.dependencies] -protobuf = ">=3.19.0,<7.0.0" - -[package.extras] -testing = ["google-api-core (>=1.31.5)"] - -[[package]] -name = "protobuf" -version = "5.29.5" -description = "" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "protobuf-5.29.5-cp310-abi3-win32.whl", hash = "sha256:3f1c6468a2cfd102ff4703976138844f78ebd1fb45f49011afc5139e9e283079"}, - {file = "protobuf-5.29.5-cp310-abi3-win_amd64.whl", hash = "sha256:3f76e3a3675b4a4d867b52e4a5f5b78a2ef9565549d4037e06cf7b0942b1d3fc"}, - {file = "protobuf-5.29.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e38c5add5a311f2a6eb0340716ef9b039c1dfa428b28f25a7838ac329204a671"}, - {file = "protobuf-5.29.5-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:fa18533a299d7ab6c55a238bf8629311439995f2e7eca5caaff08663606e9015"}, - {file = "protobuf-5.29.5-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:63848923da3325e1bf7e9003d680ce6e14b07e55d0473253a690c3a8b8fd6e61"}, - {file = "protobuf-5.29.5-cp38-cp38-win32.whl", hash = "sha256:ef91363ad4faba7b25d844ef1ada59ff1604184c0bcd8b39b8a6bef15e1af238"}, - {file = "protobuf-5.29.5-cp38-cp38-win_amd64.whl", hash = "sha256:7318608d56b6402d2ea7704ff1e1e4597bee46d760e7e4dd42a3d45e24b87f2e"}, - {file = "protobuf-5.29.5-cp39-cp39-win32.whl", hash = "sha256:6f642dc9a61782fa72b90878af134c5afe1917c89a568cd3476d758d3c3a0736"}, - {file = "protobuf-5.29.5-cp39-cp39-win_amd64.whl", hash = "sha256:470f3af547ef17847a28e1f47200a1cbf0ba3ff57b7de50d22776607cd2ea353"}, - {file = "protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5"}, - {file = "protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84"}, -] - [[package]] name = "pyasn1" version = "0.6.1" @@ -2448,21 +2143,6 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] -[[package]] -name = "pyparsing" -version = "3.2.5" -description = "pyparsing - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e"}, - {file = "pyparsing-3.2.5.tar.gz", hash = "sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pytest" version = "8.4.2" @@ -2980,18 +2660,6 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" -[[package]] -name = "uritemplate" -version = "4.2.0" -description = "Implementation of RFC 6570 URI Templates" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686"}, - {file = "uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e"}, -] - [[package]] name = "urllib3" version = "2.5.0" @@ -3450,4 +3118,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "1933fdd12b1645fa83d6e66b8f1683ca2d927836a0bacd2a631f04adefb115d2" +content-hash = "58a5f053059c46611ee279577fec6cadda42ea713985d70a6ecac275271e9933" diff --git a/api/pyproject.toml b/api/pyproject.toml index 3fef92c11..2182695e4 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -13,7 +13,6 @@ fastapi = ">=0.95.0" uvicorn = { extras = ["standard"], version = ">=0.21.1" } pydantic = ">=2.0.0" google-genai = ">=1.0.0" -google-generativeai = ">=0.8.0" tiktoken = ">=0.5.0" adalflow = ">=0.1.0" numpy = ">=1.24.0" From 1baf317af831680149db5073c813bdb92895dde1 Mon Sep 17 00:00:00 2001 From: Diogo Soares Rodrigues Date: Fri, 19 Dec 2025 10:20:32 -0300 Subject: [PATCH 5/5] fix: add google-generativeai as transitive dependency for adalflow compatibility The adalflow framework's GoogleGenAIClient still requires the deprecated google-generativeai package internally. This is a temporary dependency until adalflow migrates to the new google-genai SDK. Our custom code (simple_chat.py, websocket_wiki.py, google_embedder_client.py) uses the new google-genai SDK directly. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- api/poetry.lock | 334 ++++++++++++++++++++++++++++++++++++++++++++- api/pyproject.toml | 1 + 2 files changed, 334 insertions(+), 1 deletion(-) diff --git a/api/poetry.lock b/api/poetry.lock index 887775c40..e07df94ab 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -955,6 +955,105 @@ files = [ {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, ] +[[package]] +name = "google-ai-generativelanguage" +version = "0.6.15" +description = "Google Ai Generativelanguage API client library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c"}, + {file = "google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" +proto-plus = [ + {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""}, +] +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" + +[[package]] +name = "google-api-core" +version = "2.25.2" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.14\"" +files = [ + {file = "google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7"}, + {file = "google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.56.2,<2.0.0" +grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +requests = ">=2.18.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] + +[[package]] +name = "google-api-core" +version = "2.28.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version <= \"3.13\"" +files = [ + {file = "google_api_core-2.28.1-py3-none-any.whl", hash = "sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c"}, + {file = "google_api_core-2.28.1.tar.gz", hash = "sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.56.2,<2.0.0" +grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""} +grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +requests = ">=2.18.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] + +[[package]] +name = "google-api-python-client" +version = "2.187.0" +description = "Google API Client Library for Python" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_api_python_client-2.187.0-py3-none-any.whl", hash = "sha256:d8d0f6d85d7d1d10bdab32e642312ed572bdc98919f72f831b44b9a9cebba32f"}, + {file = "google_api_python_client-2.187.0.tar.gz", hash = "sha256:e98e8e8f49e1b5048c2f8276473d6485febc76c9c47892a8b4d1afa2c9ec8278"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0" +google-auth = ">=1.32.0,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +google-auth-httplib2 = ">=0.2.0,<1.0.0" +httplib2 = ">=0.19.0,<1.0.0" +uritemplate = ">=3.0.1,<5" + [[package]] name = "google-auth" version = "2.45.0" @@ -984,6 +1083,22 @@ requests = ["requests (>=2.20.0,<3.0.0)"] testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] urllib3 = ["packaging", "urllib3"] +[[package]] +name = "google-auth-httplib2" +version = "0.3.0" +description = "Google Authentication Library: httplib2 transport" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_auth_httplib2-0.3.0-py3-none-any.whl", hash = "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776"}, + {file = "google_auth_httplib2-0.3.0.tar.gz", hash = "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b"}, +] + +[package.dependencies] +google-auth = ">=1.32.0,<3.0.0" +httplib2 = ">=0.19.0,<1.0.0" + [[package]] name = "google-genai" version = "1.56.0" @@ -1012,6 +1127,142 @@ websockets = ">=13.0.0,<15.1.0" aiohttp = ["aiohttp (<3.13.3)"] local-tokenizer = ["protobuf", "sentencepiece (>=0.2.0)"] +[[package]] +name = "google-generativeai" +version = "0.8.6" +description = "Google Generative AI High level API client library and tools." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa"}, +] + +[package.dependencies] +google-ai-generativelanguage = "0.6.15" +google-api-core = "*" +google-api-python-client = "*" +google-auth = ">=2.15.0" +protobuf = "*" +pydantic = "*" +tqdm = "*" +typing-extensions = "*" + +[package.extras] +dev = ["Pillow", "absl-py", "black", "ipython", "nose2", "pandas", "pytype", "pyyaml"] + +[[package]] +name = "googleapis-common-protos" +version = "1.72.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, + {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, +] + +[package.dependencies] +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0)"] + +[[package]] +name = "grpcio" +version = "1.76.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpcio-1.76.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc"}, + {file = "grpcio-1.76.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde"}, + {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:035d90bc79eaa4bed83f524331d55e35820725c9fbb00ffa1904d5550ed7ede3"}, + {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4215d3a102bd95e2e11b5395c78562967959824156af11fa93d18fdd18050990"}, + {file = "grpcio-1.76.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:49ce47231818806067aea3324d4bf13825b658ad662d3b25fada0bdad9b8a6af"}, + {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8cc3309d8e08fd79089e13ed4819d0af72aa935dd8f435a195fd152796752ff2"}, + {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:971fd5a1d6e62e00d945423a567e42eb1fa678ba89072832185ca836a94daaa6"}, + {file = "grpcio-1.76.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d9adda641db7207e800a7f089068f6f645959f2df27e870ee81d44701dd9db3"}, + {file = "grpcio-1.76.0-cp310-cp310-win32.whl", hash = "sha256:063065249d9e7e0782d03d2bca50787f53bd0fb89a67de9a7b521c4a01f1989b"}, + {file = "grpcio-1.76.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6ae758eb08088d36812dd5d9af7a9859c05b1e0f714470ea243694b49278e7b"}, + {file = "grpcio-1.76.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2e1743fbd7f5fa713a1b0a8ac8ebabf0ec980b5d8809ec358d488e273b9cf02a"}, + {file = "grpcio-1.76.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:a8c2cf1209497cf659a667d7dea88985e834c24b7c3b605e6254cbb5076d985c"}, + {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:08caea849a9d3c71a542827d6df9d5a69067b0a1efbea8a855633ff5d9571465"}, + {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f0e34c2079d47ae9f6188211db9e777c619a21d4faba6977774e8fa43b085e48"}, + {file = "grpcio-1.76.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8843114c0cfce61b40ad48df65abcfc00d4dba82eae8718fab5352390848c5da"}, + {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8eddfb4d203a237da6f3cc8a540dad0517d274b5a1e9e636fd8d2c79b5c1d397"}, + {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:32483fe2aab2c3794101c2a159070584e5db11d0aa091b2c0ea9c4fc43d0d749"}, + {file = "grpcio-1.76.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dcfe41187da8992c5f40aa8c5ec086fa3672834d2be57a32384c08d5a05b4c00"}, + {file = "grpcio-1.76.0-cp311-cp311-win32.whl", hash = "sha256:2107b0c024d1b35f4083f11245c0e23846ae64d02f40b2b226684840260ed054"}, + {file = "grpcio-1.76.0-cp311-cp311-win_amd64.whl", hash = "sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d"}, + {file = "grpcio-1.76.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8"}, + {file = "grpcio-1.76.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280"}, + {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4"}, + {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11"}, + {file = "grpcio-1.76.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6"}, + {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8"}, + {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980"}, + {file = "grpcio-1.76.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882"}, + {file = "grpcio-1.76.0-cp312-cp312-win32.whl", hash = "sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958"}, + {file = "grpcio-1.76.0-cp312-cp312-win_amd64.whl", hash = "sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347"}, + {file = "grpcio-1.76.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2"}, + {file = "grpcio-1.76.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468"}, + {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3"}, + {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb"}, + {file = "grpcio-1.76.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae"}, + {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77"}, + {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03"}, + {file = "grpcio-1.76.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42"}, + {file = "grpcio-1.76.0-cp313-cp313-win32.whl", hash = "sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f"}, + {file = "grpcio-1.76.0-cp313-cp313-win_amd64.whl", hash = "sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8"}, + {file = "grpcio-1.76.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62"}, + {file = "grpcio-1.76.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd"}, + {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc"}, + {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a"}, + {file = "grpcio-1.76.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba"}, + {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09"}, + {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc"}, + {file = "grpcio-1.76.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc"}, + {file = "grpcio-1.76.0-cp314-cp314-win32.whl", hash = "sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e"}, + {file = "grpcio-1.76.0-cp314-cp314-win_amd64.whl", hash = "sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e"}, + {file = "grpcio-1.76.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:8ebe63ee5f8fa4296b1b8cfc743f870d10e902ca18afc65c68cf46fd39bb0783"}, + {file = "grpcio-1.76.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:3bf0f392c0b806905ed174dcd8bdd5e418a40d5567a05615a030a5aeddea692d"}, + {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b7604868b38c1bfd5cf72d768aedd7db41d78cb6a4a18585e33fb0f9f2363fd"}, + {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e6d1db20594d9daba22f90da738b1a0441a7427552cc6e2e3d1297aeddc00378"}, + {file = "grpcio-1.76.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d099566accf23d21037f18a2a63d323075bebace807742e4b0ac210971d4dd70"}, + {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ebea5cc3aa8ea72e04df9913492f9a96d9348db876f9dda3ad729cfedf7ac416"}, + {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0c37db8606c258e2ee0c56b78c62fc9dee0e901b5dbdcf816c2dd4ad652b8b0c"}, + {file = "grpcio-1.76.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ebebf83299b0cb1721a8859ea98f3a77811e35dce7609c5c963b9ad90728f886"}, + {file = "grpcio-1.76.0-cp39-cp39-win32.whl", hash = "sha256:0aaa82d0813fd4c8e589fac9b65d7dd88702555f702fb10417f96e2a2a6d4c0f"}, + {file = "grpcio-1.76.0-cp39-cp39-win_amd64.whl", hash = "sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a"}, + {file = "grpcio-1.76.0.tar.gz", hash = "sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73"}, +] + +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + +[package.extras] +protobuf = ["grpcio-tools (>=1.76.0)"] + +[[package]] +name = "grpcio-status" +version = "1.71.2" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3"}, + {file = "grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.71.2" +protobuf = ">=5.26.1,<6.0dev" + [[package]] name = "h11" version = "0.16.0" @@ -1046,6 +1297,21 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<1.0)"] +[[package]] +name = "httplib2" +version = "0.31.0" +description = "A comprehensive HTTP client library." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "httplib2-0.31.0-py3-none-any.whl", hash = "sha256:b9cd78abea9b4e43a7714c6e0f8b6b8561a6fc1e95d5dbd367f5bf0ef35f5d24"}, + {file = "httplib2-0.31.0.tar.gz", hash = "sha256:ac7ab497c50975147d4f7b1ade44becc7df2f8954d42b38b3d69c515f531135c"}, +] + +[package.dependencies] +pyparsing = ">=3.0.4,<4" + [[package]] name = "httptools" version = "0.7.1" @@ -1915,6 +2181,45 @@ files = [ {file = "propcache-0.4.1.tar.gz", hash = "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d"}, ] +[[package]] +name = "proto-plus" +version = "1.27.0" +description = "Beautiful, Pythonic protocol buffers" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "proto_plus-1.27.0-py3-none-any.whl", hash = "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82"}, + {file = "proto_plus-1.27.0.tar.gz", hash = "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<7.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "5.29.5" +description = "" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "protobuf-5.29.5-cp310-abi3-win32.whl", hash = "sha256:3f1c6468a2cfd102ff4703976138844f78ebd1fb45f49011afc5139e9e283079"}, + {file = "protobuf-5.29.5-cp310-abi3-win_amd64.whl", hash = "sha256:3f76e3a3675b4a4d867b52e4a5f5b78a2ef9565549d4037e06cf7b0942b1d3fc"}, + {file = "protobuf-5.29.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e38c5add5a311f2a6eb0340716ef9b039c1dfa428b28f25a7838ac329204a671"}, + {file = "protobuf-5.29.5-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:fa18533a299d7ab6c55a238bf8629311439995f2e7eca5caaff08663606e9015"}, + {file = "protobuf-5.29.5-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:63848923da3325e1bf7e9003d680ce6e14b07e55d0473253a690c3a8b8fd6e61"}, + {file = "protobuf-5.29.5-cp38-cp38-win32.whl", hash = "sha256:ef91363ad4faba7b25d844ef1ada59ff1604184c0bcd8b39b8a6bef15e1af238"}, + {file = "protobuf-5.29.5-cp38-cp38-win_amd64.whl", hash = "sha256:7318608d56b6402d2ea7704ff1e1e4597bee46d760e7e4dd42a3d45e24b87f2e"}, + {file = "protobuf-5.29.5-cp39-cp39-win32.whl", hash = "sha256:6f642dc9a61782fa72b90878af134c5afe1917c89a568cd3476d758d3c3a0736"}, + {file = "protobuf-5.29.5-cp39-cp39-win_amd64.whl", hash = "sha256:470f3af547ef17847a28e1f47200a1cbf0ba3ff57b7de50d22776607cd2ea353"}, + {file = "protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5"}, + {file = "protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84"}, +] + [[package]] name = "pyasn1" version = "0.6.1" @@ -2143,6 +2448,21 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] +[[package]] +name = "pyparsing" +version = "3.2.5" +description = "pyparsing - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e"}, + {file = "pyparsing-3.2.5.tar.gz", hash = "sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "8.4.2" @@ -2660,6 +2980,18 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" +[[package]] +name = "uritemplate" +version = "4.2.0" +description = "Implementation of RFC 6570 URI Templates" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686"}, + {file = "uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e"}, +] + [[package]] name = "urllib3" version = "2.5.0" @@ -3118,4 +3450,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "58a5f053059c46611ee279577fec6cadda42ea713985d70a6ecac275271e9933" +content-hash = "1933fdd12b1645fa83d6e66b8f1683ca2d927836a0bacd2a631f04adefb115d2" diff --git a/api/pyproject.toml b/api/pyproject.toml index 2182695e4..5c6d9e6c3 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -13,6 +13,7 @@ fastapi = ">=0.95.0" uvicorn = { extras = ["standard"], version = ">=0.21.1" } pydantic = ">=2.0.0" google-genai = ">=1.0.0" +google-generativeai = ">=0.8.0" # Required by adalflow's GoogleGenAIClient - temporary until adalflow migrates to google-genai tiktoken = ">=0.5.0" adalflow = ">=0.1.0" numpy = ">=1.24.0"