feat:support cnb repo

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "i18n-ally.localesPaths": [
+        "src/messages"
+    ]
+}

README.md

@@ -2,7 +2,7 @@
 
 ![DeepWiki Banner](screenshots/Deepwiki.png)
 
-**DeepWiki** is my own implementation attempt of DeepWiki, automatically creates beautiful, interactive wikis for any GitHub, GitLab, or BitBucket repository! Just enter a repo name, and DeepWiki will:
+**DeepWiki** is my own implementation attempt of DeepWiki, automatically creates beautiful, interactive wikis for any GitHub, GitLab, BitBucket, or CNB repository! Just enter a repo name, and DeepWiki will:
 
 1. Analyze the code structure
 2. Generate comprehensive documentation
@@ -18,7 +18,7 @@
 
 ## ✨ Features
 
-- **Instant Documentation**: Turn any GitHub, GitLab or BitBucket repo into a wiki in seconds
+- **Instant Documentation**: Turn any GitHub, GitLab, BitBucket or CNB repo into a wiki in seconds
 - **Private Repository Support**: Securely access private repositories with personal access tokens
 - **Smart Analysis**: AI-powered understanding of code structure and relationships
 - **Beautiful Diagrams**: Automatic Mermaid diagrams to visualize architecture and data flow
@@ -104,15 +104,15 @@ yarn dev
 #### Step 4: Use DeepWiki!
 
 1. Open [http://localhost:3000](http://localhost:3000) in your browser
-2. Enter a GitHub, GitLab, or Bitbucket repository (like `https://github.com/openai/codex`, `https://github.com/microsoft/autogen`, `https://gitlab.com/gitlab-org/gitlab`, or `https://bitbucket.org/redradish/atlassian_app_versions`)
+2. Enter a GitHub, GitLab, Bitbucket, or CNB repository (like `https://github.com/openai/codex`, `https://github.com/microsoft/autogen`, `https://gitlab.com/gitlab-org/gitlab`, `https://bitbucket.org/redradish/atlassian_app_versions`, or `https://cnb.cool/learning-docker/project-1-jupyter`)
 3. For private repositories, click "+ Add access tokens" and enter your GitHub or GitLab personal access token
 4. Click "Generate Wiki" and watch the magic happen!
 
 ## 🔍 How It Works
 
 DeepWiki uses AI to:
 
-1. Clone and analyze the GitHub, GitLab, or Bitbucket repository (including private repos with token authentication)
+1. Clone and analyze the GitHub, GitLab, Bitbucket, or CNB repository (including private repos with token authentication)
 2. Create embeddings of the code for smart retrieval
 3. Generate documentation with context-aware AI (using Google Gemini, OpenAI, OpenRouter, Azure OpenAI, or local Ollama models)
 4. Create visual diagrams to explain code relationships
@@ -122,7 +122,7 @@ DeepWiki uses AI to:
 
 ```mermaid
 graph TD
-    A[User inputs GitHub/GitLab/Bitbucket repo] --> AA{Private repo?}
+    A[User inputs GitHub/GitLab/Bitbucket/CNB repo] --> AA{Private repo?}
     AA -->|Yes| AB[Add access token]
     AA -->|No| B[Clone Repository]
     AB --> B

api/api.py

@@ -32,9 +32,8 @@
     allow_headers=["*"],  # Allows all headers
 )
 
-# Helper function to get adalflow root path
-def get_adalflow_default_root_path():
-    return os.path.expanduser(os.path.join("~", ".adalflow"))
+# Import helper function from utils
+from .utils import get_adalflow_default_root_path
 
 # --- Pydantic Models ---
 class WikiPage(BaseModel):
@@ -393,13 +392,16 @@ def generate_json_export(repo_url: str, pages: List[WikiPage]) -> str:
 # Import the simplified chat implementation
 from api.simple_chat import chat_completions_stream
 from api.websocket_wiki import handle_websocket_chat
+# Removed httpx import - no longer needed since CNB API proxy was removed
 
 # Add the chat_completions_stream endpoint to the main app
 app.add_api_route("/chat/completions/stream", chat_completions_stream, methods=["POST"])
 
 # Add the WebSocket endpoint
 app.add_websocket_route("/ws/chat", handle_websocket_chat)
 
+# Removed CNB API proxy endpoint - backend handles all repository types via git clone
+
 # --- Wiki Cache Helper Functions ---
 
 WIKI_CACHE_DIR = os.path.join(get_adalflow_default_root_path(), "wikicache")

api/config/embedder.json

@@ -1,13 +1,23 @@
 {
   "embedder": {
     "client_class": "OpenAIClient",
-    "batch_size": 500,
+    "initialize_kwargs": {
+      "api_key": "${OPENAI_API_KEY}",
+      "base_url": "${OPENAI_BASE_URL}"
+    },
+    "batch_size": 10,
     "model_kwargs": {
-      "model": "text-embedding-3-small",
+      "model": "text-embedding-v4",
       "dimensions": 256,
       "encoding_format": "float"
     }
   },
+  "embedder_ollama": {
+    "client_class": "OllamaClient",
+    "model_kwargs": {
+      "model": "nomic-embed-text"
+    }
+  },
   "retriever": {
     "top_k": 20
   },

api/data_pipeline.py

@@ -9,7 +9,7 @@
 import base64
 import re
 import glob
-from adalflow.utils import get_adalflow_default_root_path
+from .utils import get_adalflow_default_root_path
 from adalflow.core.db import LocalDB
 from api.config import configs, DEFAULT_EXCLUDED_DIRS, DEFAULT_EXCLUDED_FILES
 from api.ollama_patch import OllamaDocumentProcessor
@@ -57,11 +57,12 @@ def count_tokens(text: str, is_ollama_embedder: bool = None) -> int:
 
 def download_repo(repo_url: str, local_path: str, type: str = "github", access_token: str = None) -> str:
     """
-    Downloads a Git repository (GitHub, GitLab, or Bitbucket) to a specified local path.
+    Downloads a Git repository (GitHub, GitLab, Bitbucket, CNB, or other Git hosting services) to a specified local path.
 
     Args:
         repo_url (str): The URL of the Git repository to clone.
         local_path (str): The local directory where the repository will be cloned.
+        type (str): The type of repository (github, gitlab, bitbucket, cnb, web, local).
         access_token (str, optional): Access token for private repositories.
 
     Returns:
@@ -101,6 +102,10 @@ def download_repo(repo_url: str, local_path: str, type: str = "github", access_t
             elif type == "bitbucket":
                 # Format: https://x-token-auth:{token}@bitbucket.org/owner/repo.git
                 clone_url = urlunparse((parsed.scheme, f"x-token-auth:{access_token}@{parsed.netloc}", parsed.path, '', '', ''))
+            elif type == "cnb" or type == "web":
+                # For CNB and generic web-based Git repositories, use a generic token format
+                # This works for most Git hosting services that support HTTP basic auth
+                clone_url = urlunparse((parsed.scheme, f"{access_token}@{parsed.netloc}", parsed.path, '', '', ''))
 
             logger.info("Using access token for authentication")
 
@@ -648,13 +653,49 @@ def get_bitbucket_file_content(repo_url: str, file_path: str, access_token: str
         raise ValueError(f"Failed to get file content: {str(e)}")
 
 
+def get_web_file_content(repo_url: str, file_path: str, repo_type: str = "cnb") -> str:
+    """
+    Retrieves the content of a file from a locally cloned web-based Git repository.
+
+    Args:
+        repo_url (str): The URL of the repository
+        file_path (str): The path to the file within the repository
+        repo_type (str): The type of repository (cnb, web, etc.)
+
+    Returns:
+        str: The content of the file as a string
+
+    Raises:
+        ValueError: If the file cannot be found or read
+    """
+    try:
+        # Use the unified utility function to get the local repository path
+        from .utils import get_repo_local_path
+        local_repo_path = get_repo_local_path(repo_url, repo_type)
+
+        # Construct the full file path
+        full_file_path = os.path.join(local_repo_path, file_path)
+
+        # Check if file exists
+        if not os.path.exists(full_file_path):
+            raise ValueError(f"File not found: {file_path}")
+
+        # Read and return file content
+        with open(full_file_path, 'r', encoding='utf-8') as f:
+            return f.read()
+
+    except Exception as e:
+        raise ValueError(f"Failed to get file content from local repository: {str(e)}")
+
+
 def get_file_content(repo_url: str, file_path: str, type: str = "github", access_token: str = None) -> str:
     """
-    Retrieves the content of a file from a Git repository (GitHub or GitLab).
+    Retrieves the content of a file from a Git repository (GitHub, GitLab, Bitbucket, CNB, or other Git hosting services).
 
     Args:
         repo_url (str): The URL of the repository
         file_path (str): The path to the file within the repository
+        type (str): The type of repository (github, gitlab, bitbucket, cnb, web, local)
         access_token (str, optional): Access token for private repositories
 
     Returns:
@@ -669,8 +710,11 @@ def get_file_content(repo_url: str, file_path: str, type: str = "github", access
         return get_gitlab_file_content(repo_url, file_path, access_token)
     elif type == "bitbucket":
         return get_bitbucket_file_content(repo_url, file_path, access_token)
+    elif type == "cnb" or type == "web":
+        # For cnb and web-type repositories, read from local cloned repository
+        return get_web_file_content(repo_url, file_path, type)
     else:
-        raise ValueError("Unsupported repository URL. Only GitHub and GitLab are supported.")
+        raise ValueError("Unsupported repository type. Supported types: github, gitlab, bitbucket, cnb, web.")
 
 class DatabaseManager:
     """
@@ -715,19 +759,9 @@ def reset_database(self):
         self.repo_paths = None
 
     def _extract_repo_name_from_url(self, repo_url_or_path: str, repo_type: str) -> str:
-        # Extract owner and repo name to create unique identifier
-        url_parts = repo_url_or_path.rstrip('/').split('/')
-
-        if repo_type in ["github", "gitlab", "bitbucket"] and len(url_parts) >= 5:
-            # GitHub URL format: https://github.com/owner/repo
-            # GitLab URL format: https://gitlab.com/owner/repo or https://gitlab.com/group/subgroup/repo
-            # Bitbucket URL format: https://bitbucket.org/owner/repo
-            owner = url_parts[-2]
-            repo = url_parts[-1].replace(".git", "")
-            repo_name = f"{owner}_{repo}"
-        else:
-            repo_name = url_parts[-1].replace(".git", "")
-        return repo_name
+        # Use the unified utility function for consistent repo name extraction
+        from .utils import extract_repo_name_from_url
+        return extract_repo_name_from_url(repo_url_or_path, repo_type)
 
     def _create_repo(self, repo_url_or_path: str, repo_type: str = "github", access_token: str = None) -> None:
         """

api/requirements.txt

@@ -17,4 +17,5 @@ boto3>=1.34.0
 websockets>=11.0.3
 azure-identity>=1.12.0
 azure-core>=1.24.0
+# Removed httpx>=0.24.0 - no longer needed since CNB API proxy was removed
 

api/utils.py

@@ -0,0 +1,65 @@
+import os
+from typing import Tuple
+
+
+def get_adalflow_default_root_path() -> str:
+    """
+    Get the default root path for adalflow data storage.
+
+    Returns:
+        str: The default root path
+    """
+    return os.path.expanduser("~/.adalflow")
+
+
+def extract_repo_name_from_url(repo_url: str, repo_type: str) -> str:
+    """
+    Extract repository name from URL to create a unique identifier.
+    This function provides consistent repo name generation across the application.
+
+    Args:
+        repo_url (str): The repository URL
+        repo_type (str): The type of repository (github, gitlab, bitbucket, cnb, web, local)
+
+    Returns:
+        str: The repository name identifier
+
+    Examples:
+        For GitHub: https://github.com/owner/repo -> "owner_repo"
+        For CNB: https://cnb.cool/opencamp/learning-docker/project-1-jupyter -> "project-1-jupyter"
+        For GitLab: https://gitlab.com/group/subgroup/repo -> "subgroup_repo"
+    """
+    # Extract parts from URL
+    url_parts = repo_url.rstrip('/').split('/')
+
+    if repo_type in ["github", "gitlab", "bitbucket"] and len(url_parts) >= 5:
+        # Traditional git hosting services: use owner_repo format
+        # GitHub URL format: https://github.com/owner/repo
+        # GitLab URL format: https://gitlab.com/owner/repo or https://gitlab.com/group/subgroup/repo
+        # Bitbucket URL format: https://bitbucket.org/owner/repo
+        owner = url_parts[-2]
+        repo = url_parts[-1].replace(".git", "")
+        repo_name = f"{owner}_{repo}"
+    else:
+        # For CNB and other web-based repositories: use only the repo name
+        # This handles cases like https://cnb.cool/opencamp/learning-docker/project-1-jupyter
+        # where the directory structure is complex but we only want the final repo name
+        repo_name = url_parts[-1].replace(".git", "")
+
+    return repo_name
+
+
+def get_repo_local_path(repo_url: str, repo_type: str) -> str:
+    """
+    Get the local path for a repository based on its URL and type.
+
+    Args:
+        repo_url (str): The repository URL
+        repo_type (str): The type of repository (github, gitlab, bitbucket, cnb, web, local)
+
+    Returns:
+        str: The local path where the repository should be stored
+    """
+    root_path = get_adalflow_default_root_path()
+    repo_name = extract_repo_name_from_url(repo_url, repo_type)
+    return os.path.join(root_path, "repos", repo_name)

next.config.ts

@@ -63,6 +63,7 @@ const nextConfig: NextConfig = {
         source: '/api/lang/config',
         destination: `${TARGET_SERVER_BASE_URL}/lang/config`,
       },
+      // Removed CNB API proxy - backend handles all repository types via git clone
     ];
   },
 };