Merge pull request #1915 from Comfy-Org/feat/implement-batch-tracking-clean

[feat] Implement comprehensive batch tracking and OpenAPI-driven data models

.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, feat/*, fix/* ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  validate-openapi:
+    name: Validate OpenAPI Specification
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Check if OpenAPI changed
+      id: openapi-changed
+      uses: tj-actions/changed-files@v44
+      with:
+        files: openapi.yaml
+    
+    - name: Setup Node.js
+      if: steps.openapi-changed.outputs.any_changed == 'true'
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+    
+    - name: Install Redoc CLI
+      if: steps.openapi-changed.outputs.any_changed == 'true'
+      run: |
+        npm install -g @redocly/cli
+    
+    - name: Validate OpenAPI specification
+      if: steps.openapi-changed.outputs.any_changed == 'true'
+      run: |
+        redocly lint openapi.yaml
+
+  code-quality:
+    name: Code Quality Checks
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch all history for proper diff
+    
+    - name: Get changed Python files
+      id: changed-py-files
+      uses: tj-actions/changed-files@v44
+      with:
+        files: |
+          **/*.py
+        files_ignore: |
+          comfyui_manager/legacy/**
+          
+    - name: Setup Python
+      if: steps.changed-py-files.outputs.any_changed == 'true'
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    
+    - name: Install dependencies
+      if: steps.changed-py-files.outputs.any_changed == 'true'
+      run: |
+        pip install ruff
+    
+    - name: Run ruff linting on changed files
+      if: steps.changed-py-files.outputs.any_changed == 'true'
+      run: |
+        echo "Changed files: ${{ steps.changed-py-files.outputs.all_changed_files }}"
+        echo "${{ steps.changed-py-files.outputs.all_changed_files }}" | xargs -r ruff check

.gitignore

@@ -19,5 +19,6 @@ pip_overrides.json
 check2.sh
 /venv/
 build
+dist
 *.egg-info
 .env

comfyui_manager/data_models/README.md

@@ -0,0 +1,67 @@
+# Data Models
+
+This directory contains Pydantic models for ComfyUI Manager, providing type safety, validation, and serialization for the API and internal data structures.
+
+## Overview
+
+- `generated_models.py` - All models auto-generated from OpenAPI spec
+- `__init__.py` - Package exports for all models
+
+**Note**: All models are now auto-generated from the OpenAPI specification. Manual model files (`task_queue.py`, `state_management.py`) have been deprecated in favor of a single source of truth.
+
+## Generating Types from OpenAPI
+
+The state management models are automatically generated from the OpenAPI specification using `datamodel-codegen`. This ensures type safety and consistency between the API specification and the Python code.
+
+### Prerequisites
+
+Install the code generator:
+```bash
+pipx install datamodel-code-generator
+```
+
+### Generation Command
+
+To regenerate all models after updating the OpenAPI spec:
+
+```bash
+datamodel-codegen \
+  --use-subclass-enum \
+  --field-constraints \
+  --strict-types bytes \
+  --input openapi.yaml \
+  --output comfyui_manager/data_models/generated_models.py \
+  --output-model-type pydantic_v2.BaseModel
+```
+
+### When to Regenerate
+
+You should regenerate the models when:
+
+1. **Adding new API endpoints** that return new data structures
+2. **Modifying existing schemas** in the OpenAPI specification  
+3. **Adding new state management features** that require new models
+
+### Important Notes
+
+- **Single source of truth**: All models are now generated from `openapi.yaml`
+- **No manual models**: All previously manual models have been migrated to the OpenAPI spec
+- **OpenAPI requirements**: New schemas must be referenced in API paths to be generated by datamodel-codegen
+- **Validation**: Always validate the OpenAPI spec before generation:
+  ```bash
+  python3 -c "import yaml; yaml.safe_load(open('openapi.yaml'))"
+  ```
+
+### Example: Adding New State Models
+
+1. Add your schema to `openapi.yaml` under `components/schemas/`
+2. Reference the schema in an API endpoint response
+3. Run the generation command above
+4. Update `__init__.py` to export the new models
+5. Import and use the models in your code
+
+### Troubleshooting
+
+- **Models not generated**: Ensure schemas are under `components/schemas/` (not `parameters/`)
+- **Missing models**: Verify schemas are referenced in at least one API path
+- **Import errors**: Check that new models are added to `__init__.py` exports

comfyui_manager/data_models/__init__.py

@@ -0,0 +1,125 @@
+"""
+Data models for ComfyUI Manager.
+
+This package contains Pydantic models used throughout the ComfyUI Manager
+for data validation, serialization, and type safety.
+
+All models are auto-generated from the OpenAPI specification to ensure
+consistency between the API and implementation.
+"""
+
+from .generated_models import (
+    # Core Task Queue Models
+    QueueTaskItem,
+    TaskHistoryItem,
+    TaskStateMessage,
+    TaskExecutionStatus,
+    
+    # WebSocket Message Models
+    MessageTaskDone,
+    MessageTaskStarted,
+    MessageTaskFailed,
+    MessageUpdate,
+    ManagerMessageName,
+    
+    # State Management Models
+    BatchExecutionRecord,
+    ComfyUISystemState,
+    BatchOperation,
+    InstalledNodeInfo,
+    InstalledModelInfo,
+    ComfyUIVersionInfo,
+    
+    # Other models
+    OperationType,
+    OperationResult,
+    ManagerPackInfo,
+    ManagerPackInstalled,
+    SelectedVersion,
+    ManagerChannel,
+    ManagerDatabaseSource,
+    ManagerPackState,
+    ManagerPackInstallType,
+    ManagerPack,
+    InstallPackParams,
+    UpdatePackParams,
+    UpdateAllPacksParams,
+    UpdateComfyUIParams,
+    FixPackParams,
+    UninstallPackParams,
+    DisablePackParams,
+    EnablePackParams,
+    UpdateAllQueryParams,
+    UpdateComfyUIQueryParams,
+    ComfyUISwitchVersionQueryParams,
+    QueueStatus,
+    ManagerMappings,
+    ModelMetadata,
+    NodePackageMetadata,
+    SnapshotItem,
+    Error,
+    InstalledPacksResponse,
+    HistoryResponse,
+    HistoryListResponse,
+    InstallType,
+    SecurityLevel,
+    RiskLevel,
+)
+
+__all__ = [
+    # Core Task Queue Models
+    "QueueTaskItem",
+    "TaskHistoryItem",
+    "TaskStateMessage",
+    "TaskExecutionStatus",
+    
+    # WebSocket Message Models
+    "MessageTaskDone",
+    "MessageTaskStarted",
+    "MessageTaskFailed",
+    "MessageUpdate",
+    "ManagerMessageName",
+    
+    # State Management Models
+    "BatchExecutionRecord",
+    "ComfyUISystemState",
+    "BatchOperation",
+    "InstalledNodeInfo",
+    "InstalledModelInfo",
+    "ComfyUIVersionInfo",
+    
+    # Other models
+    "OperationType",
+    "OperationResult",
+    "ManagerPackInfo",
+    "ManagerPackInstalled",
+    "SelectedVersion",
+    "ManagerChannel",
+    "ManagerDatabaseSource",
+    "ManagerPackState",
+    "ManagerPackInstallType",
+    "ManagerPack",
+    "InstallPackParams",
+    "UpdatePackParams",
+    "UpdateAllPacksParams",
+    "UpdateComfyUIParams",
+    "FixPackParams",
+    "UninstallPackParams",
+    "DisablePackParams",
+    "EnablePackParams",
+    "UpdateAllQueryParams",
+    "UpdateComfyUIQueryParams",
+    "ComfyUISwitchVersionQueryParams",
+    "QueueStatus",
+    "ManagerMappings",
+    "ModelMetadata",
+    "NodePackageMetadata",
+    "SnapshotItem",
+    "Error",
+    "InstalledPacksResponse",
+    "HistoryResponse",
+    "HistoryListResponse",
+    "InstallType",
+    "SecurityLevel",
+    "RiskLevel",
+]

comfyui_manager/data_models/generated_models.py

@@ -0,0 +1,537 @@
+# generated by datamodel-codegen:
+#   filename:  openapi.yaml
+#   timestamp: 2025-06-21T23:40:24+00:00
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field, RootModel
+
+
+class OperationType(str, Enum):
+    install = "install"
+    uninstall = "uninstall"
+    update = "update"
+    update_comfyui = "update-comfyui"
+    fix = "fix"
+    disable = "disable"
+    enable = "enable"
+    install_model = "install-model"
+
+
+class OperationResult(str, Enum):
+    success = "success"
+    failed = "failed"
+    skipped = "skipped"
+    error = "error"
+    skip = "skip"
+
+
+class TaskExecutionStatus(BaseModel):
+    status_str: OperationResult
+    completed: bool = Field(..., description="Whether the task completed")
+    messages: List[str] = Field(..., description="Additional status messages")
+
+
+class ManagerMessageName(str, Enum):
+    cm_task_completed = "cm-task-completed"
+    cm_task_started = "cm-task-started"
+    cm_queue_status = "cm-queue-status"
+
+
+class ManagerPackInfo(BaseModel):
+    id: str = Field(
+        ...,
+        description="Either github-author/github-repo or name of pack from the registry",
+    )
+    version: str = Field(..., description="Semantic version or Git commit hash")
+    ui_id: Optional[str] = Field(None, description="Task ID - generated internally")
+
+
+class ManagerPackInstalled(BaseModel):
+    ver: str = Field(
+        ...,
+        description="The version of the pack that is installed (Git commit hash or semantic version)",
+    )
+    cnr_id: Optional[str] = Field(
+        None, description="The name of the pack if installed from the registry"
+    )
+    aux_id: Optional[str] = Field(
+        None,
+        description="The name of the pack if installed from github (author/repo-name format)",
+    )
+    enabled: bool = Field(..., description="Whether the pack is enabled")
+
+
+class SelectedVersion(str, Enum):
+    latest = "latest"
+    nightly = "nightly"
+
+
+class ManagerChannel(str, Enum):
+    default = "default"
+    recent = "recent"
+    legacy = "legacy"
+    forked = "forked"
+    dev = "dev"
+    tutorial = "tutorial"
+
+
+class ManagerDatabaseSource(str, Enum):
+    remote = "remote"
+    local = "local"
+    cache = "cache"
+
+
+class ManagerPackState(str, Enum):
+    installed = "installed"
+    disabled = "disabled"
+    not_installed = "not_installed"
+    import_failed = "import_failed"
+    needs_update = "needs_update"
+
+
+class ManagerPackInstallType(str, Enum):
+    git_clone = "git-clone"
+    copy = "copy"
+    cnr = "cnr"
+
+
+class SecurityLevel(str, Enum):
+    strong = "strong"
+    normal = "normal"
+    normal_ = "normal-"
+    weak = "weak"
+
+
+class RiskLevel(str, Enum):
+    block = "block"
+    high = "high"
+    middle = "middle"
+
+
+class UpdateState(Enum):
+    false = "false"
+    true = "true"
+
+
+class ManagerPack(ManagerPackInfo):
+    author: Optional[str] = Field(
+        None, description="Pack author name or 'Unclaimed' if added via GitHub crawl"
+    )
+    files: Optional[List[str]] = Field(
+        None,
+        description="Repository URLs for installation (typically contains one GitHub URL)",
+    )
+    reference: Optional[str] = Field(
+        None, description="The type of installation reference"
+    )
+    title: Optional[str] = Field(None, description="The display name of the pack")
+    cnr_latest: Optional[SelectedVersion] = None
+    repository: Optional[str] = Field(None, description="GitHub repository URL")
+    state: Optional[ManagerPackState] = None
+    update_state: Optional[UpdateState] = Field(
+        None, alias="update-state", description="Update availability status"
+    )
+    stars: Optional[int] = Field(None, description="GitHub stars count")
+    last_update: Optional[datetime] = Field(None, description="Last update timestamp")
+    health: Optional[str] = Field(None, description="Health status of the pack")
+    description: Optional[str] = Field(None, description="Pack description")
+    trust: Optional[bool] = Field(None, description="Whether the pack is trusted")
+    install_type: Optional[ManagerPackInstallType] = None
+
+
+class InstallPackParams(ManagerPackInfo):
+    selected_version: Union[str, SelectedVersion] = Field(
+        ..., description="Semantic version, Git commit hash, latest, or nightly"
+    )
+    repository: Optional[str] = Field(
+        None,
+        description="GitHub repository URL (required if selected_version is nightly)",
+    )
+    pip: Optional[List[str]] = Field(None, description="PyPi dependency names")
+    mode: ManagerDatabaseSource
+    channel: ManagerChannel
+    skip_post_install: Optional[bool] = Field(
+        None, description="Whether to skip post-installation steps"
+    )
+
+
+class UpdateAllPacksParams(BaseModel):
+    mode: Optional[ManagerDatabaseSource] = None
+    ui_id: Optional[str] = Field(None, description="Task ID - generated internally")
+
+
+class UpdatePackParams(BaseModel):
+    node_name: str = Field(..., description="Name of the node package to update")
+    node_ver: Optional[str] = Field(
+        None, description="Current version of the node package"
+    )
+
+
+class UpdateComfyUIParams(BaseModel):
+    is_stable: Optional[bool] = Field(
+        True,
+        description="Whether to update to stable version (true) or nightly (false)",
+    )
+    target_version: Optional[str] = Field(
+        None,
+        description="Specific version to switch to (for version switching operations)",
+    )
+
+
+class FixPackParams(BaseModel):
+    node_name: str = Field(..., description="Name of the node package to fix")
+    node_ver: str = Field(..., description="Version of the node package")
+
+
+class UninstallPackParams(BaseModel):
+    node_name: str = Field(..., description="Name of the node package to uninstall")
+    is_unknown: Optional[bool] = Field(
+        False, description="Whether this is an unknown/unregistered package"
+    )
+
+
+class DisablePackParams(BaseModel):
+    node_name: str = Field(..., description="Name of the node package to disable")
+    is_unknown: Optional[bool] = Field(
+        False, description="Whether this is an unknown/unregistered package"
+    )
+
+
+class EnablePackParams(BaseModel):
+    cnr_id: str = Field(
+        ..., description="ComfyUI Node Registry ID of the package to enable"
+    )
+
+
+class UpdateAllQueryParams(BaseModel):
+    client_id: str = Field(
+        ..., description="Client identifier that initiated the request"
+    )
+    ui_id: str = Field(..., description="Base UI identifier for task tracking")
+    mode: Optional[ManagerDatabaseSource] = None
+
+
+class UpdateComfyUIQueryParams(BaseModel):
+    client_id: str = Field(
+        ..., description="Client identifier that initiated the request"
+    )
+    ui_id: str = Field(..., description="UI identifier for task tracking")
+    stable: Optional[bool] = Field(
+        True,
+        description="Whether to update to stable version (true) or nightly (false)",
+    )
+
+
+class ComfyUISwitchVersionQueryParams(BaseModel):
+    ver: str = Field(..., description="Version to switch to")
+    client_id: str = Field(
+        ..., description="Client identifier that initiated the request"
+    )
+    ui_id: str = Field(..., description="UI identifier for task tracking")
+
+
+class QueueStatus(BaseModel):
+    total_count: int = Field(
+        ..., description="Total number of tasks (pending + running)"
+    )
+    done_count: int = Field(..., description="Number of completed tasks")
+    in_progress_count: int = Field(..., description="Number of tasks currently running")
+    pending_count: Optional[int] = Field(
+        None, description="Number of tasks waiting to be executed"
+    )
+    is_processing: bool = Field(..., description="Whether the task worker is active")
+    client_id: Optional[str] = Field(
+        None, description="Client ID (when filtered by client)"
+    )
+
+
+class ManagerMappings1(BaseModel):
+    title_aux: Optional[str] = Field(None, description="The display name of the pack")
+
+
+class ManagerMappings(
+    RootModel[Optional[Dict[str, List[Union[List[str], ManagerMappings1]]]]]
+):
+    root: Optional[Dict[str, List[Union[List[str], ManagerMappings1]]]] = Field(
+        None, description="Tuple of [node_names, metadata]"
+    )
+
+
+class ModelMetadata(BaseModel):
+    name: str = Field(..., description="Name of the model")
+    type: str = Field(..., description="Type of model")
+    base: Optional[str] = Field(None, description="Base model type")
+    save_path: Optional[str] = Field(None, description="Path for saving the model")
+    url: str = Field(..., description="Download URL")
+    filename: str = Field(..., description="Target filename")
+    ui_id: Optional[str] = Field(None, description="ID for UI reference")
+
+
+class InstallType(str, Enum):
+    git = "git"
+    copy = "copy"
+    pip = "pip"
+
+
+class NodePackageMetadata(BaseModel):
+    title: Optional[str] = Field(None, description="Display name of the node package")
+    name: Optional[str] = Field(None, description="Repository/package name")
+    files: Optional[List[str]] = Field(None, description="Source URLs for the package")
+    description: Optional[str] = Field(
+        None, description="Description of the node package functionality"
+    )
+    install_type: Optional[InstallType] = Field(None, description="Installation method")
+    version: Optional[str] = Field(None, description="Version identifier")
+    id: Optional[str] = Field(
+        None, description="Unique identifier for the node package"
+    )
+    ui_id: Optional[str] = Field(None, description="ID for UI reference")
+    channel: Optional[str] = Field(None, description="Source channel")
+    mode: Optional[str] = Field(None, description="Source mode")
+
+
+class SnapshotItem(RootModel[str]):
+    root: str = Field(..., description="Name of the snapshot")
+
+
+class Error(BaseModel):
+    error: str = Field(..., description="Error message")
+
+
+class InstalledPacksResponse(RootModel[Optional[Dict[str, ManagerPackInstalled]]]):
+    root: Optional[Dict[str, ManagerPackInstalled]] = None
+
+
+class HistoryListResponse(BaseModel):
+    ids: Optional[List[str]] = Field(
+        None, description="List of available batch history IDs"
+    )
+
+
+class InstalledNodeInfo(BaseModel):
+    name: str = Field(..., description="Node package name")
+    version: str = Field(..., description="Installed version")
+    repository_url: Optional[str] = Field(None, description="Git repository URL")
+    install_method: str = Field(
+        ..., description="Installation method (cnr, git, pip, etc.)"
+    )
+    enabled: Optional[bool] = Field(
+        True, description="Whether the node is currently enabled"
+    )
+    install_date: Optional[datetime] = Field(
+        None, description="ISO timestamp of installation"
+    )
+
+
+class InstalledModelInfo(BaseModel):
+    name: str = Field(..., description="Model filename")
+    path: str = Field(..., description="Full path to model file")
+    type: str = Field(..., description="Model type (checkpoint, lora, vae, etc.)")
+    size_bytes: Optional[int] = Field(None, description="File size in bytes", ge=0)
+    hash: Optional[str] = Field(None, description="Model file hash for verification")
+    install_date: Optional[datetime] = Field(
+        None, description="ISO timestamp when added"
+    )
+
+
+class ComfyUIVersionInfo(BaseModel):
+    version: str = Field(..., description="ComfyUI version string")
+    commit_hash: Optional[str] = Field(None, description="Git commit hash")
+    branch: Optional[str] = Field(None, description="Git branch name")
+    is_stable: Optional[bool] = Field(
+        False, description="Whether this is a stable release"
+    )
+    last_updated: Optional[datetime] = Field(
+        None, description="ISO timestamp of last update"
+    )
+
+
+class BatchOperation(BaseModel):
+    operation_id: str = Field(..., description="Unique operation identifier")
+    operation_type: OperationType
+    target: str = Field(
+        ..., description="Target of the operation (node name, model name, etc.)"
+    )
+    target_version: Optional[str] = Field(
+        None, description="Target version for the operation"
+    )
+    result: OperationResult
+    error_message: Optional[str] = Field(
+        None, description="Error message if operation failed"
+    )
+    start_time: datetime = Field(
+        ..., description="ISO timestamp when operation started"
+    )
+    end_time: Optional[datetime] = Field(
+        None, description="ISO timestamp when operation completed"
+    )
+    client_id: Optional[str] = Field(
+        None, description="Client that initiated the operation"
+    )
+
+
+class ComfyUISystemState(BaseModel):
+    snapshot_time: datetime = Field(
+        ..., description="ISO timestamp when snapshot was taken"
+    )
+    comfyui_version: ComfyUIVersionInfo
+    frontend_version: Optional[str] = Field(
+        None, description="ComfyUI frontend version if available"
+    )
+    python_version: str = Field(..., description="Python interpreter version")
+    platform_info: str = Field(
+        ..., description="Operating system and platform information"
+    )
+    installed_nodes: Optional[Dict[str, InstalledNodeInfo]] = Field(
+        None, description="Map of installed node packages by name"
+    )
+    installed_models: Optional[Dict[str, InstalledModelInfo]] = Field(
+        None, description="Map of installed models by name"
+    )
+    manager_config: Optional[Dict[str, Any]] = Field(
+        None, description="ComfyUI Manager configuration settings"
+    )
+    comfyui_root_path: Optional[str] = Field(
+        None, description="ComfyUI root installation directory"
+    )
+    model_paths: Optional[Dict[str, List[str]]] = Field(
+        None, description="Map of model types to their configured paths"
+    )
+    manager_version: Optional[str] = Field(None, description="ComfyUI Manager version")
+    security_level: Optional[SecurityLevel] = None
+    network_mode: Optional[str] = Field(
+        None, description="Network mode (online, offline, private)"
+    )
+    cli_args: Optional[Dict[str, Any]] = Field(
+        None, description="Selected ComfyUI CLI arguments"
+    )
+    custom_nodes_count: Optional[int] = Field(
+        None, description="Total number of custom node packages", ge=0
+    )
+    failed_imports: Optional[List[str]] = Field(
+        None, description="List of custom nodes that failed to import"
+    )
+    pip_packages: Optional[Dict[str, str]] = Field(
+        None, description="Map of installed pip packages to their versions"
+    )
+    embedded_python: Optional[bool] = Field(
+        None,
+        description="Whether ComfyUI is running from an embedded Python distribution",
+    )
+
+
+class BatchExecutionRecord(BaseModel):
+    batch_id: str = Field(..., description="Unique batch identifier")
+    start_time: datetime = Field(..., description="ISO timestamp when batch started")
+    end_time: Optional[datetime] = Field(
+        None, description="ISO timestamp when batch completed"
+    )
+    state_before: ComfyUISystemState
+    state_after: Optional[ComfyUISystemState] = Field(
+        None, description="System state after batch execution"
+    )
+    operations: Optional[List[BatchOperation]] = Field(
+        None, description="List of operations performed in this batch"
+    )
+    total_operations: Optional[int] = Field(
+        0, description="Total number of operations in batch", ge=0
+    )
+    successful_operations: Optional[int] = Field(
+        0, description="Number of successful operations", ge=0
+    )
+    failed_operations: Optional[int] = Field(
+        0, description="Number of failed operations", ge=0
+    )
+    skipped_operations: Optional[int] = Field(
+        0, description="Number of skipped operations", ge=0
+    )
+
+
+class QueueTaskItem(BaseModel):
+    ui_id: str = Field(..., description="Unique identifier for the task")
+    client_id: str = Field(..., description="Client identifier that initiated the task")
+    kind: OperationType
+    params: Union[
+        InstallPackParams,
+        UpdatePackParams,
+        UpdateAllPacksParams,
+        UpdateComfyUIParams,
+        FixPackParams,
+        UninstallPackParams,
+        DisablePackParams,
+        EnablePackParams,
+        ModelMetadata,
+    ]
+
+
+class TaskHistoryItem(BaseModel):
+    ui_id: str = Field(..., description="Unique identifier for the task")
+    client_id: str = Field(..., description="Client identifier that initiated the task")
+    kind: str = Field(..., description="Type of task that was performed")
+    timestamp: datetime = Field(..., description="ISO timestamp when task completed")
+    result: str = Field(..., description="Task result message or details")
+    status: Optional[TaskExecutionStatus] = None
+    batch_id: Optional[str] = Field(
+        None, description="ID of the batch this task belongs to"
+    )
+    end_time: Optional[datetime] = Field(
+        None, description="ISO timestamp when task execution ended"
+    )
+
+
+class TaskStateMessage(BaseModel):
+    history: Dict[str, TaskHistoryItem] = Field(
+        ..., description="Map of task IDs to their history items"
+    )
+    running_queue: List[QueueTaskItem] = Field(
+        ..., description="Currently executing tasks"
+    )
+    pending_queue: List[QueueTaskItem] = Field(
+        ..., description="Tasks waiting to be executed"
+    )
+    installed_packs: Dict[str, ManagerPackInstalled] = Field(
+        ..., description="Map of currently installed node packages by name"
+    )
+
+
+class MessageTaskDone(BaseModel):
+    ui_id: str = Field(..., description="Task identifier")
+    result: str = Field(..., description="Task result message")
+    kind: str = Field(..., description="Type of task")
+    status: Optional[TaskExecutionStatus] = None
+    timestamp: datetime = Field(..., description="ISO timestamp when task completed")
+    state: TaskStateMessage
+
+
+class MessageTaskStarted(BaseModel):
+    ui_id: str = Field(..., description="Task identifier")
+    kind: str = Field(..., description="Type of task")
+    timestamp: datetime = Field(..., description="ISO timestamp when task started")
+    state: TaskStateMessage
+
+
+class MessageTaskFailed(BaseModel):
+    ui_id: str = Field(..., description="Task identifier")
+    error: str = Field(..., description="Error message")
+    kind: str = Field(..., description="Type of task")
+    timestamp: datetime = Field(..., description="ISO timestamp when task failed")
+    state: TaskStateMessage
+
+
+class MessageUpdate(
+    RootModel[Union[MessageTaskDone, MessageTaskStarted, MessageTaskFailed]]
+):
+    root: Union[MessageTaskDone, MessageTaskStarted, MessageTaskFailed] = Field(
+        ..., description="Union type for all possible WebSocket message updates"
+    )
+
+
+class HistoryResponse(BaseModel):
+    history: Optional[Dict[str, TaskHistoryItem]] = Field(
+        None, description="Map of task IDs to their history items"
+    )

comfyui_manager/glob/CLAUDE.md

@@ -0,0 +1,11 @@
+- Anytime you make a change to the data being sent or received, you should follow this process:
+  1. Adjust the openapi.yaml file first
+  2. Verify the syntax of the openapi.yaml file using `yaml.safe_load`
+  3. Regenerate the types following the instructions in the `data_models/README.md` file
+  4. Verify the new data model is generated
+  5. Verify the syntax of the generated types files
+  6. Run formatting and linting on the generated types files
+  7. Adjust the `__init__.py` files in the `data_models` directory to match/export the new data model
+  8. Only then, make the changes to the rest of the codebase
+  9. Run the CI tests to verify that the changes are working
+- The comfyui_manager is a python package that is used to manage the comfyui server. There are two sub-packages `glob` and `legacy`. These represent the current version (`glob`) and the previous version (`legacy`), not including common utilities and data models. When developing, we work in the `glob` package. You can ignore the `legacy` package entirely, unless you have a very good reason to research how things were done in the legacy or prior major versions of the package. But in those cases, you should just look for the sake of knowledge or reflection, not for changing code (unless explicitly asked to do so).

comfyui_manager/glob/constants.py

@@ -0,0 +1,54 @@
+
+SECURITY_MESSAGE_MIDDLE_OR_BELOW = "ERROR: To use this action, a security_level of `middle or below` is required. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
+SECURITY_MESSAGE_NORMAL_MINUS = "ERROR: To use this feature, you must either set '--listen' to a local IP and set the security level to 'normal-' or lower, or set the security level to 'middle' or 'weak'. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
+SECURITY_MESSAGE_GENERAL = "ERROR: This installation is not allowed in this security_level. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
+SECURITY_MESSAGE_NORMAL_MINUS_MODEL = "ERROR: Downloading models that are not in '.safetensors' format is only allowed for models registered in the 'default' channel at this security level. If you want to download this model, set the security level to 'normal-' or lower."
+
+
+def is_loopback(address):
+    import ipaddress
+
+    try:
+        return ipaddress.ip_address(address).is_loopback
+    except ValueError:
+        return False
+
+
+model_dir_name_map = {
+    "checkpoints": "checkpoints",
+    "checkpoint": "checkpoints",
+    "unclip": "checkpoints",
+    "text_encoders": "text_encoders",
+    "clip": "text_encoders",
+    "vae": "vae",
+    "lora": "loras",
+    "t2i-adapter": "controlnet",
+    "t2i-style": "controlnet",
+    "controlnet": "controlnet",
+    "clip_vision": "clip_vision",
+    "gligen": "gligen",
+    "upscale": "upscale_models",
+    "embedding": "embeddings",
+    "embeddings": "embeddings",
+    "unet": "diffusion_models",
+    "diffusion_model": "diffusion_models",
+}
+
+# List of all model directory names used for checking installed models
+MODEL_DIR_NAMES = [
+    "checkpoints",
+    "loras",
+    "vae",
+    "text_encoders",
+    "diffusion_models",
+    "clip_vision",
+    "embeddings",
+    "diffusers",
+    "vae_approx",
+    "controlnet",
+    "gligen",
+    "upscale_models",
+    "hypernetworks",
+    "photomaker",
+    "classifiers",
+]

comfyui_manager/glob/utils/environment_utils.py

@@ -0,0 +1,142 @@
+import os
+import git
+import logging
+import traceback
+
+from comfyui_manager.common import context
+import folder_paths
+from comfy.cli_args import args
+import latent_preview
+
+from comfyui_manager.glob import manager_core as core
+from comfyui_manager.common import cm_global
+
+
+comfy_ui_hash = "-"
+comfyui_tag = None
+
+
+def print_comfyui_version():
+    global comfy_ui_hash
+    global comfyui_tag
+
+    is_detached = False
+    try:
+        repo = git.Repo(os.path.dirname(folder_paths.__file__))
+        core.comfy_ui_revision = len(list(repo.iter_commits("HEAD")))
+
+        comfy_ui_hash = repo.head.commit.hexsha
+        cm_global.variables["comfyui.revision"] = core.comfy_ui_revision
+
+        core.comfy_ui_commit_datetime = repo.head.commit.committed_datetime
+        cm_global.variables["comfyui.commit_datetime"] = core.comfy_ui_commit_datetime
+
+        is_detached = repo.head.is_detached
+        current_branch = repo.active_branch.name
+
+        comfyui_tag = context.get_comfyui_tag()
+
+        try:
+            if (
+                not os.environ.get("__COMFYUI_DESKTOP_VERSION__")
+                and core.comfy_ui_commit_datetime.date()
+                < core.comfy_ui_required_commit_datetime.date()
+            ):
+                logging.warning(
+                    f"\n\n## [WARN] ComfyUI-Manager: Your ComfyUI version ({core.comfy_ui_revision})[{core.comfy_ui_commit_datetime.date()}] is too old. Please update to the latest version. ##\n\n"
+                )
+        except Exception:
+            pass
+
+        # process on_revision_detected -->
+        if "cm.on_revision_detected_handler" in cm_global.variables:
+            for k, f in cm_global.variables["cm.on_revision_detected_handler"]:
+                try:
+                    f(core.comfy_ui_revision)
+                except Exception:
+                    logging.error(f"[ERROR] '{k}' on_revision_detected_handler")
+                    traceback.print_exc()
+
+            del cm_global.variables["cm.on_revision_detected_handler"]
+        else:
+            logging.warning(
+                "[ComfyUI-Manager] Some features are restricted due to your ComfyUI being outdated."
+            )
+        # <--
+
+        if current_branch == "master":
+            if comfyui_tag:
+                logging.info(
+                    f"### ComfyUI Version: {comfyui_tag} | Released on '{core.comfy_ui_commit_datetime.date()}'"
+                )
+            else:
+                logging.info(
+                    f"### ComfyUI Revision: {core.comfy_ui_revision} [{comfy_ui_hash[:8]}] | Released on '{core.comfy_ui_commit_datetime.date()}'"
+                )
+        else:
+            if comfyui_tag:
+                logging.info(
+                    f"### ComfyUI Version: {comfyui_tag} on '{current_branch}' | Released on '{core.comfy_ui_commit_datetime.date()}'"
+                )
+            else:
+                logging.info(
+                    f"### ComfyUI Revision: {core.comfy_ui_revision} on '{current_branch}' [{comfy_ui_hash[:8]}] | Released on '{core.comfy_ui_commit_datetime.date()}'"
+                )
+    except Exception:
+        if is_detached:
+            logging.info(
+                f"### ComfyUI Revision: {core.comfy_ui_revision} [{comfy_ui_hash[:8]}] *DETACHED | Released on '{core.comfy_ui_commit_datetime.date()}'"
+            )
+        else:
+            logging.info(
+                "### ComfyUI Revision: UNKNOWN (The currently installed ComfyUI is not a Git repository)"
+            )
+
+
+def set_preview_method(method):
+    if method == "auto":
+        args.preview_method = latent_preview.LatentPreviewMethod.Auto
+    elif method == "latent2rgb":
+        args.preview_method = latent_preview.LatentPreviewMethod.Latent2RGB
+    elif method == "taesd":
+        args.preview_method = latent_preview.LatentPreviewMethod.TAESD
+    else:
+        args.preview_method = latent_preview.LatentPreviewMethod.NoPreviews
+
+    core.get_config()["preview_method"] = method
+
+
+def set_update_policy(mode):
+    core.get_config()["update_policy"] = mode
+
+
+def set_db_mode(mode):
+    core.get_config()["db_mode"] = mode
+
+
+def setup_environment():
+    git_exe = core.get_config()["git_exe"]
+
+    if git_exe != "":
+        git.Git().update_environment(GIT_PYTHON_GIT_EXECUTABLE=git_exe)
+
+
+def initialize_environment():
+    context.comfy_path = os.path.dirname(folder_paths.__file__)
+    core.js_path = os.path.join(context.comfy_path, "web", "extensions")
+
+    # Legacy database paths - kept for potential future use
+    # local_db_model = os.path.join(manager_util.comfyui_manager_path, "model-list.json")
+    # local_db_alter = os.path.join(manager_util.comfyui_manager_path, "alter-list.json")
+    # local_db_custom_node_list = os.path.join(
+    #     manager_util.comfyui_manager_path, "custom-node-list.json"
+    # )
+    # local_db_extension_node_mappings = os.path.join(
+    #     manager_util.comfyui_manager_path, "extension-node-map.json"
+    # )
+
+    set_preview_method(core.get_config()["preview_method"])
+    print_comfyui_version()
+    setup_environment()
+
+    core.check_invalid_nodes()

comfyui_manager/glob/utils/formatting_utils.py

@@ -0,0 +1,60 @@
+import locale
+import sys
+import re
+
+
+def handle_stream(stream, prefix):
+    stream.reconfigure(encoding=locale.getpreferredencoding(), errors="replace")
+    for msg in stream:
+        if (
+            prefix == "[!]"
+            and ("it/s]" in msg or "s/it]" in msg)
+            and ("%|" in msg or "it [" in msg)
+        ):
+            if msg.startswith("100%"):
+                print("\r" + msg, end="", file=sys.stderr),
+            else:
+                print("\r" + msg[:-1], end="", file=sys.stderr),
+        else:
+            if prefix == "[!]":
+                print(prefix, msg, end="", file=sys.stderr)
+            else:
+                print(prefix, msg, end="")
+
+
+def convert_markdown_to_html(input_text):
+    pattern_a = re.compile(r"\[a/([^]]+)]\(([^)]+)\)")
+    pattern_w = re.compile(r"\[w/([^]]+)]")
+    pattern_i = re.compile(r"\[i/([^]]+)]")
+    pattern_bold = re.compile(r"\*\*([^*]+)\*\*")
+    pattern_white = re.compile(r"%%([^*]+)%%")
+
+    def replace_a(match):
+        return f"<a href='{match.group(2)}' target='blank'>{match.group(1)}</a>"
+
+    def replace_w(match):
+        return f"<p class='cm-warn-note'>{match.group(1)}</p>"
+
+    def replace_i(match):
+        return f"<p class='cm-info-note'>{match.group(1)}</p>"
+
+    def replace_bold(match):
+        return f"<B>{match.group(1)}</B>"
+
+    def replace_white(match):
+        return f"<font color='white'>{match.group(1)}</font>"
+
+    input_text = (
+        input_text.replace("\\[", "&#91;")
+        .replace("\\]", "&#93;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+    )
+
+    result_text = re.sub(pattern_a, replace_a, input_text)
+    result_text = re.sub(pattern_w, replace_w, result_text)
+    result_text = re.sub(pattern_i, replace_i, result_text)
+    result_text = re.sub(pattern_bold, replace_bold, result_text)
+    result_text = re.sub(pattern_white, replace_white, result_text)
+
+    return result_text.replace("\n", "<BR>")

comfyui_manager/glob/utils/model_utils.py

@@ -0,0 +1,161 @@
+import os
+import logging
+import concurrent.futures
+import folder_paths
+
+from comfyui_manager.glob import manager_core as core
+from comfyui_manager.glob.constants import model_dir_name_map, MODEL_DIR_NAMES
+
+
+def get_model_dir(data, show_log=False):
+    if "download_model_base" in folder_paths.folder_names_and_paths:
+        models_base = folder_paths.folder_names_and_paths["download_model_base"][0][0]
+    else:
+        models_base = folder_paths.models_dir
+
+    # NOTE: Validate to prevent path traversal.
+    if any(char in data["filename"] for char in {"/", "\\", ":"}):
+        return None
+
+    def resolve_custom_node(save_path):
+        save_path = save_path[13:]  # remove 'custom_nodes/'
+
+        # NOTE: Validate to prevent path traversal.
+        if save_path.startswith(os.path.sep) or ":" in save_path:
+            return None
+
+        repo_name = save_path.replace("\\", "/").split("/")[
+            0
+        ]  # get custom node repo name
+
+        # NOTE: The creation of files within the custom node path should be removed in the future.
+        repo_path = core.lookup_installed_custom_nodes_legacy(repo_name)
+        if repo_path is not None and repo_path[0]:
+            # Returns the retargeted path based on the actually installed repository
+            return os.path.join(os.path.dirname(repo_path[1]), save_path)
+        else:
+            return None
+
+    if data["save_path"] != "default":
+        if ".." in data["save_path"] or data["save_path"].startswith("/"):
+            if show_log:
+                logging.info(
+                    f"[WARN] '{data['save_path']}' is not allowed path. So it will be saved into 'models/etc'."
+                )
+            base_model = os.path.join(models_base, "etc")
+        else:
+            if data["save_path"].startswith("custom_nodes"):
+                base_model = resolve_custom_node(data["save_path"])
+                if base_model is None:
+                    if show_log:
+                        logging.info(
+                            f"[ComfyUI-Manager] The target custom node for model download is not installed: {data['save_path']}"
+                        )
+                    return None
+            else:
+                base_model = os.path.join(models_base, data["save_path"])
+    else:
+        model_dir_name = model_dir_name_map.get(data["type"].lower())
+        if model_dir_name is not None:
+            base_model = folder_paths.folder_names_and_paths[model_dir_name][0][0]
+        else:
+            base_model = os.path.join(models_base, "etc")
+
+    return base_model
+
+
+def get_model_path(data, show_log=False):
+    base_model = get_model_dir(data, show_log)
+    if base_model is None:
+        return None
+    else:
+        if data["filename"] == "<huggingface>":
+            return os.path.join(base_model, os.path.basename(data["url"]))
+        else:
+            return os.path.join(base_model, data["filename"])
+
+
+def check_model_installed(json_obj):
+    def is_exists(model_dir_name, filename, url):
+        if filename == "<huggingface>":
+            filename = os.path.basename(url)
+
+        dirs = folder_paths.get_folder_paths(model_dir_name)
+
+        for x in dirs:
+            if os.path.exists(os.path.join(x, filename)):
+                return True
+
+        return False
+
+    total_models_files = set()
+    for x in MODEL_DIR_NAMES:
+        for y in folder_paths.get_filename_list(x):
+            total_models_files.add(y)
+
+    def process_model_phase(item):
+        if (
+            "diffusion" not in item["filename"]
+            and "pytorch" not in item["filename"]
+            and "model" not in item["filename"]
+        ):
+            # non-general name case
+            if item["filename"] in total_models_files:
+                item["installed"] = "True"
+                return
+
+        if item["save_path"] == "default":
+            model_dir_name = model_dir_name_map.get(item["type"].lower())
+            if model_dir_name is not None:
+                item["installed"] = str(
+                    is_exists(model_dir_name, item["filename"], item["url"])
+                )
+            else:
+                item["installed"] = "False"
+        else:
+            model_dir_name = item["save_path"].split("/")[0]
+            if model_dir_name in folder_paths.folder_names_and_paths:
+                if is_exists(model_dir_name, item["filename"], item["url"]):
+                    item["installed"] = "True"
+
+            if "installed" not in item:
+                if item["filename"] == "<huggingface>":
+                    filename = os.path.basename(item["url"])
+                else:
+                    filename = item["filename"]
+
+                fullpath = os.path.join(
+                    folder_paths.models_dir, item["save_path"], filename
+                )
+
+                item["installed"] = "True" if os.path.exists(fullpath) else "False"
+
+    with concurrent.futures.ThreadPoolExecutor(8) as executor:
+        for item in json_obj["models"]:
+            executor.submit(process_model_phase, item)
+
+
+async def check_whitelist_for_model(item):
+    from comfyui_manager.data_models import ManagerDatabaseSource
+    
+    json_obj = await core.get_data_by_mode(ManagerDatabaseSource.cache.value, "model-list.json")
+
+    for x in json_obj.get("models", []):
+        if (
+            x["save_path"] == item["save_path"]
+            and x["base"] == item["base"]
+            and x["filename"] == item["filename"]
+        ):
+            return True
+
+    json_obj = await core.get_data_by_mode(ManagerDatabaseSource.local.value, "model-list.json")
+
+    for x in json_obj.get("models", []):
+        if (
+            x["save_path"] == item["save_path"]
+            and x["base"] == item["base"]
+            and x["filename"] == item["filename"]
+        ):
+            return True
+
+    return False

comfyui_manager/glob/utils/node_pack_utils.py

@@ -0,0 +1,65 @@
+import concurrent.futures
+
+from comfyui_manager.glob import manager_core as core
+
+
+def check_state_of_git_node_pack(
+    node_packs, do_fetch=False, do_update_check=True, do_update=False
+):
+    if do_fetch:
+        print("Start fetching...", end="")
+    elif do_update:
+        print("Start updating...", end="")
+    elif do_update_check:
+        print("Start update check...", end="")
+
+    def process_custom_node(item):
+        core.check_state_of_git_node_pack_single(
+            item, do_fetch, do_update_check, do_update
+        )
+
+    with concurrent.futures.ThreadPoolExecutor(4) as executor:
+        for k, v in node_packs.items():
+            if v.get("active_version") in ["unknown", "nightly"]:
+                executor.submit(process_custom_node, v)
+
+    if do_fetch:
+        print("\x1b[2K\rFetching done.")
+    elif do_update:
+        update_exists = any(
+            item.get("updatable", False) for item in node_packs.values()
+        )
+        if update_exists:
+            print("\x1b[2K\rUpdate done.")
+        else:
+            print("\x1b[2K\rAll extensions are already up-to-date.")
+    elif do_update_check:
+        print("\x1b[2K\rUpdate check done.")
+
+
+def nickname_filter(json_obj):
+    preemptions_map = {}
+
+    for k, x in json_obj.items():
+        if "preemptions" in x[1]:
+            for y in x[1]["preemptions"]:
+                preemptions_map[y] = k
+        elif k.endswith("/ComfyUI"):
+            for y in x[0]:
+                preemptions_map[y] = k
+
+    updates = {}
+    for k, x in json_obj.items():
+        removes = set()
+        for y in x[0]:
+            k2 = preemptions_map.get(y)
+            if k2 is not None and k != k2:
+                removes.add(y)
+
+        if len(removes) > 0:
+            updates[k] = [y for y in x[0] if y not in removes]
+
+    for k, v in updates.items():
+        json_obj[k][0] = v
+
+    return json_obj

comfyui_manager/glob/utils/security_utils.py

@@ -0,0 +1,54 @@
+from comfyui_manager.glob import manager_core as core
+from comfy.cli_args import args
+from comfyui_manager.data_models import SecurityLevel, RiskLevel, ManagerDatabaseSource
+
+
+def is_loopback(address):
+    import ipaddress
+    try:
+        return ipaddress.ip_address(address).is_loopback
+    except ValueError:
+        return False
+
+
+def is_allowed_security_level(level):
+    is_local_mode = is_loopback(args.listen)
+    
+    if level == RiskLevel.block.value:
+        return False
+    elif level == RiskLevel.high.value:
+        if is_local_mode:
+            return core.get_config()["security_level"] in [SecurityLevel.weak.value, SecurityLevel.normal_.value]
+        else:
+            return core.get_config()["security_level"] == SecurityLevel.weak.value
+    elif level == RiskLevel.middle.value:
+        return core.get_config()["security_level"] in [SecurityLevel.weak.value, SecurityLevel.normal.value, SecurityLevel.normal_.value]
+    else:
+        return True
+
+
+async def get_risky_level(files, pip_packages):
+    json_data1 = await core.get_data_by_mode(ManagerDatabaseSource.local.value, "custom-node-list.json")
+    json_data2 = await core.get_data_by_mode(
+        ManagerDatabaseSource.cache.value,
+        "custom-node-list.json",
+        channel_url="https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main",
+    )
+
+    all_urls = set()
+    for x in json_data1["custom_nodes"] + json_data2["custom_nodes"]:
+        all_urls.update(x.get("files", []))
+
+    for x in files:
+        if x not in all_urls:
+            return RiskLevel.high.value
+
+    all_pip_packages = set()
+    for x in json_data1["custom_nodes"] + json_data2["custom_nodes"]:
+        all_pip_packages.update(x.get("pip", []))
+
+    for p in pip_packages:
+        if p not in all_pip_packages:
+            return RiskLevel.block.value
+
+    return RiskLevel.middle.value