diff --git a/.github/ISSUE_TEMPLATE/200-installation.yml b/.github/ISSUE_TEMPLATE/200-installation.yml index fe45a2599..831129018 100644 --- a/.github/ISSUE_TEMPLATE/200-installation.yml +++ b/.github/ISSUE_TEMPLATE/200-installation.yml @@ -7,14 +7,14 @@ body: - type: markdown attributes: value: > - #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). + #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). - type: textarea attributes: label: Your current environment description: | Please run the following and paste the output below. ```sh - wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py + wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py # For security purposes, please feel free to check the contents of collect_env.py before running it. python collect_env.py ``` diff --git a/.github/ISSUE_TEMPLATE/400-bug-report.yml b/.github/ISSUE_TEMPLATE/400-bug-report.yml index 723d2aad8..e29741564 100644 --- a/.github/ISSUE_TEMPLATE/400-bug-report.yml +++ b/.github/ISSUE_TEMPLATE/400-bug-report.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: > - #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). + #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). - type: markdown attributes: value: | @@ -24,7 +24,7 @@ body: description: | Please run the following and paste the output below. ```sh - wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py + wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py # For security purposes, please feel free to check the contents of collect_env.py before running it. python collect_env.py ``` diff --git a/.github/ISSUE_TEMPLATE/500-feature-request.yml b/.github/ISSUE_TEMPLATE/500-feature-request.yml index 0e6b1bc14..98dec8276 100644 --- a/.github/ISSUE_TEMPLATE/500-feature-request.yml +++ b/.github/ISSUE_TEMPLATE/500-feature-request.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: > - #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). + #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). - type: textarea attributes: label: 🚀 The feature, motivation and pitch diff --git a/.github/ISSUE_TEMPLATE/600-new-model.yml b/.github/ISSUE_TEMPLATE/600-new-model.yml index 7cc236196..cf621ba1d 100644 --- a/.github/ISSUE_TEMPLATE/600-new-model.yml +++ b/.github/ISSUE_TEMPLATE/600-new-model.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: > - #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). + #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). #### We also highly recommend you read the contributing guidelines first to understand how to add a new model. - type: textarea diff --git a/.github/ISSUE_TEMPLATE/700-performance-discussion.yml b/.github/ISSUE_TEMPLATE/700-performance-discussion.yml index 98695d75d..7a4f374b8 100644 --- a/.github/ISSUE_TEMPLATE/700-performance-discussion.yml +++ b/.github/ISSUE_TEMPLATE/700-performance-discussion.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: > - #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). + #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+). - type: textarea attributes: label: Proposal to improve performance @@ -19,7 +19,7 @@ body: attributes: label: Report of performance regression description: > - Please provide detailed description of performance comparison to confirm the regression. You may want to run the benchmark script at https://github.com/hsliuustc0106/vllm-omni/tree/main/tests/benchmarks . + Please provide detailed description of performance comparison to confirm the regression. You may want to run the benchmark script at https://github.com/vllm-project/vllm-omni/tree/main/tests/benchmarks . validations: required: false - type: textarea @@ -35,7 +35,7 @@ body: description: | Please run the following and paste the output below. ```sh - wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py + wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py # For security purposes, please feel free to check the contents of collect_env.py before running it. python collect_env.py ``` diff --git a/.github/ISSUE_TEMPLATE/750-RFC.yml b/.github/ISSUE_TEMPLATE/750-RFC.yml index fafaafd80..405e3b890 100644 --- a/.github/ISSUE_TEMPLATE/750-RFC.yml +++ b/.github/ISSUE_TEMPLATE/750-RFC.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: > - #### Please take a look at previous [RFCs](https://github.com/hsliuustc0106/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference. + #### Please take a look at previous [RFCs](https://github.com/vllm-project/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference. - type: textarea attributes: label: Motivation. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index e6d459095..ff0770175 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Questions - url: https://github.com/hsliuustc0106/vllm-omni/discussions + url: https://github.com/vllm-project/vllm-omni/discussions about: Ask questions and discuss with other vLLM-omni community members diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cc63a80c6..ea9b94c93 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,4 +18,4 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT - [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft. -**BEFORE SUBMITTING, PLEASE READ ** (anything written below this line will be removed by GitHub Actions) +**BEFORE SUBMITTING, PLEASE READ ** (anything written below this line will be removed by GitHub Actions) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..b79b37033 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,51 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + - master + paths: + - 'docs/**' + - 'mkdocs.yml' + - 'vllm_omni/**' + - '.github/workflows/docs.yml' + workflow_dispatch: + +permissions: + contents: write + pages: write + id-token: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + pip install -e ".[docs]" + + - name: Build documentation + run: mkdocs build + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./site + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..5a06c6634 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,24 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + jobs: + post_checkout: + - git fetch --unshallow || true + +mkdocs: + configuration: mkdocs.yml + fail_on_warning: true + +# Optionally declare the Python requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/README.md b/README.md index 36a98c456..83c769f34 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Install additional requirements for vllm-omni ```bash git clone https://github.com/vllm-project/vllm-omni.git cd vllm_omni -uv pip install -r requirements.txt +uv pip install -e . ``` ## Run examples (Qwen2.5-omni) diff --git a/docs/DOCS_GUIDE.md b/docs/DOCS_GUIDE.md new file mode 100644 index 000000000..d52ad1c01 --- /dev/null +++ b/docs/DOCS_GUIDE.md @@ -0,0 +1,139 @@ +# Documentation Build Guide + +This directory contains the source files for the vLLM-omni documentation. + +## Building Documentation Locally + +### Prerequisites + +Install documentation dependencies: + +```bash +pip install -e ".[docs]" +``` + +### Build and Serve Documentation + +From the project root: + +```bash +# Serve documentation locally (auto-reload on changes) +# This starts a local web server at http://127.0.0.1:8000 +mkdocs serve + +# Build static site (generates HTML files in site/ directory) +mkdocs build +``` + +When using `mkdocs serve`, the documentation will be automatically available at `http://127.0.0.1:8000`. The server will automatically reload when you make changes to the documentation files. + +## Auto-generating API Documentation + +The documentation automatically extracts docstrings from the code using mkdocstrings. To ensure your code is documented: + +1. Add docstrings to all public classes, functions, and methods +2. Use Google or NumPy style docstrings (both are supported) +3. Rebuild the documentation to see changes + +Example docstring: + +```python +class OmniLLM: + """Main entry point for vLLM-omni inference. + + This class provides a high-level interface for running multi-modal + inference with non-autoregressive models. + + Args: + model: Model name or path + stage_configs: Optional stage configurations + **kwargs: Additional arguments passed to the engine + + Example: + >>> llm = OmniLLM(model="Qwen/Qwen2.5-Omni") + >>> outputs = llm.generate(prompts="Hello") + """ +``` + +## Documentation Structure + +``` +docs/ +├── index.md # Main documentation page +├── getting_started/ # Getting started guides +├── architecture/ # Architecture documentation +├── api/ # API reference (auto-generated from code) +├── examples/ # Code examples +└── stylesheets/ # Custom CSS +``` + +## Publishing Documentation + +### GitHub Pages (Recommended) + +The documentation is automatically deployed to GitHub Pages using GitHub Actions. + +1. **Enable GitHub Pages**: + - Go to repository `Settings` → `Pages` + - Set `Source` to `GitHub Actions` + - Save settings + +2. **Push changes**: + ```bash + git push origin main + ``` + +3. **Documentation will be available at**: + - `https://vllm-project.github.io/vllm-omni/` + +The GitHub Actions workflow (`.github/workflows/docs.yml`) will automatically: +- Build the documentation when you push to `main` branch +- Deploy it to GitHub Pages +- Update the documentation whenever you make changes + + +### Read the Docs (Alternative) + +You can also use Read the Docs for hosting: + +1. Sign up at https://readthedocs.org/ +2. Import the `vllm-project/vllm-omni` repository +3. Read the Docs will automatically build using `.readthedocs.yml` +4. Documentation will be available at: `https://vllm-omni.readthedocs.io/` + +## Configuration + +The documentation configuration is in `mkdocs.yml` at the project root. + +## Tips + +- **API Documentation**: API docs are automatically generated using `mkdocs-api-autonav` and `mkdocstrings` + - No need to manually create API pages - they're generated automatically + - Use `[module.name.ClassName][]` syntax for cross-references in Summary pages +- **Code Snippets**: Use `--8<-- "path/to/file.py"` for including code snippets +- **Markdown**: Use Markdown for all documentation (no need for RST) +- **Material Theme**: Use Material theme features like: + - Admonitions: `!!! note`, `!!! warning`, etc. + - Code blocks with syntax highlighting + - Tabs for organizing content + - Math formulas using `pymdownx.arithmatex` + +## Troubleshooting + +### Documentation not updating + +- Make sure you've saved all files +- If using `mkdocs serve`, it should auto-reload +- Check for syntax errors in `mkdocs.yml` + +### API links not working + +- Ensure class names match exactly (case-sensitive) +- Check that the module is imported correctly +- Run `mkdocs build --strict` to check for errors + +### Build errors + +- Check Python version (requires 3.9+) +- Ensure all dependencies are installed: `pip install -e ".[docs]"` +- Check `mkdocs.yml` syntax with `mkdocs build --strict` diff --git a/docs/api/overview.md b/docs/api/overview.md new file mode 100644 index 000000000..8e386044b --- /dev/null +++ b/docs/api/overview.md @@ -0,0 +1,69 @@ +# Summary + +## Entry Points + +Main entry points for vLLM-omni inference and serving. + +- [vllm_omni.entrypoints.omni_llm.OmniLLM][] +- [vllm_omni.entrypoints.omni_llm.OmniStageLLM][] +- [vllm_omni.entrypoints.omni_stage.OmniStage][] +- [vllm_omni.entrypoints.cli.serve.OmniServeCommand][] + +## Inputs + +Input data structures for multi-modal inputs. + +- [vllm_omni.inputs.data.OmniTokenInputs][] +- [vllm_omni.inputs.data.OmniTokensPrompt][] +- [vllm_omni.inputs.data.OmniEmbedsPrompt][] +- [vllm_omni.inputs.parse.parse_singleton_prompt_omni][] +- [vllm_omni.inputs.preprocess.OmniInputPreprocessor][] + +## Engine + +Engine classes for offline and online inference. + +- [vllm_omni.engine.arg_utils.OmniEngineArgs][] +- [vllm_omni.engine.processor.OmniProcessor][] +- [vllm_omni.engine.output_processor.OmniRequestState][] +- [vllm_omni.engine.output_processor.MultimodalOutputProcessor][] +- [vllm_omni.engine.PromptEmbedsPayload][] +- [vllm_omni.engine.AdditionalInformationPayload][] +- [vllm_omni.engine.OmniEngineCoreRequest][] + +## Core + +Core scheduling and caching components. + +- [vllm_omni.core.sched.scheduler.OmniScheduler][] +- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][] +- [vllm_omni.core.sched.output.OmniNewRequestData][] +- [vllm_omni.core.dit_cache_manager.DiTCacheManager][] + +## Model Executor + +Model execution components. + +- [vllm_omni.model_executor.models.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][] +- [vllm_omni.model_executor.models.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][] +- [vllm_omni.model_executor.models.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][] +- [vllm_omni.model_executor.models.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][] +- [vllm_omni.model_executor.layers.mrope.MRotaryEmbedding][] +- [vllm_omni.model_executor.models.registry.OmniModelRegistry][] + +## Configuration + +Configuration classes. + +- [vllm_omni.config.OmniModelConfig][] + +## Workers + +Worker classes and model runners for distributed inference. + +- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][] +- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][] +- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][] +- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][] +- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][] + diff --git a/docs/contributing/README.md b/docs/contributing/README.md new file mode 100644 index 000000000..43bccdc5b --- /dev/null +++ b/docs/contributing/README.md @@ -0,0 +1,170 @@ +# Contributing to vLLM-omni + +Thank you for your interest in contributing to vLLM-omni! This document provides guidelines and instructions for contributing. + +## Getting Started + +### Development Environment + +vLLM-omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands: + +```bash +uv venv --python 3.12 --seed +source .venv/bin/activate +``` + +Install vLLM from source: + +```bash +git clone https://github.com/vllm-project/vllm.git +cd vllm +git checkout v0.11.0 +VLLM_USE_PRECOMPILED=1 uv pip install -e . +``` + +Install vLLM-omni from source: + +```bash +git clone https://github.com/vllm-project/vllm-omni.git +cd vllm_omni +uv pip install -e . +``` + +!!! tip + vLLM-omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment. + +### Linting + +vLLM-omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as: + +```bash +uv pip install pre-commit +pre-commit install +``` + +vLLM-omni's `pre-commit` hooks will now run automatically every time you commit. + +!!! tip + You can manually run the `pre-commit` hooks using: + + ```bash + pre-commit run # runs on staged files + pre-commit run -a # runs on all files (short for --all-files) + ``` + +### Documentation + +MkDocs is a fast, simple and downright gorgeous static site generator that's geared towards building project documentation. Documentation source files are written in Markdown, and configured with a single YAML configuration file, `mkdocs.yml`. + +Get started with: + +```bash +uv pip install -e ".[docs]" +``` + +MkDocs comes with a built-in dev-server that lets you preview your documentation as you work on it. From the root of the repository, run: + +```bash +mkdocs serve # with API ref (~10 minutes) +API_AUTONAV_EXCLUDE=vllm_omni mkdocs serve # API ref off (~15 seconds) +``` + +Once you see `Serving on http://127.0.0.1:8000/` in the logs, the live preview is ready! Open in your browser to see it. + +For additional features and advanced configurations, refer to the: + +- [MkDocs documentation](https://www.mkdocs.org/) +- [Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/) (the MkDocs theme we use) + +### Testing + +vLLM-omni uses `pytest` to test the codebase. + +```bash +# Install the test dependencies used in CI (CUDA only) +uv pip install -e ".[dev]" --torch-backend=auto + +# Install some common test dependencies (hardware agnostic) +uv pip install pytest pytest-asyncio + +# Run all tests +pytest tests/ + +# Run tests for a single test file with detailed output +pytest -s -v tests/test_logger.py +``` + +!!! warning + Currently, not all unit tests pass when run on CPU platforms. If you don't have access to a GPU platform to run unit tests locally, rely on the continuous integration system to run the tests for now. + +## Issues + +If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible. + +!!! important + If you discover a security vulnerability, please report it by creating a GitHub issue with the `security` label. + +## Pull Requests & Code Reviews + +Thank you for your contribution to vLLM-omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-omni maintain the code quality and improve the efficiency of the review process. + +### DCO and Signed-off-by + +When contributing changes to this project, you must agree to the [DCO](https://developercertificate.org/). Commits must include a `Signed-off-by:` header which certifies agreement with the terms of the DCO. + +Using `-s` with `git commit` will automatically add this header. + +!!! tip + You can enable automatic sign-off via your IDE: + + - **PyCharm**: Click on the `Show Commit Options` icon to the right of the `Commit and Push...` button in the `Commit` window. It will bring up a `git` window where you can modify the `Author` and enable `Sign-off commit`. + - **VSCode**: Open the Settings editor and enable the `Git: Always Sign Off` (`git.alwaysSignOff`) field. + +### PR Title and Classification + +Only specific types of PRs will be reviewed. The PR title is prefixed appropriately to indicate the type of change. Please use one of the following: + +- `[Bugfix]` for bug fixes. +- `[CI/Build]` for build or continuous integration improvements. +- `[Doc]` for documentation fixes and improvements. +- `[Model]` for adding a new model or improving an existing model. Model name should appear in the title. +- `[Frontend]` For changes on the vLLM-omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.) +- `[Kernel]` for changes affecting CUDA kernels or other compute kernels. +- `[Core]` for changes in the core vLLM-omni logic (e.g., `OmniProcessor`, `OmniScheduler`, etc.) +- `[Hardware][Vendor]` for hardware-specific changes. Vendor name should appear in the prefix, such as [Ascend] for Ascend NPUs. +- `[Misc]` for PRs that do not fit the above categories. Please use this sparingly. + +!!! note + If the PR spans more than one category, please include all relevant prefixes. + +### Code Quality + +The PR needs to meet the following code quality standards: + +- We adhere to Google Python style guide and Google C++ style guide. +- Pass all linter checks. +- The code needs to be well-documented to ensure future contributors can easily understand the code. +- Include sufficient tests to ensure the project stays correct and robust. This includes both unit tests and integration tests. +- Please add documentation to `docs/` if the PR modifies the user-facing behaviors of vLLM-omni. It helps vLLM-omni users understand and utilize the new features or changes. + +### Notes for Large Changes + +Please keep the changes as concise as possible. For major architectural changes (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue (RFC) discussing the technical design and justification. Otherwise, we will tag it with `rfc-required` and might not go through the PR. + +### What to Expect for the Reviews + +The goal of the vLLM-omni team is to be a _transparent reviewing machine_. We would like to make the review process transparent and efficient and make sure no contributor feels confused or frustrated. However, the vLLM-omni team is small, so we need to prioritize some PRs over others. Here is what you can expect from the review process: + +- After the PR is submitted, the PR will be assigned to a reviewer. Every reviewer will pick up the PRs based on their expertise and availability. +- After the PR is assigned, the reviewer will provide status updates every 2-3 days. If the PR is not reviewed within 7 days, please feel free to ping the reviewer or the vLLM-omni team. +- After the review, the reviewer will put an `action-required` label on the PR if there are changes required. The contributor should address the comments and ping the reviewer to re-review the PR. +- Please respond to all comments within a reasonable time frame. If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion. +- Note that not all CI checks will be executed due to limited computational resources. The reviewer will add `ready` label to the PR when the PR is ready to merge or a full CI run is needed. + +## Additional Resources + +- [Design Documents](design_documents/index.md) - Architecture and design documentation + +## Thank You + +Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM-omni. All of your contributions help make vLLM-omni a great tool and community for everyone! diff --git a/docs/api/README.md b/docs/contributing/design_documents/api_design_doc.md similarity index 50% rename from docs/api/README.md rename to docs/contributing/design_documents/api_design_doc.md index 58490a80e..4ca442645 100644 --- a/docs/api/README.md +++ b/docs/contributing/design_documents/api_design_doc.md @@ -1,13 +1,10 @@ -# vLLM-omni API Documentation +# API Design Documentation -This directory contains comprehensive API design documentation for all core modules in vLLM-omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules. +This document contains comprehensive API design documentation for all core modules in vLLM-omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules. -## 📋 API Design Templates +## 📋 Module API -### 1. [API Design Template](API_DESIGN_TEMPLATE.md) -**Master template** for creating API documentation for any module in vLLM-omni. Use this as a starting point for new modules. - -### 2. [Core Module API](core_module_api.md) +### Core Module API **Core module** provides fundamental scheduling, caching, and resource management functionality. **Key Components:** @@ -16,7 +13,7 @@ This directory contains comprehensive API design documentation for all core modu - Resource allocation and coordination - Inter-module communication -### 3. [Engine Module API](engine_module_api.md) +### Engine Module API **Engine module** handles model loading, inference execution, and output processing. **Key Components:** @@ -25,7 +22,7 @@ This directory contains comprehensive API design documentation for all core modu - Input preprocessing for various modalities - Output postprocessing and formatting -### 4. [Executor Module API](executor_module_api.md) +### Executor Module API **Executor module** coordinates and manages request execution across different engines and workers. **Key Components:** @@ -34,7 +31,7 @@ This directory contains comprehensive API design documentation for all core modu - Worker management and task distribution - Error handling and recovery -### 5. [Worker Module API](worker_module_api.md) +### Worker Module API **Worker module** provides the actual execution environment for model inference. **Key Components:** @@ -43,79 +40,6 @@ This directory contains comprehensive API design documentation for all core modu - Request batching and processing - Performance optimization -## 🎯 How to Use These Templates - -### For New Module Development - -1. **Start with the Master Template** - - Copy `API_DESIGN_TEMPLATE.md` - - Rename to `[module_name]_api.md` - - Follow the structure exactly - -2. **Fill in Module-Specific Details** - - Update the module overview - - Define core classes and interfaces - - Specify public API methods - - Add configuration options - - Define error handling strategy - - Provide usage examples - -3. **Review and Validate** - - Ensure all sections are complete - - Verify code examples work - - Check for consistency with other modules - - Validate configuration options - -### For Existing Module Updates - -1. **Update API Documentation** - - Modify existing API files - - Add new methods and classes - - Update examples and configuration - - Maintain backward compatibility notes - -2. **Version Control** - - Track changes in git - - Use clear commit messages - - Tag major API changes - -## 📚 Template Structure - -Each API template follows this standardized structure: - -### 1. Module Overview -- **Purpose**: What the module does -- **Responsibilities**: Key responsibilities -- **Dependencies**: Required modules -- **Integration Points**: How it connects with other modules - -### 2. Core Classes/Interfaces -- **Base Classes**: Abstract base classes -- **Implementation Classes**: Concrete implementations -- **Data Structures**: Key data models - -### 3. Public API Methods -- **Initialization**: Constructor and setup -- **Core Operations**: Main functionality -- **Configuration**: Settings management -- **Lifecycle Management**: Start/stop/cleanup -- **Monitoring**: Status and metrics - -### 4. Configuration -- **Configuration Classes**: Dataclasses for settings -- **Required Parameters**: Must-have settings -- **Optional Parameters**: Optional settings with defaults -- **Validation**: Parameter validation rules - -### 5. Error Handling -- **Custom Exceptions**: Module-specific errors -- **Error Codes**: Standardized error codes -- **Recovery Strategies**: Error recovery approaches - -### 6. Examples -- **Basic Usage**: Simple usage examples -- **Advanced Usage**: Complex scenarios -- **Integration Examples**: Multi-module usage ## 🔧 Implementation Guidelines @@ -155,11 +79,11 @@ Each API template follows this standardized structure: 2. **Set Up Development Environment** ```bash # Clone the repository - git clone https://github.com/hsliuustc0106/vllm-omni.git + git clone https://github.com/vllm-project/vllm-omni.git cd vllm-omni # Install dependencies - pip install -r requirements-dev.txt + pip install -e ".[dev]" # Set up pre-commit hooks pre-commit install @@ -189,28 +113,12 @@ Each API template follows this standardized structure: - Update documentation - Submit pull requests -## 📖 Additional Resources - -### Architecture Overview -- [vLLM-omni Architecture Design](../architecture/vLLM-omni%20arch%20design%20doc.md) -- [Component Categorization](../../ARCHITECTURE_CATEGORIZATION.md) - -### Development Guides -- [Development Setup](../../README.md#development) -- [Testing Guidelines](../../tests/README.md) -- [Contributing Guidelines](../../CONTRIBUTING.md) - -### Examples -- [Basic Examples](../../examples/basic/) -- [Advanced Examples](../../examples/advanced/) -- [Multimodal Examples](../../examples/multimodal/) - ## 🤝 Contributing We welcome contributions to improve the API documentation and implementation. Please: 1. **Follow the Template Structure** - - Use the master template as a guide + - Use the [api template](https://github.com/vllm-project/vllm-omni/docs/contributing/design_documents/api_design_template.md) as a guide - Maintain consistency across modules - Include all required sections diff --git a/docs/api/API_DESIGN_TEMPLATE.md b/docs/contributing/design_documents/api_design_template.md similarity index 100% rename from docs/api/API_DESIGN_TEMPLATE.md rename to docs/contributing/design_documents/api_design_template.md diff --git a/docs/architecture/high_level_arch_design.md b/docs/contributing/design_documents/architecture_overview.md similarity index 99% rename from docs/architecture/high_level_arch_design.md rename to docs/contributing/design_documents/architecture_overview.md index 82a9f1a14..99b21e4b4 100644 --- a/docs/architecture/high_level_arch_design.md +++ b/docs/contributing/design_documents/architecture_overview.md @@ -1,4 +1,4 @@ -# vLLM-omni Architecture Design Document +# Architecture Overview # Introduction @@ -114,7 +114,7 @@ vLLM-omni will import core modules from existing vLLM framework and introduce se For any questions or feedback regarding this design document, please contact Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk), -Hongsheng Liu: [hsliuustc@gmail.com](mailto:hsliuustc@gmail.com) +Hongsheng Liu: [vllm-project@gmail.com](mailto:vllm-project@gmail.com) [image1]: diff --git a/docs/contributing/design_documents/index.md b/docs/contributing/design_documents/index.md new file mode 100644 index 000000000..5598b46ca --- /dev/null +++ b/docs/contributing/design_documents/index.md @@ -0,0 +1,13 @@ +# Design Documents + +This section contains design documents and architecture specifications for vLLM-omni. + +## Architecture Documents + +- [vLLM-omni Software Design Document](vllm_omni_design.md) +- [Architecture Overview](architecture_overview.md) + +## API Design Documentation + +- [vLLM-omni API Documentation](api_design_doc.md) + diff --git a/docs/architecture/vllm_omni_design.md b/docs/contributing/design_documents/vllm_omni_design.md similarity index 99% rename from docs/architecture/vllm_omni_design.md rename to docs/contributing/design_documents/vllm_omni_design.md index 4dd47757f..6275c8950 100644 --- a/docs/architecture/vllm_omni_design.md +++ b/docs/contributing/design_documents/vllm_omni_design.md @@ -1,4 +1,4 @@ -# vLLM-omni Software Design Document +# Architecture Design Principles ## Overview diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..733b3e4e8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,33 @@ +# Welcome to vLLM-omni + +**vLLM-omni** extends vLLM capabilities to support multi-modality model inference and serving, particularly focusing on non-autoregressive structures and non-textual outputs. It aims to support: + +- **Multi-modal Models**: Text, image, video, audio, and sensor data processing +- **Non-autoregressive Architectures**: Diffusion Transformers (DiT) and other parallel generation models +- **Heterogeneous Outputs**: Beyond traditional text generation to multimodal outputs + +## 🎯 Key Features + +### Multi-Engine Support + +- **Autoregressive Engine**: Traditional text generation with enhanced KV-caching +- **Diffusion Engine**: Support for DiT models and iterative generation +- **Hybrid Engine**: Combined AR+DiT processing pipelines + +### Modality Processing + +- **Text**: Advanced tokenization and embedding generation +- **Image**: Vision encoder integration (CLIP, etc.) +- **Audio**: Speech processing and audio embedding + +## 🤖 Supported Models + +- Qwen-omni (Thinker-Talker-Codec structure) +- HunyunaImage 3.0 (Ongoing) +- Qwen-Image (Ongoing) + +## 📚 Documentation Navigation + +- To run open-source models on vLLM-Omni, we recommend starting with the [:material-code-tags: User Quide](user_guide/getting_started/quickstart.md) +- To develop and contribute to vLLM-Omni, we recommend starting with the [:material-tools: Developer Guide](contributing/README.md) + diff --git a/docs/mkdocs/hooks/generate_examples.py b/docs/mkdocs/hooks/generate_examples.py new file mode 100644 index 000000000..518a66f7a --- /dev/null +++ b/docs/mkdocs/hooks/generate_examples.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-omni project +import itertools +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + +import regex as re + +logger = logging.getLogger("mkdocs") + +ROOT_DIR = Path(__file__).parent.parent.parent.parent +ROOT_DIR_RELATIVE = "../../../../.." +EXAMPLE_DIR = ROOT_DIR / "examples" +EXAMPLE_DOC_DIR = ROOT_DIR / "docs/user_guide/examples" + + +def fix_case(text: str) -> str: + subs = { + "api": "API", + "cli": "CLI", + "cpu": "CPU", + "llm": "LLM", + "mae": "MAE", + "tpu": "TPU", + "gguf": "GGUF", + "lora": "LoRA", + "rlhf": "RLHF", + "vllm": "vLLM", + "omni": "Omni", + "openai": "OpenAI", + "lmcache": "LMCache", + "multilora": "MultiLoRA", + "mlpspeculator": "MLPSpeculator", + r"fp\d+": lambda x: x.group(0).upper(), # e.g. fp16, fp32 + r"int\d+": lambda x: x.group(0).upper(), # e.g. int8, int16 + } + for pattern, repl in subs.items(): + text = re.sub(rf"\b{pattern}\b", repl, text, flags=re.IGNORECASE) + return text + + +@dataclass +class Example: + """ + Example class for generating documentation content from a given path. + + Attributes: + path (Path): The path to the main directory or file. + category (str): The category of the document. + main_file (Path): The main file in the directory. + other_files (list[Path]): list of other files in the directory. + title (str): The title of the document. + + Methods: + __post_init__(): Initializes the main_file, other_files, and title attributes. + determine_main_file() -> Path: Determines the main file in the given path. + determine_other_files() -> list[Path]: Determines other files in the directory excluding the main file. + determine_title() -> str: Determines the title of the document. + generate() -> str: Generates the documentation content. + """ # noqa: E501 + + path: Path + category: str = None + main_file: Path = field(init=False) + other_files: list[Path] = field(init=False) + title: str = field(init=False) + + def __post_init__(self): + self.main_file = self.determine_main_file() + self.other_files = self.determine_other_files() + self.title = self.determine_title() + + @property + def is_code(self) -> bool: + return self.main_file.suffix != ".md" + + def determine_main_file(self) -> Path: + """ + Determines the main file in the given path. + If the path is a file, it returns the path itself. Otherwise, it searches + for Markdown files (*.md) in the directory and returns the first one found. + Returns: + Path: The main file path, either the original path if it's a file or the first + Markdown file found in the directory. + Raises: + IndexError: If no Markdown files are found in the directory. + """ # noqa: E501 + return self.path if self.path.is_file() else list(self.path.glob("*.md")).pop() + + def determine_other_files(self) -> list[Path]: + """ + Determine other files in the directory excluding the main file. + + This method checks if the given path is a file. If it is, it returns an empty list. + Otherwise, it recursively searches through the directory and returns a list of all + files that are not the main file. + + Returns: + list[Path]: A list of Path objects representing the other files in the directory. + """ # noqa: E501 + if self.path.is_file(): + return [] + # Text file extensions that can be safely included + text_extensions = {".py", ".md", ".sh", ".yaml", ".yml", ".json", ".txt", ".toml", ".cfg", ".ini"} + is_other_file = lambda file: ( + file.is_file() + and file != self.main_file + and file.suffix in text_extensions + ) + return [file for file in self.path.rglob("*") if is_other_file(file)] + + def determine_title(self) -> str: + if not self.is_code: + # Specify encoding for building on Windows + with open(self.main_file, encoding="utf-8") as f: + first_line = f.readline().strip() + match = re.match(r"^#\s+(?P.+)$", first_line) + if match: + return match.group("title") + return fix_case(self.path.stem.replace("_", " ").title()) + + def fix_relative_links(self, content: str) -> str: + """ + Fix relative links in markdown content by converting them to gh-file + format. + + Args: + content (str): The markdown content to process + + Returns: + str: Content with relative links converted to gh-file format + """ + # Regex to match markdown links [text](relative_path) + # This matches links that don't start with http, https, ftp, or # + link_pattern = r"\[([^\]]*)\]\((?!(?:https?|ftp)://|#)([^)]+)\)" + + def replace_link(match): + link_text = match.group(1) + relative_path = match.group(2) + + # Make relative to repo root + gh_file = (self.main_file.parent / relative_path).resolve() + gh_file = gh_file.relative_to(ROOT_DIR) + + # Make GitHub URL + url = "https://github.com/vllm-project/vllm-omni/" + url += "tree/main" if self.path.is_dir() else "blob/main" + gh_url = f"{url}/{gh_file}" + + return f"[{link_text}]({gh_url})" + + return re.sub(link_pattern, replace_link, content) + + def generate(self) -> str: + content = f"# {self.title}\n\n" + url = "https://github.com/vllm-project/vllm-omni/" + url += "tree/main" if self.path.is_dir() else "blob/main" + content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n" + + # Use long code fence to avoid issues with + # included files containing code fences too + code_fence = "``````" + + if self.is_code: + # Use relative path from ROOT_DIR for snippet inclusion + main_file_rel_path = self.main_file.relative_to(ROOT_DIR) + content += ( + f"{code_fence}{self.main_file.suffix[1:]}\n" + f'--8<-- "{main_file_rel_path}"\n' + f"{code_fence}\n" + ) + else: + with open(self.main_file) as f: + # Skip the title from md snippets as it's been included above + main_content = f.readlines()[1:] + content += self.fix_relative_links("".join(main_content)) + content += "\n" + + if not self.other_files: + return content + + content += "## Example materials\n\n" + for file in sorted(self.other_files): + # Use relative path from ROOT_DIR for snippet inclusion + file_rel_path = file.relative_to(ROOT_DIR) + content += f'??? abstract "{file.relative_to(self.path)}"\n' + if file.suffix != ".md": + content += f" {code_fence}{file.suffix[1:]}\n" + content += f' --8<-- "{file_rel_path}"\n' + if file.suffix != ".md": + content += f" {code_fence}\n" + + return content + + +def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool): + logger.info("Generating example documentation") + logger.debug("Root directory: %s", ROOT_DIR.resolve()) + logger.debug("Example directory: %s", EXAMPLE_DIR.resolve()) + logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve()) + + # Create the EXAMPLE_DOC_DIR if it doesn't exist + if not EXAMPLE_DOC_DIR.exists(): + EXAMPLE_DOC_DIR.mkdir(parents=True) + + categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir()) + + examples = [] + glob_patterns = ["*.py", "*.md"] + # Find categorised examples + for category in categories: + globs = [category.glob(pattern) for pattern in glob_patterns] + for path in itertools.chain(*globs): + examples.append(Example(path, category.stem)) + # Find examples in subdirectories + for path in category.glob("*/*.md"): + examples.append(Example(path.parent, category.stem)) + + # Generate the example documentation + for example in sorted(examples, key=lambda e: e.path.stem): + example_name = f"{example.path.stem}.md" + doc_path = EXAMPLE_DOC_DIR / example.category / example_name + if not doc_path.parent.exists(): + doc_path.parent.mkdir(parents=True) + # Specify encoding for building on Windows + with open(doc_path, "w+", encoding="utf-8") as f: + f.write(example.generate()) + logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR)) diff --git a/docs/mkdocs/stylesheets/extra.css b/docs/mkdocs/stylesheets/extra.css new file mode 100644 index 000000000..e306f7f12 --- /dev/null +++ b/docs/mkdocs/stylesheets/extra.css @@ -0,0 +1,56 @@ +/* Custom CSS for vLLM-omni documentation - based on vLLM's extra.css */ + +/* Improve code block readability */ +.highlight { + border-radius: 4px; +} + +/* Style API documentation */ +.md-typeset .doc-contents { + margin-bottom: 1.5em; +} + +/* Improve table readability */ +.md-typeset table { + display: table; + width: 100%; +} + +/* Light mode: darker section titles (from vLLM) */ +body[data-md-color-scheme="default"] .md-nav__item--section > label.md-nav__link .md-ellipsis { + color: rgba(0, 0, 0, 0.7) !important; + font-weight: 700; +} + +/* Dark mode: lighter gray section titles (from vLLM) */ +body[data-md-color-scheme="slate"] .md-nav__item--section > label.md-nav__link .md-ellipsis { + color: rgba(255, 255, 255, 0.75) !important; + font-weight: 700; +} + +/* Navigation: Expand first level of API Reference by default, collapse deeper levels */ +/* Show direct children of API Reference section (vllm_omni's children: config, core, entrypoints, etc.) */ +.md-nav__item--nested[data-md-state] > .md-nav { + display: none; +} + +/* Expand active items */ +.md-nav__item--nested.md-nav__item--active > .md-nav { + display: block; +} + +/* For API Reference: show first level children (config, core, entrypoints, etc.) by default */ +.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav { + display: block; +} + +/* But keep deeper levels collapsed */ +.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav { + display: none; +} + +/* Expand deeper levels only when active */ +.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested.md-nav__item--active > .md-nav { + display: block; +} + diff --git a/docs/user_guide/examples/index.md b/docs/user_guide/examples/index.md new file mode 100644 index 000000000..3f3e5fd00 --- /dev/null +++ b/docs/user_guide/examples/index.md @@ -0,0 +1,9 @@ +# Examples + +vLLM-omni's examples are split into two categories: + +- If you are using vLLM-omni from within Python code, see the *Offline Inference* section. +- If you are using vLLM-omni from an HTTP application or client, see the *Online Serving* section. + +For detailed example documentation, check the [examples directory](https://github.com/vllm-project/vllm-omni/tree/main/examples) in the repository. + diff --git a/docs/user_guide/examples/offline_inference/qwen_2_5_omni.md b/docs/user_guide/examples/offline_inference/qwen_2_5_omni.md new file mode 100644 index 000000000..38e2f327a --- /dev/null +++ b/docs/user_guide/examples/offline_inference/qwen_2_5_omni.md @@ -0,0 +1,97 @@ +# Offline Example of vLLM-omni for Qwen2.5-omni + +Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen_2_5_omni>. + + +## 🛠️ Installation + +### Environment setup + +Use Docker to keep consistent basic environment (Optional, Recommended) +```bash +docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash +``` + +Set up basic uv environment +```bash +pip install uv +uv venv --python 3.12 --seed +source .venv/bin/activate +``` + +### Installation of vLLM (for users) + +Now we build it based on vLLM v0.11.0. Please install it with command below. +```bash +uv pip install vllm==0.11.0 --torch-backend=auto +``` + +### Installation of vLLM (for developers) + +If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file. + +```bash +git clone https://github.com/vllm-project/vllm.git +cd vllm +git checkout v0.11.0 +``` +Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file. +```bash +export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl +``` +Install vllm with command below (If you have no existing PyTorch). +```bash +uv pip install --editable . +``` + +Install vllm with command below (If you already have PyTorch). +```bash +python use_existing_torch.py +uv pip install -r requirements/build.txt +uv pip install --no-build-isolation --editable . +``` + +### Verification for successful installation of vLLM +Just run the command below. If no error, it demonstrates that the installation is successfull. +```bash +python -c "import vllm._C" +``` + +### Installation of vLLM-omni +Install additional requirements for vllm-omni +```bash +git clone https://github.com/vllm-project/vllm-omni.git +cd vllm_omni +uv pip install -e . +``` + +## Run examples (Qwen2.5-omni) + +Get into the example folder +```bash +cd examples/offline_inference/qwen_2_5_omni +``` +Then run +```bash +bash run.sh +``` +The output audio is saved in ./output_audio + +## Example materials + +??? abstract "end2end.py" + ``````py + --8<-- "examples/offline_inference/qwen_2_5_omni/end2end.py" + `````` +??? abstract "processing_omni.py" + ``````py + --8<-- "examples/offline_inference/qwen_2_5_omni/processing_omni.py" + `````` +??? abstract "run.sh" + ``````sh + --8<-- "examples/offline_inference/qwen_2_5_omni/run.sh" + `````` +??? abstract "utils.py" + ``````py + --8<-- "examples/offline_inference/qwen_2_5_omni/utils.py" + `````` diff --git a/docs/user_guide/examples/online_serving/index.md b/docs/user_guide/examples/online_serving/index.md new file mode 100644 index 000000000..6722ed568 --- /dev/null +++ b/docs/user_guide/examples/online_serving/index.md @@ -0,0 +1,5 @@ +# Online Serving + +## Available Examples + +Online serving is under active development. Coming soon. diff --git a/docs/user_guide/getting_started/quickstart.md b/docs/user_guide/getting_started/quickstart.md new file mode 100644 index 000000000..a28696754 --- /dev/null +++ b/docs/user_guide/getting_started/quickstart.md @@ -0,0 +1,56 @@ +# Quickstart + +This guide will help you get started with vLLM-omni. + +## Installation + +Use Docker to keep consistent basic environment (Optional, Recommended) + +```bash +docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --rm --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash +``` + +Set up basic uv environment + +```bash +pip install uv +uv venv --python 3.12 --seed +source .venv/bin/activate +``` + +Install vLLM + +```bash +uv pip install vllm==0.11.0 --torch-backend=auto +``` + +Install vLLM-omni + +```bash +git clone https://github.com/vllm-project/vllm-omni.git +cd vllm_omni +uv pip install -e . +``` + +## Run examples (Qwen2.5-omni) + +Get into the example folder + +```bash +cd examples/offline_inference/qwen_2_5_omni +``` + +Then run. + +```bash +bash run.sh +``` + +The output audio is saved in ./output_audio + +## Next Steps + +- Read the [architecture documentation](../../contributing/design_documents/vllm_omni_design.md) +- Check out the [API reference](../../api/overview.md) +- Explore the [examples](../examples/index.md) + diff --git a/examples/offline_inference/qwen_2_5_omni/README.md b/examples/offline_inference/qwen_2_5_omni/README.md index 734060b6b..d07e115ee 100644 --- a/examples/offline_inference/qwen_2_5_omni/README.md +++ b/examples/offline_inference/qwen_2_5_omni/README.md @@ -59,7 +59,7 @@ Install additional requirements for vllm-omni ```bash git clone https://github.com/vllm-project/vllm-omni.git cd vllm_omni -uv pip install -r requirements.txt +uv pip install -e . ``` ## Run examples (Qwen2.5-omni) diff --git a/examples/offline_inference/qwen_2_5_omni/output_audio/output_0.wav b/examples/offline_inference/qwen_2_5_omni/output_audio/output_0.wav deleted file mode 100644 index 45caf1d0e..000000000 Binary files a/examples/offline_inference/qwen_2_5_omni/output_audio/output_0.wav and /dev/null differ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..b5a54c6b7 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,180 @@ +site_name: vLLM-omni +site_description: Multi-modality models inference and serving with non-autoregressive structures +site_author: vLLM-omni Team +site_url: https://vllm-project.github.io/vllm-omni/ + +repo_name: vllm-project/vllm-omni +repo_url: https://github.com/vllm-project/vllm-omni +edit_uri: edit/main/docs/ + +# Copyright +copyright: Copyright © 2025 vLLM-omni Team + +# Theme +theme: + name: material + palette: + # Light mode + - scheme: default + primary: blue + accent: blue + toggle: + icon: material/brightness-7 + name: Switch to dark mode + # Dark mode + - scheme: slate + primary: blue + accent: blue + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - content.action.edit + - content.code.copy + - content.tabs.link + - navigation.instant + - navigation.instant.progress + - navigation.tracking + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.indexes + - navigation.top + - search.suggest + - search.highlight + - search.share + - content.code.annotate + - content.tabs + - content.tooltips + - toc.follow + icon: + repo: fontawesome/brands/github + +hooks: + - docs/mkdocs/hooks/generate_examples.py + +# Plugins +plugins: + - meta + - search + # - awesome-nav # Uncomment if installed + - glightbox + - git-revision-date-localized: + # exclude files + exclude: + - api/* + - user_guide/examples/** + - contributing/design_documents/api_design_template.md + - DOCS_GUIDE.md + - minify: + minify_html: true + minify_js: true + minify_css: true + cache_safe: true + - api-autonav: + modules: ["vllm_omni"] + api_root_uri: "api" + nav_item_prefix: "" # No prefix in navigation tree (clean names) + show_full_namespace: false # Show only module name, not full path + exclude: + - "re:vllm_omni\\._.*" # Internal modules + - mkdocstrings: + handlers: + python: + paths: [vllm_omni] + options: + show_symbol_type_heading: true + show_symbol_type_toc: true + show_source: true + show_root_heading: true + show_root_toc_entry: true + show_root_full_path: false + show_object_full_path: false + show_if_no_docstring: true + show_signature_annotations: true + separate_signature: true + show_overloads: true + signature_crossrefs: true + filters: [] + summary: + modules: true + inventories: + - https://docs.python.org/3/objects.inv + - https://typing-extensions.readthedocs.io/en/latest/objects.inv + - https://docs.aiohttp.org/en/stable/objects.inv + - https://pillow.readthedocs.io/en/stable/objects.inv + - https://numpy.org/doc/stable/objects.inv + - https://pytorch.org/docs/stable/objects.inv + - https://psutil.readthedocs.io/en/stable/objects.inv + +# Markdown extensions +markdown_extensions: + - attr_list + - md_in_html + - admonition + - pymdownx.details + # For content tabs + - pymdownx.superfences + - pymdownx.tabbed: + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + alternate_style: true + # For code highlighting + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + # For emoji and icons + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + # For in page [TOC] (not sidebar) + - toc: + permalink: true + # For math rendering + - pymdownx.arithmatex: + generic: true + # For task lists + - pymdownx.tasklist: + custom_checkbox: true + # For tables + - tables + +# Navigation +nav: + - Home: index.md + - User Guide: + - Getting Started: + - user_guide/getting_started/quickstart.md + - Examples: + - user_guide/examples/index.md + - Offline Inference: + - user_guide/examples/offline_inference/qwen_2_5_omni.md + - Online Serving: + - user_guide/examples/online_serving/index.md + - API Reference: + - Summary: api/overview.md + - Developer Guide: + - contributing/README.md + - Design Documents: + - contributing/design_documents/index.md + - contributing/design_documents/vllm_omni_design.md + - contributing/design_documents/architecture_overview.md + - contributing/design_documents/api_design_doc.md + +# Extra +extra: + version: + provider: mike + +# Extra CSS +extra_css: + - mkdocs/stylesheets/extra.css + +# Makes the url format end in .html rather than act as a dir +# So index.md generates as index.html and is available under URL /index.html +# https://www.mkdocs.org/user-guide/configuration/#use_directory_urls +use_directory_urls: false diff --git a/pyproject.toml b/pyproject.toml index 2219a7110..0f433c094 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,10 @@ name = "vllm-omni" version = "0.1.0" description = "vLLM-omni: Multi-modality models inference and serving with non-autoregressive structures" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9,<3.14" # Align with vLLM v0.11 supported Python versions license = {text = "Apache-2.0"} authors = [ - {name = "vLLM-omni Team", email = "hsliuustc@gmail.com"} + {name = "vLLM-omni Team", email = "vllm-project@gmail.com"} ] keywords = ["vllm", "multimodal", "diffusion", "transformer", "inference", "serving"] classifiers = [ @@ -19,31 +19,25 @@ classifiers = [ "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "vllm>=0.2.0", - "torch>=2.0.0", - "transformers>=4.30.0", - "numpy>=1.21.0", - "pillow>=8.0.0", - "opencv-python>=4.5.0", - "PyYAML>=6.0", - "gradio>=3.0.0", - "fastapi>=0.100.0", - "uvicorn>=0.20.0", - "pydantic>=2.0.0", - "asyncio-mqtt>=0.11.0", - "ray>=2.0.0", + # Core runtime dependencies (required for actual usage) + "omegaconf>=2.3.0", + "decord>=0.6.0", + "librosa>=0.11.0", + "resampy>=0.4.3", + # "vllm==0.11.0", # install manually ] [project.optional-dependencies] + dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.21.0", @@ -54,22 +48,27 @@ dev = [ "mypy>=1.0.0", "pre-commit>=3.0.0", ] + docs = [ - "sphinx>=6.0.0", - "sphinx-rtd-theme>=1.2.0", - "myst-parser>=1.0.0", -] -examples = [ - "jupyter>=1.0.0", - "matplotlib>=3.5.0", - "seaborn>=0.11.0", + "mkdocs>=1.5.0", + "mkdocs-api-autonav", + "mkdocs-material", + "mkdocstrings-python", + "mkdocs-gen-files", + # "mkdocs-awesome-nav", # Not available on PyPI + "mkdocs-glightbox", + "mkdocs-git-revision-date-localized-plugin", + "mkdocs-minify-plugin", + "regex", + "pydantic", ] + [project.urls] -Homepage = "https://github.com/hsliuustc0106/vllm-omni" -Repository = "https://github.com/hsliuustc0106/vllm-omni" +Homepage = "https://github.com/vllm-project/vllm-omni" +Repository = "https://github.com/vllm-project/vllm-omni" Documentation = "https://vllm-omni.readthedocs.io" -"Bug Tracker" = "https://github.com/hsliuustc0106/vllm-omni/issues" +"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues" [project.scripts] vllm = "vllm_omni.entrypoints.cli.main:main" @@ -81,7 +80,7 @@ include = ["vllm_omni*"] [tool.black] line-length = 88 -target-version = ['py38', 'py39', 'py310', 'py311'] +target-version = ['py39', 'py310', 'py311', 'py312'] include = '\.pyi?$' extend-exclude = ''' /( @@ -108,7 +107,7 @@ max-line-length = 88 extend-ignore = ["E203", "W503"] [tool.mypy] -python_version = "3.8" +python_version = "3.9" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4a7116da4..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -omegaconf==2.3.0 -decord==0.6.0 -librosa==0.11.0 -resampy==0.4.3 diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index afeb09c1b..ac9b67e1c 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -12,7 +12,13 @@ Architecture: processing """ -from . import patch # noqa: F401 +try: + from . import patch # noqa: F401 +except ModuleNotFoundError as exc: # pragma: no cover - optional dependency + if exc.name != "vllm": + raise + # Allow importing vllm_omni without vllm (e.g., documentation builds) + patch = None # type: ignore from .config import OmniModelConfig # Main entry points diff --git a/vllm_omni/distributed/__init__.py b/vllm_omni/distributed/__init__.py index e69de29bb..139597f9c 100644 --- a/vllm_omni/distributed/__init__.py +++ b/vllm_omni/distributed/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index abadb919c..6feff5940 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -1,4 +1,10 @@ -from typing import Any, NotRequired, Optional +from typing import Any, Optional + +try: + from typing import NotRequired +except ImportError: + # Python < 3.11: use typing_extensions + from typing_extensions import NotRequired import torch @@ -39,8 +45,7 @@ def token_inputs_omni( prompt_embeds: Optional[torch.Tensor] = None, additional_information: Optional[dict[str, Any]] = None, ) -> OmniTokenInputs: - """Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional - values.""" + """Construct token inputs with optional embeddings and metadata.""" inputs = OmniTokenInputs(type="token", prompt_token_ids=prompt_token_ids) if prompt is not None: diff --git a/vllm_omni/inputs/preprocess.py b/vllm_omni/inputs/preprocess.py index b1943318b..5bae25b8d 100644 --- a/vllm_omni/inputs/preprocess.py +++ b/vllm_omni/inputs/preprocess.py @@ -62,7 +62,7 @@ class OmniInputPreprocessor(InputPreprocessor): Returns: - * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance + * Input container compatible with vLLM's singleton prompt handling. """ parsed = parse_singleton_prompt_omni(prompt) diff --git a/vllm_omni/model_executor/models/qwen2_5_omni.py b/vllm_omni/model_executor/models/qwen2_5_omni.py index b8e3a2d9a..1768c4395 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni.py @@ -640,7 +640,9 @@ class Qwen2_5OmniForConditionalGeneration( # Use thinker model for sampling return self.model.sample(logits, sampling_metadata) - def generate_speech(self, text_tokens: torch.Tensor, voice_type: str = "default"): + def generate_speech( + self, text_tokens: torch.Tensor, voice_type: str = "default" + ) -> torch.Tensor: """ Generate speech from text tokens using the talker and token2wav models. This method is kept for backward compatibility and direct speech generation. diff --git a/vllm_omni/model_executor/models/qwen2_5_omni_token2wav.py b/vllm_omni/model_executor/models/qwen2_5_omni_token2wav.py index 9af9dcdbe..d2271ba1f 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni_token2wav.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni_token2wav.py @@ -748,7 +748,9 @@ class SnakeBeta(nn.Module): return hidden_states -def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): +def kaiser_sinc_filter1d( + cutoff: float, half_width: float, kernel_size: int +) -> torch.Tensor: """Generates a 1D Kaiser-windowed sinc filter. Args: @@ -1370,14 +1372,14 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): @torch.no_grad() def fast_block_sample( self, - conditioning_vector, - reference_mel_spectrogram, - quantized_code, + conditioning_vector: torch.Tensor, + reference_mel_spectrogram: torch.Tensor, + quantized_code: torch.Tensor, y0: torch.Tensor, - num_steps=10, - guidance_scale=0.5, + num_steps: int = 10, + guidance_scale: float = 0.5, sway_coefficient: Optional[float] = -1.0, - ): + ) -> torch.Tensor: """ Block-wise ODE sampling starting from provided initial state y0.