vllm-omni/.buildkite/amd/test-template-amd-omni.j2

{# vllm-omni customized version
   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
   Last synced: 2026-08-04
   Modifications: Use the shared MI300 queues and Omni command-passing contract
#}
{% set docker_registry_amd = "rocm/vllm-omni" %}
{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
{% set default_working_dir = "/app/vllm-omni" %}

{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
{% macro amd_native_k8s_plugin(indent, step) -%}
{{ indent }}plugins:
{{ indent }}  - kubernetes:
{{ indent }}      podSpecPatch:
{{ indent }}        automountServiceAccountToken: false
{{ indent }}        securityContext:
{{ indent }}          seccompProfile:
{{ indent }}            type: RuntimeDefault
{{ indent }}        imagePullSecrets:
{{ indent }}          - name: docker-config
{{ indent }}        containers:
{{ indent }}          - name: container-0
{{ indent }}            image: {{ docker_image_amd }}
{{ indent }}            imagePullPolicy: Always
{{ indent }}            securityContext:
{{ indent }}              allowPrivilegeEscalation: false
{{ indent }}              capabilities:
{{ indent }}                add:
{{ indent }}                  - IPC_LOCK
{{ indent }}            resources:
{{ indent }}              limits:
{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
{{ indent }}              requests:
{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
{{ indent }}            volumeMounts:
{{ indent }}              - name: devshm
{{ indent }}                mountPath: /dev/shm
{{ indent }}            env:
{{ indent }}              - name: AMD_CI_RUNTIME
{{ indent }}                value: native
{{ indent }}              - name: NATIVE_CI
{{ indent }}                value: "true"
{{ indent }}              - name: VLLM_CI_DOCKER_DISABLED
{{ indent }}                value: "1"
{{ indent }}              - name: VLLM_CI_EXPECTED_GPU_COUNT
{{ indent }}                value: "{{ step.agent_pool[-1:] }}"
{{ indent }}              - name: HF_HOME
{{ indent }}                value: /home/buildkite-agent/huggingface
{{ indent }}              - name: MIOPEN_DEBUG_CONV_DIRECT
{{ indent }}                value: "0"
{{ indent }}              - name: MIOPEN_DEBUG_CONV_GEMM
{{ indent }}                value: "0"
{{ indent }}              - name: VLLM_ROCM_USE_AITER
{{ indent }}                value: "0"
{{ indent }}              - name: PYTORCH_ROCM_ARCH
{{ indent }}                value: ""
{{ indent }}        volumes:
{{ indent }}          - name: devshm
{{ indent }}            emptyDir:
{{ indent }}              medium: Memory
{{ indent }}              sizeLimit: 16Gi
{%- endmacro %}

{% if env %}
env:
{% for name in env %}
  {{ name }}: "{{ env[name] }}"
{% endfor %}
{% endif %}
steps:
  - group: "AMD Tests"
    depends_on: ~
    steps:
      - label: "AMD: :docker: build image"
        depends_on: ~
        soft_fail: false
        commands:
          - "bash .buildkite/amd/scripts/build-ci-image.sh"
        key: "amd-build"
        env:
          DOCKER_BUILDKIT: "1"
          PYTORCH_ROCM_ARCH: "gfx942"
          ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
        retry:
          automatic:
            - exit_status: -1  # Agent was lost
              limit: 2
            - exit_status: -10  # Agent was lost
              limit: 2
            - exit_status: 128  # Git connectivity issues
              limit: 2
            - exit_status: 1  # Machine occasionally fail
              limit: 1
        agents:
          queue: amd-cpu

{% for step_group in steps %}
{% if step_group.group %}
  - group: "{{ step_group.group }}"
{% if step_group.depends_on %}
    depends_on: {{ step_group.depends_on }}
{% endif %}
    steps:
{% for step in step_group.steps %}
{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
      - label: "{{ step.agent_pool }}: {{ step.label }}"
        depends_on: amd-build
        agents:
          {% if step.agent_pool %}
          queue: amd_{{ step.agent_pool }}
          {% else %}
          queue: amd_mi300_1
          {% endif %}
{{ amd_native_k8s_plugin("        ", step) }}
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
{% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
        command: bash .buildkite/amd/scripts/run-amd-test.sh
        env:
          AMD_CI_RUNTIME: "native"
          NATIVE_CI: "true"
          VLLM_CI_DOCKER_DISABLED: "1"
          VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
          VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
          HF_HOME: "/home/buildkite-agent/huggingface"
          HF_HUB_DOWNLOAD_TIMEOUT: "300"
          HF_HUB_ETAG_TIMEOUT: "60"
          MIOPEN_DEBUG_CONV_DIRECT: "0"
          MIOPEN_DEBUG_CONV_GEMM: "0"
          VLLM_ROCM_USE_AITER: "0"
          PYTORCH_ROCM_ARCH: ""
{% if step.env %}
{% for name in step.env %}
          {{ name }}: "{{ step.env[name] }}"
{% endfor %}
{% endif %}
          TEST_COMMANDS: |-
            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
{% if env %}
{% for name in env %}
            export {{ name }}="{{ env[name] }}"
{% endfor %}
{% endif %}
{% if step.env %}
{% for name in step.env %}
            export {{ name }}="{{ step.env[name] }}"
{% endfor %}
{% endif %}
{% if "mi250" in step.agent_pool %}
            python3 -m pip uninstall -y amd-aiter
{% endif %}
            {{ indented_cmd | safe }}
        priority: 100
{% if step.parallelism %}
        parallelism: {{ step.parallelism }}
{% endif %}
{% if step.artifact_paths %}
        artifact_paths:
{% for path in step.artifact_paths %}
          - "{{ path }}"
{% endfor %}
{% endif %}
{% if step.retry %}
        retry:
          automatic:
{% for retry_rule in step.retry.automatic %}
            - exit_status: {{ retry_rule.exit_status }}
              limit: {{ retry_rule.limit }}
{% endfor %}
{% endif %}
        {% if step.timeout_in_minutes %}
        timeout_in_minutes: {{ step.timeout_in_minutes }}
        {% endif %}
        {% if step.grade and step.grade == "Blocking" %}
        soft_fail: false
        {% else %}
        soft_fail: true
        {% endif%}
{% endif %}
{% endfor %}
{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
{% set step = step_group %}
  - label: "{{ step.agent_pool }}: {{ step.label }}"
    depends_on: amd-build
    agents:
      {% if step.agent_pool %}
      queue: amd_{{ step.agent_pool }}
      {% else %}
      queue: amd_mi300_1
      {% endif %}
{{ amd_native_k8s_plugin("    ", step) }}
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
{% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
    command: bash .buildkite/amd/scripts/run-amd-test.sh
    env:
      AMD_CI_RUNTIME: "native"
      NATIVE_CI: "true"
      VLLM_CI_DOCKER_DISABLED: "1"
      VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
      VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
      HF_HOME: "/home/buildkite-agent/huggingface"
      HF_HUB_DOWNLOAD_TIMEOUT: "300"
      HF_HUB_ETAG_TIMEOUT: "60"
      MIOPEN_DEBUG_CONV_DIRECT: "0"
      MIOPEN_DEBUG_CONV_GEMM: "0"
      VLLM_ROCM_USE_AITER: "0"
      PYTORCH_ROCM_ARCH: ""
{% if step.env %}
{% for name in step.env %}
      {{ name }}: "{{ step.env[name] }}"
{% endfor %}
{% endif %}
      TEST_COMMANDS: |-
        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
{% if env %}
{% for name in env %}
        export {{ name }}="{{ env[name] }}"
{% endfor %}
{% endif %}
{% if step.env %}
{% for name in step.env %}
        export {{ name }}="{{ step.env[name] }}"
{% endfor %}
{% endif %}
{% if "mi250" in step.agent_pool %}
        python3 -m pip uninstall -y amd-aiter
{% endif %}
        {{ indented_cmd | safe }}
    priority: 100
{% if step.parallelism %}
    parallelism: {{ step.parallelism }}
{% endif %}
{% if step.artifact_paths %}
    artifact_paths:
{% for path in step.artifact_paths %}
      - "{{ path }}"
{% endfor %}
{% endif %}
{% if step.retry %}
    retry:
      automatic:
{% for retry_rule in step.retry.automatic %}
        - exit_status: {{ retry_rule.exit_status }}
          limit: {{ retry_rule.limit }}
{% endfor %}
{% endif %}
    {% if step.timeout_in_minutes %}
    timeout_in_minutes: {{ step.timeout_in_minutes }}
    {% endif %}
    {% if step.grade and step.grade == "Blocking" %}
    soft_fail: false
    {% else %}
    soft_fail: true
    {% endif%}
{% endif %}
{% endfor %}