diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 560d704bf..45b9dd283 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -32,6 +32,6 @@ jobs: for file in ${{ steps.getfile.outputs.files }}; do echo $file done - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 script/test-cm-core/src/script/process_tests.py ${{ steps.getfile.outputs.files }} diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 65262dea6..6016a99b7 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -48,9 +48,16 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Pull repo where test cases are uploaded run: | diff --git a/.github/workflows/test-cm-script-features.yml b/.github/workflows/test-cm-script-features.yml index d793f9334..728223df1 100644 --- a/.github/workflows/test-cm-script-features.yml +++ b/.github/workflows/test-cm-script-features.yml @@ -30,9 +30,16 @@ jobs: if: matrix.os == 'windows-latest' run: | git config --system core.longpaths true - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Run test_docker on linux diff --git a/.github/workflows/test-cm-tutorial-retinanet.yml b/.github/workflows/test-cm-tutorial-retinanet.yml index 2899a26d1..5fa649772 100644 --- a/.github/workflows/test-cm-tutorial-retinanet.yml +++ b/.github/workflows/test-cm-tutorial-retinanet.yml @@ -26,9 +26,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test CM Tutorial Retinanet diff --git a/.github/workflows/test-cm-tutorial-tvm-pip.yml b/.github/workflows/test-cm-tutorial-tvm-pip.yml index ce18079c4..8cc87ca0e 100644 --- a/.github/workflows/test-cm-tutorial-tvm-pip.yml +++ b/.github/workflows/test-cm-tutorial-tvm-pip.yml @@ -27,9 +27,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test CM Tutorial TVM pip install with VirtualMachine Runtime @@ -49,9 +51,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test CM Tutorial TVM pip install with GraphExecutor Runtime diff --git a/.github/workflows/test-cm-tutorial-tvm.yml b/.github/workflows/test-cm-tutorial-tvm.yml index d5d2937ea..96b653b0f 100644 --- a/.github/workflows/test-cm-tutorial-tvm.yml +++ b/.github/workflows/test-cm-tutorial-tvm.yml @@ -28,9 +28,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test CM Tutorial TVM diff --git a/.github/workflows/test-image-classification-onnx.yml b/.github/workflows/test-image-classification-onnx.yml index 6d8bc91d5..075de9c76 100644 --- a/.github/workflows/test-image-classification-onnx.yml +++ b/.github/workflows/test-image-classification-onnx.yml @@ -30,9 +30,16 @@ jobs: if: matrix.os == 'windows-latest' run: | git config --system core.longpaths true - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test image classification with ONNX diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 44b0d6b6d..463c9dbc7 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -51,9 +51,16 @@ jobs: if: matrix.os == 'windows-latest' run: | git config --system core.longpaths true - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} #cm pull repo mlcommons@cm4abtf --branch=poc diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index 9aa9b8293..0e6ebe596 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -35,9 +35,16 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} if: matrix.os == 'windows-latest' diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 72b0d1fe3..7a700f574 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -32,16 +32,23 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm cm run script --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }} - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} if: matrix.os == 'windows-latest' run: | - cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet + cm run script --tags=app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} if: matrix.os != 'windows-latest' run: | diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 9700e4b79..613457691 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -69,7 +69,7 @@ jobs: ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT # Fetch PAT and store in environment variable - name: Push Results - if: github.repository_owner == 'mlcommons' + if: github.repository_owner == 'mlcommons_off' env: GITHUB_TOKEN: ${{ env.PAT }} run: | diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 62d5a5139..5511151eb 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -25,9 +25,16 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install cm4mlops on Windows + if: matrix.os == 'windows-latest' + run: | + $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops + - name: Install dependencies on Unix Platforms + if: matrix.os != 'windows-latest' + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | diff --git a/.github/workflows/test-mlperf-inference-rnnt.yml b/.github/workflows/test-mlperf-inference-rnnt.yml index 08b681fea..9937f9260 100644 --- a/.github/workflows/test-mlperf-inference-rnnt.yml +++ b/.github/workflows/test-mlperf-inference-rnnt.yml @@ -28,9 +28,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: Test MLPerf Inference RNNT diff --git a/.github/workflows/test-mlperf-inference-tvm.yml b/.github/workflows/test-mlperf-inference-tvm.yml index e07d4ce11..359b5be72 100644 --- a/.github/workflows/test-mlperf-inference-tvm.yml +++ b/.github/workflows/test-mlperf-inference-tvm.yml @@ -28,9 +28,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm - name: MLPerf Inference ResNet50 using TVM diff --git a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml index ac4922f3c..0055a1c31 100644 --- a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml +++ b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml @@ -26,9 +26,11 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Unix Platforms + run: | + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model run: | diff --git a/.github/workflows/test-qaic-compute-sdk-build.yml b/.github/workflows/test-qaic-compute-sdk-build.yml index 0bf4f4928..6bb069685 100644 --- a/.github/workflows/test-qaic-compute-sdk-build.yml +++ b/.github/workflows/test-qaic-compute-sdk-build.yml @@ -26,8 +26,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" - cm pull repo mlcommons@ck + CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops cm run script --tags=get,sys-utils-cm --quiet - name: Test QAIC Compute SDK for compilation diff --git a/.github/workflows/test-qaic-software-kit.yml b/.github/workflows/test-qaic-software-kit.yml index febc14b4c..7e8e6e662 100644 --- a/.github/workflows/test-qaic-software-kit.yml +++ b/.github/workflows/test-qaic-software-kit.yml @@ -29,10 +29,9 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Pull MLOps repository run: | - python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm" - cm pull repo mlcommons@ck + pip install cm4mlops cm run script --tags=get,sys-utils-cm --quiet - name: Test Software Kit for compilation on Ubuntu 20.04 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 000000000..9f08749f8 --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,43 @@ +# Contributors + +Thank you for your interest in contributing to **MLPerf Automations**! We welcome contributions that help improve the project and expand its functionality. + +--- + +## How to Become a Contributor + +We value all contributions, whether they are code, documentation, bug reports, or feature suggestions. If you contribute **more than 50 lines of code** (including tests and documentation), you will be officially recognized as a project contributor. + +**Note:** Trivial contributions, such as minor typo fixes or small formatting changes, will not count toward the 50-line threshold. + +To contribute: +1. **Fork** the repository. +2. **Create** a new branch for your feature or bug fix. +3. **Submit** a pull request (PR) describing your changes. +Please see [here](CONTRIBUTING.md) for further guidelines for official contribution to any MLCommons repository. + +--- + +## Contributor Recognition + +Once your contribution exceeds 50 lines of code (in total), we will: +- Add your name to this `CONTRIBUTORS.md` file. +- Highlight your contribution in the next release notes. +- Grant you access to suggest and vote on new features. + +--- + +## Current Contributors + +- **Grigori Fursin** - *Initial Development, CLI workflow support via CMind, Added core automation features* +- **Arjun Suresh** - *Initial Development, Added core automation features* +- **Anandhu Sooraj** - *Added multiple CM scripts for MLPerf Inference* +- **Thomaz Zhu** - *Added CPP implementation for MLPerf Inference Onnxruntime* +- **Sahil Avaran** - *Adding logging support in MLPerf script automation* +- **[Your Name Here]** - This could be you! 🎉 + +--- + +We believe in collaborative growth, and every contribution makes a difference. Feel free to reach out by opening an issue if you have any questions or ideas. + +Happy Coding! 🚀 diff --git a/README.md b/README.md index ea0454c2e..800db041b 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,65 @@ # MLPerf Automations and Scripts -This repository contains the automations and scripts used to run MLPerf benchmarks, primarily focusing on MLPerf inference benchmarks. The automations used here are largely based on and extended from the [Collective Mind script automations](https://github.com/mlcommons/cm4mlops/tree/main/automation/script). +[![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md) +[![Downloads](https://static.pepy.tech/badge/cm4mlops)](https://pepy.tech/project/cm4mlops) +[![CM Script Automation Test](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-cm-script-features.yml) +[![MLPerf Inference ABTF POC Test](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml/badge.svg)](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml) +Welcome to the **MLPerf Automations and Scripts** repository! This repository provides tools, automations, and scripts to facilitate running MLPerf benchmarks, with a primary focus on **MLPerf Inference benchmarks**. -## Collective Mind (CM) Automations +The automations build upon and extend the powerful [Collective Mind (CM) script automations](https://github.com/mlcommons/cm4mlops/tree/main/automation/script) to streamline benchmarking and workflow processes. -**CM (Collective Mind)** is a Python package with a CLI and API designed to create and manage automations. Two key automations developed using CM are **Script** and **Cache**, which streamline ML workflows, including managing Docker runs. +--- +## 🚀 Key Features +- **Automated Benchmarking** – Simplifies running MLPerf Inference benchmarks with minimal manual intervention. +- **Modular and Extensible** – Easily extend the scripts to support additional benchmarks and configurations. +- **Seamless Integration** – Compatible with Docker, cloud environments, and local machines. +- **Collective Mind (CM) Integration** – Utilizes the CM framework to enhance reproducibility and automation. -## License +--- -[Apache 2.0](LICENSE.md) +## 🧰 Collective Mind (CM) Automations + +The **Collective Mind (CM)** framework is a Python-based package offering both CLI and API support for creating and managing automations. CM automations enhance ML workflows by simplifying complex tasks such as Docker container management and caching. + +### Core Automations +- **Script Automation** – Automates script execution across different environments. +- **Cache Management** – Manages reusable cached results to accelerate workflow processes. + +Learn more about CM in the [CM4MLOps documentation](https://github.com/mlcommons/cm4mlops). + +--- + +## 🤝 Contributing +We welcome contributions from the community! To contribute: +1. Submit pull requests (PRs) to the **`dev`** branch. +2. Review our [CONTRIBUTORS.md](here) for guidelines and best practices. +3. Explore more about MLPerf Inference automation in the official [MLPerf Inference Documentation](https://docs.mlcommons.org/inference/). + +Your contributions help drive the project forward! + +--- + +## 📰 News +Stay tuned for upcoming updates and announcements. + +--- + +## 📄 License +This project is licensed under the [Apache 2.0 License](LICENSE.md). + +--- + +## 💡 Acknowledgments and Funding +This project is made possible through the generous support of: +- [OctoML](https://octoml.ai) +- [cKnowledge.org](https://cKnowledge.org) +- [cTuning Foundation](https://cTuning.org) +- [MLCommons](https://mlcommons.org) + +We appreciate their contributions and sponsorship! + +--- + +Thank you for your interest and support in MLPerf Automations and Scripts! diff --git a/automation/script/module.py b/automation/script/module.py index f37dc9830..262d7d574 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -32,6 +32,7 @@ def __init__(self, cmind, automation_file): self.run_state['fake_deps'] = False self.run_state['parent'] = None self.run_state['version_info'] = [] + self.run_state['cache'] = False self.file_with_cached_state = 'cm-cached-state.json' @@ -74,7 +75,8 @@ def __init__(self, cmind, automation_file): 'accept_license', 'skip_system_deps', 'git_ssh', - 'gh_token'] + 'gh_token', + 'hf_token'] ############################################################ @@ -826,6 +828,7 @@ def _run(self, i): 'alias', '') run_state['script_repo_git'] = script_artifact.repo_meta.get( 'git', False) + run_state['cache'] = meta.get('cache', False) if not recursion: run_state['script_entry_repo_to_report_errors'] = meta.get( @@ -1125,7 +1128,7 @@ def _run(self, i): # Check if the output of a selected script should be cached cache = False if i.get( 'skip_cache', - False) else meta.get( + False) else run_state.get( 'cache', False) cache = cache or ( @@ -6163,6 +6166,9 @@ def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps Internal: update env and state from meta """ + if meta.get('cache', '') != '': + run_state['cache'] = meta['cache'] + default_env = meta.get('default_env', {}) for key in default_env: env.setdefault(key, default_env[key]) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 8ff24cad4..250d2dc86 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -26,10 +26,10 @@ default_env: CM_MLPERF_LOADGEN_SCENARIO: Offline CM_OUTPUT_FOLDER_NAME: test_results CM_MLPERF_RUN_STYLE: test - CM_TEST_QUERY_COUNT: '10' + CM_TEST_QUERY_COUNT: "10" CM_MLPERF_QUANTIZATION: off CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX: reference - CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX: '' + CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX: "" docker: real_run: False @@ -67,7 +67,6 @@ input_mapping: network: CM_NETWORK_LOADGEN sut_servers: CM_NETWORK_LOADGEN_SUT_SERVERS - # Duplicate CM environment variables to the ones used in native apps env_key_mappings: CM_HOST_: HOST_ @@ -89,7 +88,6 @@ new_state_keys: # Dependencies on other CM scripts deps: - # Detect host OS features - tags: detect,os @@ -102,8 +100,8 @@ deps: # Detect/install python - tags: get,python names: - - python - - python3 + - python + - python3 # Detect CUDA if required - tags: get,cuda,_cudnn @@ -111,21 +109,18 @@ deps: - cuda enable_if_env: CM_MLPERF_DEVICE: - - gpu + - gpu CM_MLPERF_BACKEND: - - onnxruntime - - tf - - tflite - - pytorch + - onnxruntime + - tf + - tflite + - pytorch # Detect TensorRT if required - tags: get,nvidia,tensorrt enable_if_env: CM_MLPERF_BACKEND: - - tensorrt - - - + - tensorrt ######################################################################## # Install ML engines via CM @@ -133,54 +128,54 @@ deps: ## Onnx CPU Runtime - tags: get,generic-python-lib,_onnxruntime names: - - ml-engine-onnxruntime - - onnxruntime + - ml-engine-onnxruntime + - onnxruntime enable_if_env: CM_MLPERF_BACKEND: - - onnxruntime - - tvm-onnx + - onnxruntime + - tvm-onnx CM_MLPERF_DEVICE: - - cpu - - rocm + - cpu + - rocm ## Onnx CUDA Runtime - tags: get,generic-python-lib,_onnxruntime_gpu names: - - ml-engine-onnxruntime-cuda + - ml-engine-onnxruntime-cuda enable_if_env: CM_MLPERF_BACKEND: - - onnxruntime - - tvm-onnx + - onnxruntime + - tvm-onnx CM_MLPERF_DEVICE: - - gpu + - gpu skip_if_env: CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 + - 3d-unet-99 + - 3d-unet-99.9 ## resnet50 and 3d-unet need both onnxruntime and onnxruntime_gpu on cuda - tags: get,generic-python-lib,_onnxruntime enable_if_env: CM_MLPERF_BACKEND: - - onnxruntime + - onnxruntime CM_MLPERF_DEVICE: - - gpu + - gpu CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 - - resnet50 + - 3d-unet-99 + - 3d-unet-99.9 + - resnet50 - tags: get,generic-python-lib,_onnxruntime_gpu env: CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS: "" enable_if_env: CM_MLPERF_BACKEND: - - onnxruntime + - onnxruntime CM_MLPERF_DEVICE: - - gpu + - gpu CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 - - resnet50 + - 3d-unet-99 + - 3d-unet-99.9 + - resnet50 ## Pytorch (CPU) - tags: get,generic-python-lib,_torch @@ -194,30 +189,30 @@ deps: - dlrm-v2-99.9 enable_if_env: CM_MLPERF_BACKEND: - - pytorch - - tvm-pytorch + - pytorch + - tvm-pytorch CM_MLPERF_DEVICE: - - cpu - - rocm + - cpu + - rocm ## Pytorch (CUDA) - tags: get,generic-python-lib,_torch_cuda names: - - ml-engine-pytorch - - pytorch + - ml-engine-pytorch + - pytorch enable_if_env: CM_MLPERF_BACKEND: - - pytorch - - tvm-pytorch - - ray + - pytorch + - tvm-pytorch + - ray CM_MLPERF_DEVICE: - - gpu + - gpu ## Torchvision (CPU) - tags: get,generic-python-lib,_torchvision names: - - ml-engine-torchvision - - torchvision + - ml-engine-torchvision + - torchvision skip_if_env: CM_MODEL: - dlrm-v2-99 @@ -225,16 +220,16 @@ deps: - rgat enable_if_env: CM_MLPERF_BACKEND: - - pytorch - - tvm-pytorch + - pytorch + - tvm-pytorch CM_MLPERF_DEVICE: - - cpu + - cpu ## Torchvision (CUDA) - tags: get,generic-python-lib,_torchvision_cuda names: - - ml-engine-torchvision - - torchvision + - ml-engine-torchvision + - torchvision skip_if_env: CM_MODEL: - dlrm-v2-99 @@ -242,32 +237,32 @@ deps: - rgat enable_if_env: CM_MLPERF_BACKEND: - - pytorch - - tvm-pytorch - - ray + - pytorch + - tvm-pytorch + - ray CM_MLPERF_DEVICE: - - gpu + - gpu ## tensorrt - tags: get,generic-python-lib,_tensorrt names: - - ml-engine-tensorrt + - ml-engine-tensorrt enable_if_env: CM_MLPERF_BACKEND: - - ray + - ray ## torch_tensorrt - tags: get,generic-python-lib,_torch_tensorrt names: - - ml-engine-torch_tensorrt + - ml-engine-torch_tensorrt enable_if_env: CM_MLPERF_BACKEND: - - ray + - ray ## Ray - tags: get,generic-python-lib,_ray names: - - ray + - ray enable_if_env: CM_MLPERF_BACKEND: - ray @@ -277,7 +272,7 @@ deps: # async_timeout to be installed, so we need to install it manually. - tags: get,generic-python-lib,_async_timeout names: - - async_timeout + - async_timeout enable_if_env: CM_MLPERF_BACKEND: - ray @@ -285,49 +280,48 @@ deps: ## Transformers - tags: get,generic-python-lib,_transformers names: - - ml-engine-transformers + - ml-engine-transformers enable_if_env: CM_MODEL: - - bert-99 - - bert-99.9 - - gptj-99 - - gptj-99.9 + - bert-99 + - bert-99.9 + - gptj-99 + - gptj-99.9 ## Tensorflow - tags: get,generic-python-lib,_tensorflow names: - - ml-engine-tensorflow - - tensorflow + - ml-engine-tensorflow + - tensorflow enable_if_env: CM_MLPERF_BACKEND: - - tf + - tf ## NCNN - tags: get,generic-python-lib,_package.ncnn names: - - ml-engine-ncnn + - ml-engine-ncnn enable_if_env: CM_MLPERF_BACKEND: - - ncnn - + - ncnn + - tags: get,tensorflow,lib,_tflite names: - - ml-engine-tflite + - ml-engine-tflite enable_if_env: CM_MLPERF_BACKEND: - - tflite - + - tflite ######################################################################## - # Install ML models + # Install ML models - tags: get,ml-model,neural-magic,zoo # sets CM_MLPERF_CUSTOM_MODEL_PATH names: - - custom-ml-model + - custom-ml-model enable_if_env: CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB: - - "on" + - "on" update_tags_from_env_with_prefix: "_model-stub.": - CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB @@ -335,93 +329,91 @@ deps: ## ResNet50 - tags: get,ml-model,image-classification,resnet50 names: - - ml-model - - resnet50-model + - ml-model + - resnet50-model enable_if_env: CM_MODEL: - - resnet50 + - resnet50 skip_if_env: CM_MLPERF_CUSTOM_MODEL_PATH: - - "on" + - "on" ## RetinaNet - tags: get,ml-model,object-detection,retinanet names: - - ml-model - - retinanet-model + - ml-model + - retinanet-model enable_if_env: CM_MODEL: - - retinanet + - retinanet ## GPT-J - tags: get,ml-model,large-language-model,gptj names: - - ml-model - - gptj-model - - gpt-j-model + - ml-model + - gptj-model + - gpt-j-model enable_if_env: CM_MODEL: - - gptj-99 - - gptj-99.9 + - gptj-99 + - gptj-99.9 skip_if_env: CM_NETWORK_LOADGEN: - - lon - - + - lon ## RetinaNet (PyTorch weights, FP32) - tags: get,ml-model,object-detection,resnext50,fp32,_pytorch-weights names: - - ml-model - - retinanet-model + - ml-model + - retinanet-model enable_if_env: CM_MLPERF_BACKEND: - - pytorch + - pytorch CM_MLPERF_IMPLEMENTATION: - - nvidia + - nvidia CM_MODEL: - - retinanet + - retinanet ## BERT - tags: get,ml-model,language-processing,bert-large names: - - ml-model - - bert-model + - ml-model + - bert-model enable_if_env: CM_MODEL: - - bert-99 - - bert-99.9 + - bert-99 + - bert-99.9 skip_if_env: CM_MLPERF_CUSTOM_MODEL_PATH: - - "on" + - "on" ## SDXL - tags: get,ml-model,stable-diffusion,text-to-image,sdxl names: - - ml-model - - sdxl-model - - ml-model-float16 + - ml-model + - sdxl-model + - ml-model-float16 enable_if_env: CM_MODEL: - - stable-diffusion-xl + - stable-diffusion-xl skip_if_any_env: CM_MLPERF_CUSTOM_MODEL_PATH: - - "on" + - "on" skip_if_env: CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST: - - 'yes' + - "yes" ## LLAMA2-70B - tags: get,ml-model,llama2 names: - - ml-model - - llama2-model + - ml-model + - llama2-model enable_if_env: CM_MODEL: - - llama2-70b-99 - - llama2-70b-99.9 + - llama2-70b-99 + - llama2-70b-99.9 skip_if_any_env: CM_MLPERF_CUSTOM_MODEL_PATH: - "on" @@ -429,72 +421,86 @@ deps: - "on" skip_if_env: CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: - - 'yes' + - "yes" CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" ## mixtral-8x7b - tags: get,ml-model,mixtral names: - - ml-model - - mixtral-model + - ml-model + - mixtral-model enable_if_env: CM_MODEL: - - mixtral-8x7b + - mixtral-8x7b skip_if_any_env: CM_MLPERF_CUSTOM_MODEL_PATH: - - "on" + - "on" skip_if_env: CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - - 'yes' + - "yes" CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" ## 3d-unet - tags: get,ml-model,medical-imaging,3d-unet names: - - ml-model - - 3d-unet-model + - ml-model + - 3d-unet-model enable_if_env: CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 + - 3d-unet-99 + - 3d-unet-99.9 ## Rnnt - tags: get,ml-model,speech-recognition,rnnt names: - - ml-model - - rnnt-model + - ml-model + - rnnt-model enable_if_env: CM_MODEL: - - rnnt + - rnnt ## Dlrm - tags: get,ml-model,recommendation,dlrm names: - - ml-model - - dlrm-model + - ml-model + - dlrm-model enable_if_env: CM_MODEL: - - dlrm-99 - - dlrm-99.9 - - dlrm-v2-99 - - dlrm-v2-99.9 + - dlrm-99 + - dlrm-99.9 + - dlrm-v2-99 + - dlrm-v2-99.9 skip_if_env: CM_ML_MODEL_FILE_WITH_PATH: - - 'on' - + - "on" ## RGAT - tags: get,ml-model,rgat names: - - rgat-model + - rgat-model enable_if_env: CM_MODEL: - - rgat + - rgat skip_if_env: RGAT_CHECKPOINT_PATH: - - 'on' + - "on" + + ## LLAMA3_1-405B + - tags: get,ml-model,llama3 + names: + - llama3-405b-model + - llama3-402b-model + enable_if_env: + CM_MODEL: + - llama3_1-405b + - llama3-405b + skip_if_env: + CM_USE_MODEL_FROM_HOST: + - "yes" + CM_RUN_STATE_DOCKER: + - "yes" ######################################################################## # Install datasets @@ -502,138 +508,153 @@ deps: ## ImageNet (small for tests) - tags: get,dataset,image-classification,imagenet,preprocessed names: - - imagenet-preprocessed + - imagenet-preprocessed enable_if_env: CM_MODEL: - - resnet50 + - resnet50 skip_if_env: CM_MLPERF_VISION_DATASET_OPTION: - - on + - on - tags: get,dataset,image-classification,imagenet,preprocessed,_pytorch names: - - imagenet-preprocessed + - imagenet-preprocessed enable_if_env: CM_MODEL: - - resnet50 + - resnet50 CM_MLPERF_VISION_DATASET_OPTION: - - imagenet_pytorch + - imagenet_pytorch - tags: get,dataset-aux,image-classification,imagenet-aux enable_if_env: CM_MODEL: - - resnet50 + - resnet50 ## Open Images for RetinaNet - tags: get,dataset,object-detection,open-images,openimages,preprocessed,_validation names: - - openimages-preprocessed + - openimages-preprocessed enable_if_env: CM_MODEL: - - retinanet + - retinanet ## CNNDM for Large Language Model - tags: get,dataset,cnndm,_validation names: - - cnndm-original + - cnndm-original enable_if_env: CM_MODEL: - - gptj-99 - - gptj-99.9 + - gptj-99 + - gptj-99.9 ## Squad for BERT - tags: get,dataset,squad,original names: - - squad-original + - squad-original enable_if_env: CM_MODEL: - - bert-99 - - bert-99.9 + - bert-99 + - bert-99.9 - tags: get,dataset-aux,squad-vocab enable_if_env: CM_MODEL: - - bert-99 - - bert-99.9 + - bert-99 + - bert-99.9 ## COCO for SDXL - tags: get,dataset,coco2014,_validation names: - - coco2014-preprocessed - - coco2014-dataset + - coco2014-preprocessed + - coco2014-dataset enable_if_env: CM_MODEL: - - stable-diffusion-xl + - stable-diffusion-xl ## OpenOrca for LLAMA2-70b - tags: get,preprocessed,dataset,openorca,_validation,_mlcommons names: - - openorca-preprocessed + - openorca-preprocessed enable_if_env: CM_MODEL: - - llama2-70b-99 - - llama2-70b-99.9 + - llama2-70b-99 + - llama2-70b-99.9 ## OpenOrca,mbxp,gsm8k combined dataset for mixtral-8x7b - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined names: - - openorca-mbxp-gsm8k-combined-preprocessed + - openorca-mbxp-gsm8k-combined-preprocessed enable_if_env: CM_MODEL: - - mixtral-8x7b + - mixtral-8x7b skip_if_env: CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - - 'yes' + - "yes" CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" ## Kits19 for 3d-unet - tags: get,dataset,kits19,preprocessed names: - - kits19-preprocessed + - kits19-preprocessed enable_if_env: CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 + - 3d-unet-99 + - 3d-unet-99.9 skip_if_env: CM_MLPERF_DATASET_3DUNET_DOWNLOAD_TO_HOST: - - 'yes' + - "yes" CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" ## Librispeech for rnnt - tags: get,dataset,librispeech,preprocessed names: - - librispeech-preprocessed + - librispeech-preprocessed enable_if_env: CM_MODEL: - - rnnt + - rnnt ## Criteo for dlrm - tags: get,dataset,criteo,preprocessed,_mlc names: - - criteo-preprocessed + - criteo-preprocessed enable_if_env: CM_MODEL: - - dlrm-v2-99 - - dlrm-v2-99.9 + - dlrm-v2-99 + - dlrm-v2-99.9 skip_if_env: CM_CRITEO_PREPROCESSED_PATH: - - on + - on ## igbh for rgat - tags: get,dataset,mlperf,inference,igbh names: - - igbh-dataset - - illinois-graph-benchmark-heterogeneous + - igbh-dataset + - illinois-graph-benchmark-heterogeneous enable_if_env: CM_MODEL: - - rgat + - rgat skip_if_env: CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" CM_USE_DATASET_FROM_HOST: - - 'yes' + - "yes" + + ## llama3_1 dataset + - tags: get,dataset,mlperf,inference,llama3,_validation + names: + - llama3_1-dataset + - llama3-dataset + enable_if_env: + CM_MODEL: + - llama3_1-405b + - llama3-402b + skip_if_env: + CM_USE_DATASET_FROM_HOST: + - "yes" + CM_RUN_STATE_DOCKER: + - "yes" ######################################################################## # Install MLPerf inference dependencies @@ -641,47 +662,46 @@ deps: # Creates user conf for given SUT - tags: generate,user-conf,mlperf,inference names: - - user-conf-generator + - user-conf-generator skip_if_env: CM_RUN_STATE_DOCKER: - - 'yes' + - "yes" # Install MLPerf loadgen - tags: get,loadgen names: - - loadgen - - mlperf-inference-loadgen + - loadgen + - mlperf-inference-loadgen # Download MLPerf inference source - tags: get,mlcommons,inference,src names: - - inference-src - + - inference-src # Download MLPerf inference source - tags: get,mlcommons,inference,src env: - CM_GET_MLPERF_IMPLEMENTATION_ONLY: 'yes' + CM_GET_MLPERF_IMPLEMENTATION_ONLY: "yes" names: - - mlperf-implementation + - mlperf-implementation - tags: get,generic-python-lib,_package.psutil prehook_deps: - names: - - remote-run-cmds + - remote-run-cmds tags: remote,run,cmds enable_if_env: CM_ASSH_RUN_COMMANDS: - - "on" + - "on" -posthook_deps: +posthook_deps: - names: - - mlperf-runner + - mlperf-runner tags: benchmark-mlperf skip_if_env: CM_MLPERF_SKIP_RUN: - - "on" + - "on" post_deps: - tags: save,mlperf,inference,state @@ -697,7 +717,7 @@ variations: imagenet-accuracy-script: tags: _float32 env: - CM_MLPERF_PYTHON: 'yes' + CM_MLPERF_PYTHON: "yes" CM_MLPERF_IMPLEMENTATION: reference # ML engine @@ -821,14 +841,14 @@ variations: CM_MLPERF_BACKEND: deepsparse CM_MLPERF_BACKEND_VERSION: <<>> deps: - - tags: get,generic-python-lib,_deepsparse - skip_if_env: - CM_HOST_PLATFORM_FLAVOR: - - aarch64 - - tags: get,generic-python-lib,_package.deepsparse-nightly - enable_if_env: - CM_HOST_PLATFORM_FLAVOR: - - aarch64 + - tags: get,generic-python-lib,_deepsparse + skip_if_env: + CM_HOST_PLATFORM_FLAVOR: + - aarch64 + - tags: get,generic-python-lib,_package.deepsparse-nightly + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - aarch64 add_deps_recursive: mlperf-implementation: version: deepsparse @@ -841,92 +861,91 @@ variations: CM_MLPERF_BACKEND: tvm-onnx CM_MLPERF_BACKEND_VERSION: <<>> deps: - - tags: get,generic-python-lib,_onnx - - tags: get,generic-python-lib,_numpy - version_max: "1.26.4" - version_max_usable: "1.26.4" - - tags: get,tvm - names: - - tvm - - tags: get,tvm-model,_onnx - names: - - tvm-model - update_tags_from_env_with_prefix: - _model.: - - CM_MODEL - - + - tags: get,generic-python-lib,_onnx + - tags: get,generic-python-lib,_numpy + version_max: "1.26.4" + version_max_usable: "1.26.4" + - tags: get,tvm + names: + - tvm + - tags: get,tvm-model,_onnx + names: + - tvm-model + update_tags_from_env_with_prefix: + _model.: + - CM_MODEL + tvm-tflite: group: framework env: CM_MLPERF_BACKEND: tvm-tflite CM_MLPERF_BACKEND_VERSION: <<>> deps: - - tags: get,generic-python-lib,_tflite - - tags: get,tvm - names: - - tvm - - tags: get,tvm-model,_tflite - names: - - tvm-model - update_tags_from_env_with_prefix: - _model.: - - CM_MODEL + - tags: get,generic-python-lib,_tflite + - tags: get,tvm + names: + - tvm + - tags: get,tvm-model,_tflite + names: + - tvm-model + update_tags_from_env_with_prefix: + _model.: + - CM_MODEL tvm-pytorch: group: framework env: CM_MLPERF_BACKEND: tvm-pytorch CM_MLPERF_BACKEND_VERSION: <<>> - CM_PREPROCESS_PYTORCH: 'yes' + CM_PREPROCESS_PYTORCH: "yes" MLPERF_TVM_TORCH_QUANTIZED_ENGINE: qnnpack deps: - - tags: get,generic-python-lib,_torch - names: - - torch - - pytorch - - tags: get,tvm - names: - - tvm - - tags: get,tvm-model,_pytorch - names: - - tvm-model - update_tags_from_env_with_prefix: - _model.: - - CM_MODEL + - tags: get,generic-python-lib,_torch + names: + - torch + - pytorch + - tags: get,tvm + names: + - tvm + - tags: get,tvm-model,_pytorch + names: + - tvm-model + update_tags_from_env_with_prefix: + _model.: + - CM_MODEL # Reference MLPerf models gptj-99.9: group: models base: - - gptj_ + - gptj_ env: CM_MODEL: gptj-99.9 gptj-99: group: models base: - - gptj_ + - gptj_ env: CM_MODEL: gptj-99 gptj_: deps: - - tags: get,generic-python-lib,_package.datasets - - tags: get,generic-python-lib,_package.attrs - - tags: get,generic-python-lib,_package.accelerate + - tags: get,generic-python-lib,_package.datasets + - tags: get,generic-python-lib,_package.attrs + - tags: get,generic-python-lib,_package.accelerate bert-99.9: group: models base: - - bert + - bert env: CM_MODEL: bert-99.9 bert-99: group: models base: - - bert + - bert env: CM_MODEL: bert-99 @@ -934,29 +953,29 @@ variations: env: CM_MLPERF_MODEL_SKIP_BATCHING: true deps: - - tags: get,generic-python-lib,_package.pydantic - - tags: get,generic-python-lib,_tokenization - - tags: get,generic-python-lib,_six - - tags: get,generic-python-lib,_package.absl-py - - tags: get,generic-python-lib,_protobuf - names: - - protobuf - version_max: "3.19" - enable_if_env: - CM_MLPERF_BACKEND: - - tf - - tflite - - tags: get,generic-python-lib,_boto3 - enable_if_env: - CM_MLPERF_BACKEND: - - pytorch - - tags: get,generic-python-lib,_torch - names: - - ml-engine-pytorch - - pytorch - skip_if_env: - CM_MLPERF_DEVICE: - - gpu + - tags: get,generic-python-lib,_package.pydantic + - tags: get,generic-python-lib,_tokenization + - tags: get,generic-python-lib,_six + - tags: get,generic-python-lib,_package.absl-py + - tags: get,generic-python-lib,_protobuf + names: + - protobuf + version_max: "3.19" + enable_if_env: + CM_MLPERF_BACKEND: + - tf + - tflite + - tags: get,generic-python-lib,_boto3 + enable_if_env: + CM_MLPERF_BACKEND: + - pytorch + - tags: get,generic-python-lib,_torch + names: + - ml-engine-pytorch + - pytorch + skip_if_env: + CM_MLPERF_DEVICE: + - gpu add_deps_recursive: inference-src: tags: _deeplearningexamples @@ -1101,7 +1120,7 @@ variations: - tags: get,generic-python-lib,_mxeval names: - rouge-score - + mixtral-8x7b,cuda: default_env: CM_MLPERF_LOADGEN_BATCH_SIZE: 1 @@ -1109,14 +1128,14 @@ variations: 3d-unet-99.9: group: models base: - - 3d-unet + - 3d-unet env: CM_MODEL: 3d-unet-99.9 3d-unet-99: group: models base: - - 3d-unet + - 3d-unet env: CM_MODEL: 3d-unet-99 @@ -1125,23 +1144,23 @@ variations: CM_TMP_IGNORE_MLPERF_QUERY_COUNT: true CM_MLPERF_MODEL_SKIP_BATCHING: true deps: - - tags: get,generic-python-lib,_package.nibabel - - tags: get,generic-python-lib,_package.scipy - names: - - scipy - version: 1.10.1 + - tags: get,generic-python-lib,_package.nibabel + - tags: get,generic-python-lib,_package.scipy + names: + - scipy + version: 1.10.1 dlrm-v2-99.9: group: models base: - - dlrm-v2_ + - dlrm-v2_ env: CM_MODEL: dlrm-v2-99.9 dlrm-v2-99: group: models base: - - dlrm-v2_ + - dlrm-v2_ env: CM_MODEL: dlrm-v2-99 @@ -1152,34 +1171,33 @@ variations: dlrm-v2_,pytorch: deps: - - tags: get,dlrm,src - names: - - dlrm-src - # to force the version - - tags: get,generic-python-lib,_torch - names: - - torch - - pytorch - - ml-engine-pytorch - - tags: get,generic-python-lib,_mlperf_logging - - tags: get,generic-python-lib,_opencv-python - - tags: get,generic-python-lib,_tensorboard - - tags: get,generic-python-lib,_protobuf - - tags: get,generic-python-lib,_scikit-learn - - tags: get,generic-python-lib,_tqdm - - tags: get,generic-python-lib,_onnx - - tags: get,generic-python-lib,_numpy - names: - - numpy - - tags: get,generic-python-lib,_package.pyre-extensions - - tags: get,generic-python-lib,_package.torchsnapshot - - tags: get,generic-python-lib,_package.torchmetrics - - tags: get,generic-python-lib,_package.torchrec - - tags: get,generic-python-lib,_package.fbgemm-gpu - - tags: get,generic-python-lib,_package.fbgemm-gpu-cpu - - tags: get,generic-python-lib,_package.fvcore - - tags: set,user,limit,_large-nofile - + - tags: get,dlrm,src + names: + - dlrm-src + # to force the version + - tags: get,generic-python-lib,_torch + names: + - torch + - pytorch + - ml-engine-pytorch + - tags: get,generic-python-lib,_mlperf_logging + - tags: get,generic-python-lib,_opencv-python + - tags: get,generic-python-lib,_tensorboard + - tags: get,generic-python-lib,_protobuf + - tags: get,generic-python-lib,_scikit-learn + - tags: get,generic-python-lib,_tqdm + - tags: get,generic-python-lib,_onnx + - tags: get,generic-python-lib,_numpy + names: + - numpy + - tags: get,generic-python-lib,_package.pyre-extensions + - tags: get,generic-python-lib,_package.torchsnapshot + - tags: get,generic-python-lib,_package.torchmetrics + - tags: get,generic-python-lib,_package.torchrec + - tags: get,generic-python-lib,_package.fbgemm-gpu + - tags: get,generic-python-lib,_package.fbgemm-gpu-cpu + - tags: get,generic-python-lib,_package.fvcore + - tags: set,user,limit,_large-nofile rnnt: group: models @@ -1188,51 +1206,51 @@ variations: CM_MLPERF_MODEL_SKIP_BATCHING: true CM_TMP_IGNORE_MLPERF_QUERY_COUNT: true deps: - - tags: get,generic-python-lib,_package.pydantic - version_max: "1.10.9" - - tags: get,generic-python-lib,_librosa - names: - - librosa - - tags: get,generic-python-lib,_inflect - - tags: get,generic-python-lib,_unidecode - - tags: get,generic-python-lib,_toml + - tags: get,generic-python-lib,_package.pydantic + version_max: "1.10.9" + - tags: get,generic-python-lib,_librosa + names: + - librosa + - tags: get,generic-python-lib,_inflect + - tags: get,generic-python-lib,_unidecode + - tags: get,generic-python-lib,_toml retinanet: group: models deps: - - tags: get,generic-python-lib,_opencv-python - - tags: get,generic-python-lib,_numpy - names: - - numpy - - tags: get,generic-python-lib,_pycocotools + - tags: get,generic-python-lib,_opencv-python + - tags: get,generic-python-lib,_numpy + names: + - numpy + - tags: get,generic-python-lib,_pycocotools env: CM_MODEL: retinanet - CM_MLPERF_USE_MLCOMMONS_RUN_SCRIPT: 'yes' - CM_MLPERF_LOADGEN_MAX_BATCHSIZE: '1' + CM_MLPERF_USE_MLCOMMONS_RUN_SCRIPT: "yes" + CM_MLPERF_LOADGEN_MAX_BATCHSIZE: "1" resnet50: group: models default: true env: CM_MODEL: resnet50 - CM_MLPERF_USE_MLCOMMONS_RUN_SCRIPT: 'yes' + CM_MLPERF_USE_MLCOMMONS_RUN_SCRIPT: "yes" deps: - - tags: get,generic-python-lib,_opencv-python - - tags: get,generic-python-lib,_numpy - names: - - numpy - - tags: get,generic-python-lib,_pycocotools + - tags: get,generic-python-lib,_opencv-python + - tags: get,generic-python-lib,_numpy + names: + - numpy + - tags: get,generic-python-lib,_pycocotools prehook_deps: - - tags: get,generic-python-lib,_protobuf - names: - - protobuf - version_max: "4.23.4" - version_max_usable: "4.23.4" - enable_if_env: - CM_MLPERF_BACKEND: - - tf - - tflite + - tags: get,generic-python-lib,_protobuf + names: + - protobuf + version_max: "4.23.4" + version_max_usable: "4.23.4" + enable_if_env: + CM_MLPERF_BACKEND: + - tf + - tflite rgat: group: models @@ -1252,42 +1270,72 @@ variations: - tags: get,generic-python-lib,_package.numpy version_max: "1.26.4" version_max_usable: "1.26.4" - - tags: get,generic-python-lib,_package.pydantic + - tags: get,generic-python-lib,_package.pydantic - tags: get,generic-python-lib,_package.igb,_url.git+https://github.com/IllinoisGraphBenchmark/IGB-Datasets.git - tags: get,generic-python-lib,_package.torch-geometric update_tags_from_env_with_prefix: - _find_links_url.: + _find_links_url.: - CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL - tags: get,generic-python-lib,_package.torch-scatter update_tags_from_env_with_prefix: - _find_links_url.: + _find_links_url.: - CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL - tags: get,generic-python-lib,_package.torch-sparse update_tags_from_env_with_prefix: - _find_links_url.: + _find_links_url.: - CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL - tags: get,generic-python-lib,_package.dgl update_tags_from_env_with_prefix: - _find_links_url.: + _find_links_url.: - CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL - + rgat,cuda: env: CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>.html" CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL: "https://data.dgl.ai/wheels/torch-<<>>/cu121/repo.html" - + rgat,cpu: env: CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>+cpu.html" CM_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL: "https://data.dgl.ai/wheels/torch-<<>>/repo.html" + llama3_1-405b: + group: models + env: + CM_MODEL: llama3_1-405b + adr: + pytorch: + version_max: 2.5.1 + CM_MODEL: llama3-402b + deps: + - tags: get,generic-python-lib,_package.torchvision + - tags: get,generic-python-lib,_package.torchaudio + - tags: get,generic-python-lib,_package.torch-geometric + - tags: get,generic-python-lib,_package.transformers + - tags: get,generic-python-lib,_package.sentencepiece + - tags: get,generic-python-lib,_package.accelerate + - tags: get,generic-python-lib,_package.vllm + env: + CM_GENERIC_PYTHON_PIP_EXTRA: "--upgrade" + - tags: get,generic-python-lib,_package.pybind11 + - tags: get,generic-python-lib,_package.pandas + version_max: 2.2.1 + + llama3_1-405b,cuda: + env: + CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>.html" + + llama3_1-405b,cpu: + env: + CM_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>+cpu.html" + # Target devices cpu: group: device default: true env: CM_MLPERF_DEVICE: cpu - CUDA_VISIBLE_DEVICES: '' + CUDA_VISIBLE_DEVICES: "" USE_CUDA: no USE_GPU: no @@ -1334,8 +1382,7 @@ variations: default: true add_deps_recursive: ml-model: - tags: - _fp32 + tags: _fp32 env: CM_MLPERF_QUANTIZATION: off CM_MLPERF_MODEL_PRECISION: float32 @@ -1345,8 +1392,7 @@ variations: group: precision add_deps_recursive: ml-model-float16: - tags: - _fp16 + tags: _fp16 env: CM_MLPERF_QUANTIZATION: off CM_MLPERF_MODEL_PRECISION: float16 @@ -1356,8 +1402,7 @@ variations: group: precision add_deps_recursive: ml-model-float16: - tags: - _fp16 + tags: _fp16 env: CM_MLPERF_QUANTIZATION: off CM_MLPERF_MODEL_PRECISION: bfloat16 @@ -1369,8 +1414,7 @@ variations: CM_MLPERF_MODEL_PRECISION: int8 add_deps_recursive: ml-model: - tags: - _int8 + tags: _int8 quantized: alias: int8 @@ -1381,11 +1425,9 @@ variations: CM_MLPERF_LOADGEN_MAX_BATCHSIZE: "#" add_deps_recursive: ml-model: - tags: - _batch_size.# + tags: _batch_size.# tvm-model: - tags: - _batch_size.# + tags: _batch_size.# network-sut: group: network @@ -1417,6 +1459,6 @@ variations: loadgen: version: r2.1 env: - CM_RERUN: 'yes' - CM_SKIP_SYS_UTILS: 'yes' - CM_TEST_QUERY_COUNT: '100' + CM_RERUN: "yes" + CM_SKIP_SYS_UTILS: "yes" + CM_TEST_QUERY_COUNT: "100" diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 1a2d3b023..cbb8b9bc3 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -68,7 +68,7 @@ def preprocess(i): str(env['CM_MLPERF_LOADGEN_BATCH_SIZE']) if env.get('CM_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get('CM_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and ( - env['CM_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['CM_MODEL'] or 'llama2' in env['CM_MODEL'] or 'mixtral' in env['CM_MODEL']) and env.get('CM_MLPERF_RUN_STYLE', '') != "valid": + env['CM_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['CM_MODEL'] or 'llama2' in env['CM_MODEL'] or 'mixtral' in env['CM_MODEL'] or 'llama3' in env['CM_MODEL']) and env.get('CM_MLPERF_RUN_STYLE', '') != "valid": env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \ env['CM_MLPERF_LOADGEN_QUERY_COUNT'] @@ -127,7 +127,7 @@ def preprocess(i): if 'CM_MLPERF_USER_CONF' in env: user_conf_path = env['CM_MLPERF_USER_CONF'] x = "" if os_info['platform'] == 'windows' else "'" - if 'llama2-70b' in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: + if 'llama2-70b' in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"] or "llama3" in env["CM_MODEL"]: scenario_extra_options += " --user-conf " + x + user_conf_path + x else: scenario_extra_options += " --user_conf " + x + user_conf_path + x @@ -500,6 +500,30 @@ def get_run_cmd_reference( if env.get('CM_ACTIVATE_RGAT_IN_MEMORY', '') == "yes": cmd += " --in-memory " + elif "llama3" in env['CM_MODEL']: + env['RUN_DIR'] = os.path.join( + env['CM_MLPERF_INFERENCE_SOURCE'], + "language", + "llama3.1-405b") + + if int(env.get('CM_MLPERF_INFERENCE_TP_SIZE', '')) > 1: + env['VLLM_WORKER_MULTIPROC_METHOD'] = "spawn" + + cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \ + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \ + " --dataset-path " + env['CM_DATASET_LLAMA3_PATH'] + \ + " --output-log-dir " + env['CM_MLPERF_OUTPUT_DIR'] + \ + ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ + " --model-path " + env['CM_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + \ + " --tensor-parallel-size " + env['CM_MLPERF_INFERENCE_TP_SIZE'] + \ + " --vllm " + + if env.get('CM_MLPERF_INFERENCE_NUM_WORKERS', '') != '': + cmd += f" --num-workers {env['CM_MLPERF_INFERENCE_NUM_WORKERS']}" + + cmd = cmd.replace("--count", "--total-sample-count") + cmd = cmd.replace("--max-batchsize", "--batch-size") + if env.get('CM_NETWORK_LOADGEN', '') in ["lon", "sut"]: cmd = cmd + " " + "--network " + env['CM_NETWORK_LOADGEN'] if env.get('CM_NETWORK_LOADGEN_SUT_SERVERS', []): diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index c50df363b..7596b30ef 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -221,6 +221,8 @@ variations: tags: _int32 cnndm-accuracy-script: tags: _int32 + llama3_1-405b-accuracy-script: + tags: _int32 env: CM_MLPERF_PYTHON: 'yes' CM_MLPERF_IMPLEMENTATION: mlcommons_python @@ -272,6 +274,10 @@ variations: default_variations: backend: pytorch + reference,llama3_1-405b: + default_variations: + backend: pytorch + reference,mixtral-8x7b: default_variations: backend: pytorch @@ -795,6 +801,40 @@ variations: - igbh-original - igbh-dataset + llama3_1-405b: + group: + model + add_deps_recursive: + mlperf-inference-implementation: + tags: _llama3_1-405b + env: + CM_MODEL: + llama3_1-405b + posthook_deps: + - enable_if_env: + CM_MLPERF_LOADGEN_MODE: + - accuracy + - all + CM_MLPERF_ACCURACY_RESULTS_DIR: + - 'on' + skip_if_env: + CM_MLPERF_IMPLEMENTATION: + - nvidia + names: + - mlperf-accuracy-script + - llama3_1-405b-accuracy-script + tags: run,accuracy,mlperf,_dataset_llama3 + docker: + deps: + - tags: get,ml-model,llama3 + enable_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' + names: + - llama3_1-405b + - llama3-405b + + sdxl: group: model @@ -1682,6 +1722,8 @@ variations: tags: _mlcommons intel-harness: tags: _v4.1 + inference-src: + version: r5.0 default_env: CM_SKIP_SYS_UTILS: 'yes' CM_REGENERATE_MEASURE_FILES: 'yes' diff --git a/script/generate-mlperf-inference-submission/_cm.yaml b/script/generate-mlperf-inference-submission/_cm.yaml index 8dad87f25..013997df9 100644 --- a/script/generate-mlperf-inference-submission/_cm.yaml +++ b/script/generate-mlperf-inference-submission/_cm.yaml @@ -69,6 +69,7 @@ input_mapping: device: CM_MLPERF_DEVICE division: CM_MLPERF_SUBMISSION_DIVISION duplicate: CM_MLPERF_DUPLICATE_SCENARIO_RESULTS + extra_checker_args: CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARG hw_name: CM_HW_NAME hw_notes_extra: CM_MLPERF_SUT_HW_NOTES_EXTRA infer_scenario_results: CM_MLPERF_DUPLICATE_SCENARIO_RESULTS diff --git a/script/get-dataset-mlperf-inference-llama3/_cm.yaml b/script/get-dataset-mlperf-inference-llama3/_cm.yaml new file mode 100644 index 000000000..d8af83b88 --- /dev/null +++ b/script/get-dataset-mlperf-inference-llama3/_cm.yaml @@ -0,0 +1,56 @@ +alias: get-dataset-mlperf-inference-llama3 +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- dataset +- mlperf +- llama3 +- inference +uid: c3bc69599cbc4db7 +new_env_keys: + - CM_DATASET_LLAMA3_PATH +input_mapping: + outdirname: CM_OUTDIRNAME +prehook_deps: + - env: + CM_DOWNLOAD_FINAL_ENV_NAME: CM_DATASET_LLAMA3_PATH + CM_EXTRACT_TO_FOLDER: llama-3-dataset + extra_cache_tags: dataset,llama3 + force_cache: true + enable_if_env: + CM_TMP_REQUIRE_DOWNLOAD: + - 'yes' + names: + - dae + tags: download-and-extract + update_tags_from_env_with_prefix: + _url.: + - CM_DOWNLOAD_URL +variations: + validation: + default: true + group: dataset-type + env: + CM_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + CM_DATASET_TYPE: validation + CM_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + calibration: + group: dataset-type + env: + CM_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + CM_DATASET_TYPE: calibration + CM_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + rclone: + add_deps_recursive: + dae: + tags: _rclone + default: true + env: + CM_DOWNLOAD_FILENAME: checkpoint + CM_DOWNLOAD_URL: <<>> + CM_RCLONE_CONFIG_NAME: mlc-inference + group: download-tool +print_env_at_the_end: + CM_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-dataset-mlperf-inference-llama3/customize.py b/script/get-dataset-mlperf-inference-llama3/customize.py new file mode 100644 index 000000000..745dc52fe --- /dev/null +++ b/script/get-dataset-mlperf-inference-llama3/customize.py @@ -0,0 +1,31 @@ +from cmind import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('CM_DATASET_LLAMA3_PATH', '') == '': + env['CM_TMP_REQUIRE_DOWNLOAD'] = "yes" + + if env.get('CM_OUTDIRNAME', '') != '': + env['CM_DOWNLOAD_PATH'] = env['CM_OUTDIRNAME'] + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + if env.get('CM_TMP_REQUIRE_DOWNLOAD', '') == "yes": + env['CM_DATASET_LLAMA3_PATH'] = os.path.join( + env['CM_DATASET_LLAMA3_PATH'], env['CM_DATASET_FILE_NAME']) + + return {'return': 0} diff --git a/script/get-generic-python-lib/detect-version.py b/script/get-generic-python-lib/detect-version.py index 001c39b37..fc879f04e 100644 --- a/script/get-generic-python-lib/detect-version.py +++ b/script/get-generic-python-lib/detect-version.py @@ -2,6 +2,7 @@ import sys package_name = os.environ.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '') +package_name = package_name.split("[")[0] filename = 'tmp-ver.out' diff --git a/script/get-huggingface-cli/_cm.yaml b/script/get-huggingface-cli/_cm.yaml new file mode 100644 index 000000000..6643eb222 --- /dev/null +++ b/script/get-huggingface-cli/_cm.yaml @@ -0,0 +1,26 @@ +alias: get-huggingface-cli +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +can_force_cache: true +category: DevOps automation +clean_files: [] +deps: +- tags: detect,os +tags: +- get +- huggingface +- hf-cli +- huggingface-cli +- cli +input_mapping: + token: CM_HF_TOKEN +uid: e9488a272f1d4160 +deps: + - tags: get,generic-python-lib,_package.huggingface_hub[cli] +variations: + with-login: + cache: true + force_cache: true + env: + CM_HF_DO_LOGIN: yes diff --git a/script/get-huggingface-cli/customize.py b/script/get-huggingface-cli/customize.py new file mode 100644 index 000000000..d9e63e42c --- /dev/null +++ b/script/get-huggingface-cli/customize.py @@ -0,0 +1,30 @@ +from cmind import utils +import os + + +def preprocess(i): + env = i['env'] + if env.get('CM_HF_TOKEN', '') != '': + env['CM_HF_LOGIN_CMD'] = f"""git config --global credential.helper store && huggingface-cli login --token {env['CM_HF_TOKEN']} --add-to-git-credential +""" + elif str(env.get('CM_HF_DO_LOGIN')).lower() in ["yes", "1", "true"]: + env['CM_HF_LOGIN_CMD'] = f"""git config --global credential.helper store && huggingface-cli login +""" + return {'return': 0} + + +def postprocess(i): + env = i['env'] + + r = i['automation'].parse_version({'match_text': r'huggingface_hub\s*version:\s*([\d.]+)', + 'group_number': 1, + 'env_key': 'CM_GITHUBCLI_VERSION', + 'which_env': i['env']}) + if r['return'] > 0: + return r + + version = r['version'] + + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} diff --git a/script/get-huggingface-cli/run.bat b/script/get-huggingface-cli/run.bat new file mode 100644 index 000000000..464afe5c7 --- /dev/null +++ b/script/get-huggingface-cli/run.bat @@ -0,0 +1,14 @@ +@echo off +REM Check if the environment variable CM_HF_LOGIN_CMD is defined and not empty +IF DEFINED CM_HF_LOGIN_CMD ( + echo %CM_HF_LOGIN_CMD% + call %CM_HF_LOGIN_CMD% + IF ERRORLEVEL 1 ( + echo Command failed with error code %ERRORLEVEL% + exit /b %ERRORLEVEL% + ) +) + +REM Run the Hugging Face CLI version command and save output +huggingface-cli version > tmp-ver.out + diff --git a/script/get-huggingface-cli/run.sh b/script/get-huggingface-cli/run.sh new file mode 100644 index 000000000..43d20f367 --- /dev/null +++ b/script/get-huggingface-cli/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash +if [[ -n ${CM_HF_LOGIN_CMD} ]]; then + echo "${CM_HF_LOGIN_CMD}" + eval ${CM_HF_LOGIN_CMD} + test $? -eq 0 || exit $? +fi +huggingface-cli version > tmp-ver.out diff --git a/script/get-ml-model-huggingface-zoo/_cm.yaml b/script/get-ml-model-huggingface-zoo/_cm.yaml index 154675bc1..b8235a57d 100644 --- a/script/get-ml-model-huggingface-zoo/_cm.yaml +++ b/script/get-ml-model-huggingface-zoo/_cm.yaml @@ -35,6 +35,10 @@ uid: 53cf8252a443446a variations: clone-repo: deps: + - tags: get,hf-cli,_with-login + enable_if_env: + CM_HF_TOKEN: + - on - env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ML_MODEL_PATH tags: get,git,repo,_lfs diff --git a/script/get-ml-model-llama3/_cm.yaml b/script/get-ml-model-llama3/_cm.yaml new file mode 100644 index 000000000..376553823 --- /dev/null +++ b/script/get-ml-model-llama3/_cm.yaml @@ -0,0 +1,68 @@ +alias: get-ml-model-llama3 +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: AI/ML models +input_mapping: + outdirname: CM_OUTDIRNAME +new_env_keys: +- CM_ML_MODEL_* +- LLAMA3_CHECKPOINT_PATH +prehook_deps: +- enable_if_env: + CM_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: {} + extra_cache_tags: llama3,llama-3 + force_env_keys: + - CM_GIT_CHECKOUT_FOLDER + names: + - hf-zoo + tags: get,ml-model,huggingface,zoo,_clone-repo +print_env_at_the_end: + LLAMA3_CHECKPOINT_PATH: LLAMA3 checkpoint path +tags: +- get +- raw +- ml-model +- language-processing +- llama3 +- llama3-405b +uid: 2f8cef2acc334e80 +variations: + fp16: + default: true + env: + CM_ML_MODEL_INPUT_DATA_TYPES: fp16 + CM_ML_MODEL_PRECISION: fp16 + CM_ML_MODEL_WEIGHT_DATA_TYPES: fp16 + group: precision + meta-llama/Llama-3.1-405B-Instruct: + adr: + hf-zoo: + tags: _model-stub.meta-llama/Llama-3.1-405B-Instruct + default: true + env: + CM_ML_MODEL_NAME: Llama-3-405b-instruct + CM_MODEL_ZOO_ENV_KEY: LLAMA3 + group: huggingface-stub + meta-llama/Llama-3.1-8B-Instruct: + adr: + hf-zoo: + tags: _model-stub.meta-llama/Llama-3.1-8B-Instruct + env: + CM_ML_MODEL_NAME: Llama-3-8b-instruct + CM_MODEL_ZOO_ENV_KEY: LLAMA3 + group: huggingface-stub + vllm: + default: true + env: + CM_ML_MODEL_FRAMEWORK: vllm + group: framework + stub.#: + adr: + hf-zoo: + tags: _model-stub.# + env: + CM_MODEL_ZOO_ENV_KEY: LLAMA3 + group: huggingface-stub diff --git a/script/get-ml-model-llama3/customize.py b/script/get-ml-model-llama3/customize.py new file mode 100644 index 000000000..9ec7edecd --- /dev/null +++ b/script/get-ml-model-llama3/customize.py @@ -0,0 +1,35 @@ +from cmind import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + + # skip download and register in cache if the llama3 checkpoint path is + # already defined by the user + if env.get('CM_ML_MODEL_LLAMA3_CHECKPOINT_PATH', '') != '': + env['LLAMA3_CHECKPOINT_PATH'] = env['CM_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + return {'return': 0} + + path = env.get('CM_OUTDIRNAME', '').strip() + + if path != "": + os.makedirs(path, exist_ok=True) + env['CM_GIT_CHECKOUT_FOLDER'] = os.path.join( + path, env['CM_ML_MODEL_NAME']) + + env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + env['CM_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] = env['LLAMA3_CHECKPOINT_PATH'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_PATH'] + + return {'return': 0} diff --git a/script/get-ml-model-mixtral/_cm.yaml b/script/get-ml-model-mixtral/_cm.yaml index 5b61ef9ca..358d56318 100644 --- a/script/get-ml-model-mixtral/_cm.yaml +++ b/script/get-ml-model-mixtral/_cm.yaml @@ -6,6 +6,8 @@ category: AI/ML models env: CM_ML_MODEL_DATASET: '' CM_ML_MODEL_WEIGHT_TRANSFORMATIONS: 'no' +docker: + real_run: False input_mapping: checkpoint: MIXTRAL_CHECKPOINT_PATH new_env_keys: diff --git a/script/get-mlperf-inference-src/_cm.yaml b/script/get-mlperf-inference-src/_cm.yaml index 0ff1d31e4..a9f7410a5 100644 --- a/script/get-mlperf-inference-src/_cm.yaml +++ b/script/get-mlperf-inference-src/_cm.yaml @@ -50,8 +50,6 @@ prehook_deps: _submodules.: - CM_GIT_SUBMODULES print_env_at_the_end_disabled: - CM_MLPERF_INFERENCE_CONF_PATH: Path to the MLPerf inference benchmark configuration - file CM_MLPERF_INFERENCE_SOURCE: Path to MLPerf inference benchmark sources tags: - get @@ -154,31 +152,36 @@ versions: CM_MLPERF_LAST_RELEASE: v2.1 CM_TMP_GIT_CHECKOUT: v2.1 r3.0: - adr: + ad: inference-git-repo: tags: _tag.v3.0 env: CM_MLPERF_LAST_RELEASE: v3.0 CM_TMP_GIT_CHECKOUT: '' r3.1: - adr: + ad: inference-git-repo: tags: _tag.v3.1 env: CM_MLPERF_LAST_RELEASE: v3.1 - CM_TMP_GIT_CHECKOUT: '' + CM_GIT_CHECKOUT_TAG: 'v3.1' r4.0: - adr: + ad: inference-git-repo: tags: _tag.v4.0 env: CM_MLPERF_LAST_RELEASE: v4.0 + CM_GIT_CHECKOUT_TAG: 'v4.0' r4.1: - adr: + ad: inference-git-repo: tags: _tag.v4.1 env: CM_MLPERF_LAST_RELEASE: v4.1 + CM_GIT_CHECKOUT_TAG: 'v4.1' + r5.0: + env: + CM_MLPERF_LAST_RELEASE: v5.0 tvm: env: CM_MLPERF_LAST_RELEASE: v3.1 diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index 1a62da3b6..8c545d678 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -41,7 +41,8 @@ def preprocess(i): # if not try to assign the values specified in version parameters, # if version parameters does not have the value to a parameter, set the # default one - if env.get('CM_GIT_CHECKOUT', '') == '': + if env.get('CM_GIT_CHECKOUT', '') == '' and env.get( + 'CM_GIT_CHECKOUT_TAG', '') == '': if env.get('CM_TMP_GIT_CHECKOUT', '') != '': env["CM_GIT_CHECKOUT"] = env["CM_TMP_GIT_CHECKOUT"] else: diff --git a/script/process-mlperf-accuracy/_cm.yaml b/script/process-mlperf-accuracy/_cm.yaml index 59544fd3a..3b80194d4 100644 --- a/script/process-mlperf-accuracy/_cm.yaml +++ b/script/process-mlperf-accuracy/_cm.yaml @@ -265,3 +265,7 @@ variations: env: CM_DATASET: igbh group: dataset + dataset_llama3: + env: + CM_DATASET: dataset_llama3 + group: dataset diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 21569fd57..738b21697 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -199,6 +199,10 @@ def preprocess(i): CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join( result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' --output-file '" + out_file + "'" + elif dataset == "dataset_llama3": + CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "llama3.1-405b", "evaluate-accuracy.py") + "' --checkpoint-path '" + env['CM_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --dtype '" + env['CM_ACCURACY_DTYPE'] + "' --dataset-file '" + env['CM_DATASET_LLAMA3_PATH'] + "' > '" + out_file + "'" + else: return {'return': 1, 'error': 'Unsupported dataset'} diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index ea2185060..29effb5c2 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -35,6 +35,7 @@ default_env: CM_MLPERF_SKIP_SUBMISSION_GENERATION: no CM_DOCKER_PRIVILEGED_MODE: yes CM_MLPERF_SUBMISSION_DIVISION: open + CM_MLPERF_INFERENCE_TP_SIZE: 1 input_mapping: api_server: CM_MLPERF_INFERENCE_API_SERVER @@ -109,6 +110,7 @@ input_mapping: sut: CM_MLPERF_INFERENCE_SUT_VARIATION nvidia_llama2_dataset_file_path: CM_NVIDIA_LLAMA_DATASET_FILE_PATH tp_size: CM_NVIDIA_TP_SIZE + vllm_tp_size: CM_MLPERF_INFERENCE_TP_SIZE vllm_model_name: CM_VLLM_SERVER_MODEL_NAME num_workers: CM_MLPERF_INFERENCE_NUM_WORKERS max_test_duration: CM_MLPERF_MAX_DURATION_TEST @@ -463,6 +465,7 @@ input_description: - mobilenet - efficientnet - rgat + - llama3_1-405b default: resnet50 desc: MLPerf model sort: 200 diff --git a/script/run-mlperf-inference-submission-checker/_cm.yaml b/script/run-mlperf-inference-submission-checker/_cm.yaml index 84e712a40..a302e5d12 100644 --- a/script/run-mlperf-inference-submission-checker/_cm.yaml +++ b/script/run-mlperf-inference-submission-checker/_cm.yaml @@ -36,6 +36,7 @@ deps: tags: preprocess,mlperf,inference,submission input_mapping: extra_args: CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS + extra_checker_args: CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS extra_model_benchmark_map: CM_MLPERF_EXTRA_MODEL_MAPPING input: CM_MLPERF_INFERENCE_SUBMISSION_DIR power: CM_MLPERF_POWER @@ -50,6 +51,7 @@ input_mapping: src_version: CM_MLPERF_SUBMISSION_CHECKER_VERSION submission_dir: CM_MLPERF_INFERENCE_SUBMISSION_DIR submitter: CM_MLPERF_SUBMITTER + submitter_id: CM_MLPERF_SUBMITTER_ID tar: CM_TAR_SUBMISSION_DIR post_deps: - enable_if_env: @@ -66,6 +68,12 @@ post_deps: CM_TAR_SUBMISSION_DIR: - 'yes' tags: run,tar +- enable_if_env: + CM_SUBMITTER_ID: + - 'yes' + tags: submit,mlperf,results,_inference + env: + CM_MLPERF_SUBMISSION_FILE: <<>> tags: - run - mlc